12cddd2bcSIrynaimport unittest
22cddd2bcSIrynaimport sys
32cddd2bcSIrynaimport time
42cddd2bcSIrynaimport types
52cddd2bcSIrynaimport logger
62cddd2bcSIrynaimport json
72cddd2bcSIrynafrom collections import defaultdict
82cddd2bcSIryna
92cddd2bcSIrynafrom TestInput import TestInputSingleton
10dd5e1b13SParag Agarwalfrom couchbase_helper.document import View
112cddd2bcSIrynafrom membase.api.rest_client import RestConnection, Bucket
122cddd2bcSIrynafrom membase.helper.cluster_helper import ClusterOperationHelper
132cddd2bcSIrynafrom mc_bin_client import MemcachedClient
142cddd2bcSIrynafrom memcached.helper.data_helper import MemcachedClientHelper
152cddd2bcSIrynafrom basetestcase import BaseTestCase
162cddd2bcSIrynafrom membase.api.exception import QueryViewException
172cddd2bcSIrynafrom membase.helper.rebalance_helper import RebalanceHelper
182cddd2bcSIryna
192cddd2bcSIryna
202cddd2bcSIrynaclass ViewMergingTests(BaseTestCase):
212cddd2bcSIryna
222cddd2bcSIryna    def setUp(self):
232cddd2bcSIryna        try:
242cddd2bcSIryna            if 'first_case' not in TestInputSingleton.input.test_params:
252cddd2bcSIryna                TestInputSingleton.input.test_params['default_bucket'] = False
262cddd2bcSIryna                TestInputSingleton.input.test_params['skip_cleanup'] = True
272cddd2bcSIryna            self.default_bucket_name = 'default'
282cddd2bcSIryna            super(ViewMergingTests, self).setUp()
292cddd2bcSIryna            if 'first_case' in TestInputSingleton.input.test_params:
302cddd2bcSIryna                self.cluster.rebalance(self.servers[:], self.servers[1:], [])
312cddd2bcSIryna            # We use only one bucket in this test suite
322cddd2bcSIryna            self.rest = RestConnection(self.master)
332cddd2bcSIryna            self.bucket = self.rest.get_bucket(Bucket(name=self.default_bucket_name))
342cddd2bcSIryna            # num_docs must be a multiple of the number of vbuckets
352cddd2bcSIryna            self.num_docs = self.input.param("num_docs_per_vbucket", 1) * \
362cddd2bcSIryna                            len(self.bucket.vbuckets)
372bd4873aSVolker Mische            self.is_dev_view = self.input.param("is_dev_view", False)
382cddd2bcSIryna            self.map_view_name = 'mapview1'
392cddd2bcSIryna            self.red_view_name = 'redview1'
400a2eb023SFilipe David Borba Manana            self.red_view_stats_name = 'redview_stats'
412cddd2bcSIryna            self.clients = self.init_clients()
422cddd2bcSIryna            if 'first_case' in TestInputSingleton.input.test_params:
432bd4873aSVolker Mische                 self.create_ddocs(False)
442bd4873aSVolker Mische                 self.create_ddocs(True)
452cddd2bcSIryna        except Exception as ex:
46457deffeSandreibara            self.input.test_params["stop-on-failure"] = True
47457deffeSandreibara            self.log.error("SETUP WAS FAILED. ALL TESTS WILL BE SKIPPED")
482cddd2bcSIryna            self.fail(ex)
492cddd2bcSIryna
502cddd2bcSIryna    def tearDown(self):
512cddd2bcSIryna        # clean up will only performed on the last run
522cddd2bcSIryna        if 'last_case' in TestInputSingleton.input.test_params:
532cddd2bcSIryna            TestInputSingleton.input.test_params['skip_cleanup'] = False
542cddd2bcSIryna            super(ViewMergingTests, self).tearDown()
552cddd2bcSIryna        else:
562cddd2bcSIryna            self.cluster.shutdown(force=True)
572cddd2bcSIryna            self._log_finish(self)
582cddd2bcSIryna
592cddd2bcSIryna    def test_empty_vbuckets(self):
602cddd2bcSIryna        results = self.merged_query(self.map_view_name)
612cddd2bcSIryna        self.assertEquals(results.get(u'total_rows', None), 0)
622cddd2bcSIryna        self.assertEquals(len(results.get(u'rows', None)), 0)
632cddd2bcSIryna
642cddd2bcSIryna    def test_nonexisting_views(self):
65169d3679Sandreibara        view_names = ['mapview2', 'mapview3', 'mapview4', 'mapview5']
662cddd2bcSIryna        for view_name in view_names:
6796f84b15SIrynaMironava            try:
6896f84b15SIrynaMironava                self.merged_query(view_name)
6996f84b15SIrynaMironava            except QueryViewException:
7096f84b15SIrynaMironava                self.log.info("QueryViewException is raised as expected")
7196f84b15SIrynaMironava            except Exception:
7296f84b15SIrynaMironava                self.assertFail("QueryViewException is expected, but not raised")
7396f84b15SIrynaMironava            else:
7496f84b15SIrynaMironava                self.assertFail("QueryViewException is expected, but not raised")
752cddd2bcSIryna
762cddd2bcSIryna    def test_non_empty_view(self):
772cddd2bcSIryna        num_vbuckets = len(self.rest.get_vbuckets(self.bucket))
782cddd2bcSIryna        docs = ViewMergingTests.make_docs(1, self.num_docs + 1)
792cddd2bcSIryna        self.populate_sequenced(num_vbuckets, docs)
802cddd2bcSIryna        results = self.merged_query(self.map_view_name)
812cddd2bcSIryna        self.assertEquals(results.get(u'total_rows', 0), self.num_docs)
822cddd2bcSIryna        self.assertEquals(len(results.get(u'rows', [])), self.num_docs)
832cddd2bcSIryna
842cddd2bcSIryna    def test_queries(self):
852cddd2bcSIryna        all_query_params = ['skip', 'limit', 'startkey', 'endkey', 'startkey_docid',
862cddd2bcSIryna                        'endkey_docid', 'inclusive_end', 'descending', 'key']
872cddd2bcSIryna        current_params = {}
882cddd2bcSIryna        for key in self.input.test_params:
892cddd2bcSIryna            if key in all_query_params:
902cddd2bcSIryna                current_params[key] = str(self.input.test_params[key])
912cddd2bcSIryna        results = self.merged_query(self.map_view_name, current_params)
922cddd2bcSIryna        self.verify_results(results, current_params)
932cddd2bcSIryna
942cddd2bcSIryna    def test_keys(self):
952cddd2bcSIryna        keys = [5, 3, 10, 39, 666666, 21]
962cddd2bcSIryna        results = self.merged_query(self.map_view_name,
972cddd2bcSIryna                                    {"keys": keys})
982cddd2bcSIryna        self.verify_results(results, {"keys": keys})
992cddd2bcSIryna
1002cddd2bcSIryna    def test_include_docs(self):
1012cddd2bcSIryna        results = self.merged_query(self.map_view_name,
1022cddd2bcSIryna                                    {"include_docs": "true"})
1032cddd2bcSIryna        self.verify_results(results, {"include_docs": "true"})
1042cddd2bcSIryna        num = 1
1052cddd2bcSIryna        for row in results.get(u'rows', []):
1062cddd2bcSIryna            self.assertEquals(row['doc']['json']['integer'], num)
1072cddd2bcSIryna            self.assertEquals(row['doc']['json']['string'], str(num))
1082cddd2bcSIryna            self.assertEquals(row['doc']['meta']['id'], str(num))
109169d3679Sandreibara            num += 1
1102cddd2bcSIryna
1112cddd2bcSIryna    def test_queries_reduce(self):
1122cddd2bcSIryna        all_query_params = ['skip', 'limit', 'startkey', 'endkey', 'startkey_docid',
1132cddd2bcSIryna                        'endkey_docid', 'inclusive_end', 'descending', 'reduce',
114169d3679Sandreibara                        'group', 'group_level']
1152cddd2bcSIryna        current_params = {}
1162cddd2bcSIryna        for key in self.input.test_params:
1172cddd2bcSIryna            if key in all_query_params:
1182cddd2bcSIryna                current_params[key] = str(self.input.test_params[key])
1192cddd2bcSIryna        results = self.merged_query(self.red_view_name, current_params)
1202cddd2bcSIryna        self.verify_results_reduce(results, current_params)
1212cddd2bcSIryna
1222cddd2bcSIryna    def test_stale_ok_alternated_docs(self):
1232cddd2bcSIryna        num_vbuckets = len(self.rest.get_vbuckets(self.bucket))
1242cddd2bcSIryna        docs = ViewMergingTests.make_docs(self.num_docs + 1, self.num_docs + 3)
1252cddd2bcSIryna        self.populate_alternated(num_vbuckets, docs)
1262cddd2bcSIryna        results = self.merged_query(self.map_view_name, {'stale': 'ok'})
1272cddd2bcSIryna        self.assertEquals(results.get(u'total_rows', None), self.num_docs)
1282cddd2bcSIryna        self.assertEquals(len(results.get(u'rows', None)), self.num_docs)
1292cddd2bcSIryna        self.verify_keys_are_sorted(results)
1302cddd2bcSIryna
1312cddd2bcSIryna    def test_stale_update_after_alternated_docs(self):
1322cddd2bcSIryna        results = self.merged_query(self.map_view_name, {'stale': 'update_after'})
1332cddd2bcSIryna        self.assertEquals(results.get(u'total_rows', None), self.num_docs)
1342cddd2bcSIryna        self.assertEquals(len(results.get(u'rows', None)), self.num_docs)
1352cddd2bcSIryna        time.sleep(1)
1362cddd2bcSIryna        results = self.merged_query(self.map_view_name, {'stale': 'ok'})
1372cddd2bcSIryna        self.assertEquals(results.get(u'total_rows', None), self.num_docs + 2)
1382cddd2bcSIryna        self.assertEquals(len(results.get(u'rows', None)), self.num_docs + 2)
1392cddd2bcSIryna        self.verify_keys_are_sorted(results)
1402cddd2bcSIryna
1410a2eb023SFilipe David Borba Manana    def test_stats_error(self):
1420a2eb023SFilipe David Borba Manana        nodes = self.input.param("num_nodes", 0)
1430a2eb023SFilipe David Borba Manana        params = {'stale': 'false', 'on_error': 'stop'}
1440a2eb023SFilipe David Borba Manana        if nodes == 1:
1450a2eb023SFilipe David Borba Manana            try:
1460a2eb023SFilipe David Borba Manana                self.merged_query(self.red_view_stats_name, params=params, ddoc='test2')
1470a2eb023SFilipe David Borba Manana                self.assertTrue(False, "Expected exception when querying _stats view")
1480a2eb023SFilipe David Borba Manana            except QueryViewException as ex:
14941737d3fSSarath Lakshman                # Couchbase version < 3.0
15041737d3fSSarath Lakshman                if "Builtin _stats" in ex:
15141737d3fSSarath Lakshman                    expectedStr = 'Error occured querying view ' + self.red_view_stats_name + \
15241737d3fSSarath Lakshman                        ': {"error":"error","reason":"Builtin _stats function requires map' + \
15341737d3fSSarath Lakshman                        ' values to be numbers"}'
15441737d3fSSarath Lakshman                else:
15541737d3fSSarath Lakshman                    expectedStr = 'Error occured querying view ' + self.red_view_stats_name + \
15641737d3fSSarath Lakshman                            ': {"error":"error","reason":"Reducer: Error building index for view `' + \
15741737d3fSSarath Lakshman                            self.red_view_stats_name + '`, reason: Value is not a number (key \\"1\\")"}'
15841737d3fSSarath Lakshman
1590a2eb023SFilipe David Borba Manana                self.assertEquals(str(ex).strip("\n"), expectedStr)
1600a2eb023SFilipe David Borba Manana        else:
1610a2eb023SFilipe David Borba Manana            self.assertTrue(nodes > 1)
1620a2eb023SFilipe David Borba Manana            results = self.merged_query(self.red_view_stats_name, params=params, ddoc='test2')
1630a2eb023SFilipe David Borba Manana            self.assertEquals(len(results.get(u'rows', None)), 0)
1640a2eb023SFilipe David Borba Manana            self.assertEquals(len(results.get(u'errors', None)), 1)
1650a2eb023SFilipe David Borba Manana
1662bd4873aSVolker Mische    def test_dev_view(self):
1672bd4873aSVolker Mische        # A lot of additional documents are needed in order to trigger the
1682bd4873aSVolker Mische        # dev views. If the number is too low, production views will be used
1692bd4873aSVolker Mische        docs = ViewMergingTests.make_docs(0, self.num_docs)
1702bd4873aSVolker Mische        num_vbuckets = len(self.rest.get_vbuckets(self.bucket))
1712bd4873aSVolker Mische        self.populate_alternated(num_vbuckets, docs)
1722bd4873aSVolker Mische        results = self.merged_query(self.map_view_name, {})
1732bd4873aSVolker Mische        self.verify_results_dev(results)
1742bd4873aSVolker Mische
1753d491816SVolker Mische    def test_view_startkey_endkey_validation(self):
1763d491816SVolker Mische        """Regression test for MB-6591
1773d491816SVolker Mische
1783d491816SVolker Mische        This tests makes sure that the validation of startkey/endkey works
1793d491816SVolker Mische        with view, which uses Unicode collation. Return results
1803d491816SVolker Mische        when startkey is smaller than endkey. When endkey is smaller than
1813d491816SVolker Mische        the startkey and exception should be raised. With Unicode collation
1823d491816SVolker Mische        "foo" < "Foo", with raw collation "Foo" < "foo".
1833d491816SVolker Mische        """
1843d491816SVolker Mische        startkey = '"foo"'
1853d491816SVolker Mische        endkey = '"Foo"'
1863d491816SVolker Mische        params = {"startkey": startkey, "endkey": endkey}
1873d491816SVolker Mische        results = self.merged_query(self.map_view_name, params)
1883d491816SVolker Mische        self.assertTrue('rows' in results, "Results were returned")
1893d491816SVolker Mische
1903d491816SVolker Mische        # Flip startkey and endkey
1913d491816SVolker Mische        params = {"startkey": endkey, "endkey": startkey}
1923d491816SVolker Mische        self.assertRaises(Exception, self.merged_query, self.map_view_name,
1933d491816SVolker Mische                          params)
1943d491816SVolker Mische
1952cddd2bcSIryna    def calculate_matching_keys(self, params):
1962cddd2bcSIryna        keys = range(1, self.num_docs + 1)
1972cddd2bcSIryna        if 'descending' in params:
1982cddd2bcSIryna            keys.reverse()
1992cddd2bcSIryna        if 'startkey' in params:
2002cddd2bcSIryna            if params['startkey'].find('[') > -1:
2012cddd2bcSIryna                key = eval(params['startkey'])[0]
2022cddd2bcSIryna            else:
2032cddd2bcSIryna                key = int(params['startkey'])
2042cddd2bcSIryna            keys = keys[keys.index(key):]
2052cddd2bcSIryna            if 'startkey_docid' in params:
2062cddd2bcSIryna                if params['startkey_docid'] > params['startkey']:
2072cddd2bcSIryna                    keys = keys[1:]
2082cddd2bcSIryna        if 'endkey' in params:
2092cddd2bcSIryna            if params['endkey'].find('[') > -1:
2102cddd2bcSIryna                key = eval(params['endkey'])[0]
2112cddd2bcSIryna            else:
2122cddd2bcSIryna                key = int(params['endkey'])
2132cddd2bcSIryna            keys = keys[:keys.index(key) + 1]
2142cddd2bcSIryna            if 'endkey_docid' in params:
2152cddd2bcSIryna                if params['endkey_docid'] < params['endkey']:
2162cddd2bcSIryna                    keys = keys[:-1]
2172cddd2bcSIryna        if 'inclusive_end' in params and params['inclusive_end'] == 'false':
2182cddd2bcSIryna            keys = keys[:-1]
2192cddd2bcSIryna        if 'skip' in params:
2202cddd2bcSIryna            keys = keys[(int(params['skip'])):]
2212cddd2bcSIryna        if 'limit' in params:
2222cddd2bcSIryna            keys = keys[:(int(params['limit']))]
2232cddd2bcSIryna        if 'key' in params:
2242cddd2bcSIryna            if int(params['key']) <= self.num_docs:
225169d3679Sandreibara                keys = [int(params['key']), ]
2262cddd2bcSIryna            else:
2272cddd2bcSIryna                keys = []
2282cddd2bcSIryna        if 'keys' in params:
2292cddd2bcSIryna            keys = []
2302cddd2bcSIryna            for k in params['keys']:
2312cddd2bcSIryna                if int(k) <= self.num_docs:
2322cddd2bcSIryna                    keys.append(int(k))
2332cddd2bcSIryna            # When ?keys=[...] parameter is sent, rows are not guaranteed to come
2342cddd2bcSIryna            # sorted by key.
2352cddd2bcSIryna            keys.sort()
2362cddd2bcSIryna        return keys
2372cddd2bcSIryna
2382cddd2bcSIryna    def verify_results(self, results, params):
2392cddd2bcSIryna        expected = self.calculate_matching_keys(params)
2402cddd2bcSIryna        self.assertEquals(results.get(u'total_rows', 0), self.num_docs,
2412cddd2bcSIryna                          "total_rows parameter is wrong, expected %d, actual %d"
2422cddd2bcSIryna                          % (self.num_docs, results.get(u'total_rows', 0)))
2432cddd2bcSIryna        self.assertEquals(len(results.get(u'rows', [])), len(expected),
2442cddd2bcSIryna                          "Rows number is wrong, expected %d, actual %d"
2452cddd2bcSIryna                          % (len(expected), len(results.get(u'rows', []))))
2462cddd2bcSIryna        if expected:
2472cddd2bcSIryna            if 'keys' not in params:
2482cddd2bcSIryna                # first
2492cddd2bcSIryna                self.assertEquals(results.get(u'rows', [])[0]['key'], expected[0],
2502cddd2bcSIryna                                  "First row key is wrong, expected %d, actual %d"
2512cddd2bcSIryna                                  % (expected[0], results.get(u'rows', [])[0]['key']))
2522cddd2bcSIryna                # last
2532cddd2bcSIryna                self.assertEquals(results.get(u'rows', [])[-1]['key'], expected[-1],
2542cddd2bcSIryna                                  "Last row key is wrong, expected %d, actual %d"
2552cddd2bcSIryna                                  % (expected[-1], results.get(u'rows', [])[-1]['key']))
2562cddd2bcSIryna                desc = 'descending' in params and params['descending'] == 'true'
2572cddd2bcSIryna                self.verify_keys_are_sorted(results, desc=desc)
2582cddd2bcSIryna            else:
2592cddd2bcSIryna                actual = sorted([row[u'key'] for row in results.get(u'rows', [])])
2602cddd2bcSIryna                self.assertEquals(actual, expected,
2612cddd2bcSIryna                                  "Results are wrong, expected %s, actual %s"
2622cddd2bcSIryna                                  % (expected, results.get(u'rows', [])))
2632cddd2bcSIryna
2642bd4873aSVolker Mische    def verify_results_dev(self, results):
2652bd4873aSVolker Mische        # A development view is always a subset of the production view,
2662bd4873aSVolker Mische        # hence only check for that (and not the exact items)
2672bd4873aSVolker Mische        expected = self.calculate_matching_keys({})
2682bd4873aSVolker Mische        self.assertTrue(len(results.get(u'rows', [])) < len(expected) and
2692bd4873aSVolker Mische                        len(results.get(u'rows', [])) > 0,
2702bd4873aSVolker Mische                          ("Rows number is wrong, expected to be lower than "
2712bd4873aSVolker Mische                           "%d and >0, but it was %d"
2722bd4873aSVolker Mische                          % (len(expected), len(results.get(u'rows', [])))))
2732bd4873aSVolker Mische        self.assertTrue(
2742bd4873aSVolker Mische            results.get(u'rows', [])[0]['key'] != expected[0] or
2752bd4873aSVolker Mische            results.get(u'rows', [])[-1]['key'] != expected[-1],
2762bd4873aSVolker Mische            "Dev view should be a subset, but returned the same as "
2772bd4873aSVolker Mische            "the production view")
2782bd4873aSVolker Mische        self.verify_keys_are_sorted(results)
2792bd4873aSVolker Mische
2802cddd2bcSIryna    def verify_results_reduce(self, results, params):
2812cddd2bcSIryna        if 'reduce' in params and params['reduce'] == 'false' or \
2822cddd2bcSIryna          'group_level' in params or 'group' in params and params['group'] == 'true':
2832cddd2bcSIryna            expected = self.calculate_matching_keys(params)
2842cddd2bcSIryna            self.assertEquals(len(results.get(u'rows', [])), len(expected),
2852cddd2bcSIryna                          "Rows number is wrong, expected %d, actual %d"
2862cddd2bcSIryna                          % (len(expected), len(results.get(u'rows', []))))
2872cddd2bcSIryna            if expected:
2882cddd2bcSIryna                #first
2892cddd2bcSIryna                self.assertEquals(results.get(u'rows', [])[0]['key'][0], expected[0],
2902cddd2bcSIryna                                  "First element is wrong, expected %d, actual %d"
2912cddd2bcSIryna                                  % (expected[0], results.get(u'rows', [])[0]['key'][0]))
2922cddd2bcSIryna                #last
2932cddd2bcSIryna                self.assertEquals(results.get(u'rows', [])[-1]['key'][0], expected[-1],
2942cddd2bcSIryna                                  "Last element is wrong, expected %d, actual %d"
2952cddd2bcSIryna                                  % (expected[-1], results.get(u'rows', [])[-1]['key'][0]))
2962cddd2bcSIryna        else:
2972cddd2bcSIryna            expected = self.calculate_matching_keys(params)
2982cddd2bcSIryna            self.assertEquals(results.get(u'rows', [])[0][u'value'], len(expected),
2992cddd2bcSIryna                              "Value for reduce is incorrect. Expected %s, actual %s"
300169d3679Sandreibara                              % (len(expected), results.get(u'rows', [])[0][u'value']))
3012cddd2bcSIryna
3022bd4873aSVolker Mische    def create_ddocs(self, is_dev_view):
3032cddd2bcSIryna        mapview = View(self.map_view_name, '''function(doc) {
3042cddd2bcSIryna             emit(doc.integer, doc.string);
3052bd4873aSVolker Mische          }''', dev_view=is_dev_view)
3062cddd2bcSIryna        self.cluster.create_view(self.master, 'test', mapview)
3072cddd2bcSIryna        redview = View(self.red_view_name, '''function(doc) {
3082cddd2bcSIryna             emit([doc.integer, doc.string], doc.integer);
3092bd4873aSVolker Mische          }''', '''_count''', dev_view=is_dev_view)
3102cddd2bcSIryna        self.cluster.create_view(self.master, 'test', redview)
3110a2eb023SFilipe David Borba Manana        redview_stats = View(self.red_view_stats_name, '''function(doc) {
3120a2eb023SFilipe David Borba Manana             emit(doc.string, doc.string);
3132bd4873aSVolker Mische          }''', '''_stats''', dev_view=is_dev_view)
3140a2eb023SFilipe David Borba Manana        self.cluster.create_view(self.master, 'test2', redview_stats)
3152cddd2bcSIryna        RebalanceHelper.wait_for_persistence(self.master, self.bucket, 0)
3162cddd2bcSIryna
3172cddd2bcSIryna    def init_clients(self):
3182cddd2bcSIryna        """Initialise clients for all servers there are vBuckets on
3192cddd2bcSIryna
3202cddd2bcSIryna        It returns a dict with 'ip:port' as key (this information is also
3212cddd2bcSIryna        stored this way in every vBucket in the `master` property) and
3222cddd2bcSIryna        the MemcachedClient as the value
3232cddd2bcSIryna        """
3242cddd2bcSIryna        clients = {}
3252cddd2bcSIryna
3262cddd2bcSIryna        for vbucket in self.bucket.vbuckets:
3272cddd2bcSIryna            if vbucket.master not in clients:
3282cddd2bcSIryna                ip, port = vbucket.master.split(':')
329169d3679Sandreibara                clients[vbucket.master] = MemcachedClient(ip, int(port))
3302cddd2bcSIryna        return clients
3312cddd2bcSIryna
3322cddd2bcSIryna    def populate_alternated(self, num_vbuckets, docs):
3332cddd2bcSIryna        """Every vBucket gets a doc first
3342cddd2bcSIryna
3352cddd2bcSIryna        Populating the vBuckets alternated means that every vBucket gets
3362cddd2bcSIryna        a document first, before it receives the second one and so on.
3372cddd2bcSIryna
3382cddd2bcSIryna        For example if we have 6 documents named doc-1 ... doc-6 and 3
3392cddd2bcSIryna        vBuckets the result will be:
3402cddd2bcSIryna
3412cddd2bcSIryna            vbucket-1: doc-1, doc-4
3422cddd2bcSIryna            vbucket-2: doc-2, doc-5
3432cddd2bcSIryna            vbucket-3: doc-3, doc-6
3442cddd2bcSIryna        """
3452cddd2bcSIryna        for i, doc in enumerate(docs):
3462cddd2bcSIryna            self.insert_into_vbucket(i % num_vbuckets, doc)
3472cddd2bcSIryna        RebalanceHelper.wait_for_persistence(self.master, self.bucket, 0)
3482cddd2bcSIryna
3492cddd2bcSIryna    def populate_sequenced(self, num_vbuckets, docs):
3502cddd2bcSIryna        """vBuckets get filled up one by one
3512cddd2bcSIryna
3522cddd2bcSIryna        Populating the vBuckets sequenced means that the vBucket gets
3532cddd2bcSIryna        a certain number of documents, before the next one gets some.
3542cddd2bcSIryna
3552cddd2bcSIryna        For example if we have 6 documents named doc-1 ... doc-6 and 3
3562cddd2bcSIryna        vBuckets the result will be:
3572cddd2bcSIryna
3582cddd2bcSIryna            vbucket-1: doc-1, doc-2
3592cddd2bcSIryna            vbucket-2: doc-3, doc-4
3602cddd2bcSIryna            vbucket-3: doc-5, doc-5
3612cddd2bcSIryna        """
3622cddd2bcSIryna        docs_per_vbucket = len(docs) / num_vbuckets
3632cddd2bcSIryna
3642cddd2bcSIryna        for vbucket in range(num_vbuckets):
3652cddd2bcSIryna            start = vbucket * docs_per_vbucket
3662cddd2bcSIryna            end = start + docs_per_vbucket
3672cddd2bcSIryna            for doc in docs[start:end]:
3682cddd2bcSIryna                    self.insert_into_vbucket(vbucket, doc)
3692cddd2bcSIryna        RebalanceHelper.wait_for_persistence(self.master, self.bucket, 0)
3702cddd2bcSIryna
3712cddd2bcSIryna    def insert_into_vbucket(self, vbucket_id, doc):
3722cddd2bcSIryna        """Insert a document into a certain vBucket
3732cddd2bcSIryna
3742cddd2bcSIryna        The memcached clients must already been initialised in the
3752cddd2bcSIryna        self.clients property.
3762cddd2bcSIryna        """
3772cddd2bcSIryna        vbucket = self.bucket.vbuckets[vbucket_id]
3782cddd2bcSIryna        client = self.clients[vbucket.master]
3792cddd2bcSIryna
3802cddd2bcSIryna        client.set(doc['json']['key'], 0, 0, json.dumps(doc['json']['body']).encode("ascii", "ignore"), vbucket_id)
3812cddd2bcSIryna
3822cddd2bcSIryna    @staticmethod
3832cddd2bcSIryna    def make_docs(start, end):
3842cddd2bcSIryna        """Create documents
3852cddd2bcSIryna
3862cddd2bcSIryna        `key` will be used as a key and won't end up in the final
3872cddd2bcSIryna        document body.
3882cddd2bcSIryna        `body` will be used as the document body
3892cddd2bcSIryna        """
3902cddd2bcSIryna        docs = []
3912cddd2bcSIryna        for i in range(start, end):
3922cddd2bcSIryna            doc = {
3932cddd2bcSIryna                'key': str(i),
3942cddd2bcSIryna                'body': { 'integer': i, 'string': str(i)}}
3952cddd2bcSIryna
3962cddd2bcSIryna            docs.append({"meta":{"id": str(i)}, "json": doc })
3972cddd2bcSIryna        return docs
3982cddd2bcSIryna
3992cddd2bcSIryna    def merged_query(self, view_name, params={}, ddoc='test'):
4002cddd2bcSIryna       bucket = self.default_bucket_name
4012cddd2bcSIryna       if not 'stale' in params:
4022cddd2bcSIryna           params['stale'] = 'false'
403169d3679Sandreibara       ddoc = ("", "dev_")[self.is_dev_view] + ddoc
4042cddd2bcSIryna       return self.rest.query_view(ddoc, view_name, bucket, params)
4052cddd2bcSIryna
4062cddd2bcSIryna    def verify_keys_are_sorted(self, results, desc=False):
4072cddd2bcSIryna        current_keys = [row['key'] for row in results['rows']]
4082cddd2bcSIryna        self.assertTrue(ViewMergingTests._verify_list_is_sorted(current_keys, desc=desc), 'keys are not sorted')
4092cddd2bcSIryna        self.log.info('rows are sorted by key')
4102cddd2bcSIryna
4112cddd2bcSIryna    @staticmethod
412169d3679Sandreibara    def _verify_list_is_sorted(keys, key=lambda x: x, desc=False):
4132cddd2bcSIryna        if desc:
4142cddd2bcSIryna            return all([key(keys[i]) >= key(keys[i + 1]) for i in xrange(len(keys) - 1)])
4152cddd2bcSIryna        else:
4162cddd2bcSIryna            return all([key(keys[i]) <= key(keys[i + 1]) for i in xrange(len(keys) - 1)])
417