xref: /5.5.2/testrunner/lib/builds/build_query.py (revision 05d771fc)
1#this class will contain methods which we
2#use later to
3# map a version # -> rpm url
4from datetime import datetime
5import time
6import urllib2
7import re
8import socket
9import BeautifulSoup
10import testconstants
11import logger
12import traceback
13import sys
14from testconstants import WIN_CB_VERSION_3
15from testconstants import SHERLOCK_VERSION
16from testconstants import COUCHBASE_VERSION_2
17from testconstants import COUCHBASE_VERSION_3
18from testconstants import COUCHBASE_VERSION_2_WITH_REL
19from testconstants import COUCHBASE_RELEASE_FROM_VERSION_3
20from testconstants import COUCHBASE_FROM_VERSION_3, COUCHBASE_FROM_SPOCK
21from testconstants import CB_RELEASE_REPO
22from testconstants import CB_LATESTBUILDS_REPO
23from testconstants import CE_EE_ON_SAME_FOLDER
24
25
26class MembaseBuild(object):
27    def __init__(self):
28        self.url = ''
29        self.name = ''
30        self.time = ''
31        self.size = ''
32        self.product = ''
33        self.product_version = ''
34        self.build_number = 0
35        self.os = ''
36        self.deliverable_type = ''
37        self.architecture_type = ''
38        self.toy = ''
39        self.change = None  # a MembaseChange
40        self.url_latest_build = ''
41
42    def __repr__(self):
43        return self.__str__()
44
45    #let's also have a json object for all these classes
46    def __str__(self):
47        url = 'url : {0}'.format(self.url)
48        name = 'name : {0}'.format(self.name)
49        product = 'product : {0}'.format(self.product)
50        product_version = 'product_version : {0}'.format(self.product_version)
51        os = 'os : {0}'.format(self.os)
52        deliverable_type = 'deliverable_type : {0}'.format(self.deliverable_type)
53        architecture_type = 'architecture_type : {0}'.format(self.architecture_type)
54        if self.toy:
55            toy = 'toy : {0}'.format(self.toy)
56        else:
57            toy = ''
58        return '{0} {1} {2} {3} {4} {5} {6} {7}'.format(url, name, product, product_version, os, deliverable_type,
59                                                    architecture_type, toy)
60
61
62class MembaseChange(object):
63    def __init__(self):
64        self.url = ''
65        self.name = ''
66        self.time = ''
67        self.build_number = ''
68
69
70class BuildQuery(object):
71    def __init__(self):
72        self.log = logger.Logger.get_logger()
73        pass
74
75    # let's look at buildlatets or latest/sustaining or any other
76    # location
77    def parse_builds(self):
78        #parse build page and create build object
79        pass
80
81    def find_build(self, builds, product, type, arch, version, toy='', openssl='', \
82                   direct_build_url=None, distribution_version=None, \
83                   distribution_type=""):
84        if direct_build_url is None:
85            if not isinstance(builds, list) and builds.url is not None:
86                return builds
87            else:
88                for build in builds:
89                    if build.product_version.find(version) != -1 and product == build.product\
90                       and build.architecture_type == arch and type == build.deliverable_type\
91                       and build.toy == toy:
92                        return build
93        elif direct_build_url != "":
94            if "exe" in builds.deliverable_type:
95                if "rel" in version and version[:5] in WIN_CB_VERSION_3:
96                    version = version.replace("-rel", "")
97            """ direct url only need one build """
98
99            """ if the job trigger with url, no need to check version.
100                remove builds.product_version.find(version) != -1 """
101            if product == builds.product and builds.architecture_type == arch:
102                return builds
103            else:
104                self.log.info("if build not found, url link may not match...")
105        return None
106
107    def find_membase_build(self, builds, product, deliverable_type, os_architecture, build_version, is_amazon=False):
108        if is_amazon:
109            build = BuildQuery().find_build(builds, product, deliverable_type,
110                                            os_architecture, build_version)
111            if build:
112                build.url = build.url.replace("http://builds.hq.northscale.net", \
113                                                  "http://packages.northscale.com.s3.amazonaws.com")
114                build.url = build.url.replace("enterprise", "community")
115                build.name = build.name.replace("enterprise", "community")
116                return build
117
118        for build in builds:
119            if build.product_version.find(build_version) != -1 and product == build.product\
120               and build.architecture_type == os_architecture and deliverable_type == build.deliverable_type:
121                return build
122
123        return None
124
125    def find_membase_build_with_version(self, builds, build_version):
126        for build in builds:
127            if build.product_version == build_version or build.product_version.find(build_version) != -1:
128                #or if it starts with that version ?
129                return build
130        return None
131
132    def find_membase_release_build(self, product, deliverable_type, os_architecture,
133                                    build_version, is_amazon=False, os_version=""):
134        build_details = build_version
135        if build_version[:5] in COUCHBASE_VERSION_2_WITH_REL:
136            if build_version[-4:] != "-rel":
137                build_details = build_details + "-rel"
138        elif build_version.startswith("1.8.0"):
139            build_details = "1.8.0r-55-g80f24f2"
140            product = "couchbase-server-enterprise"
141        build = MembaseBuild()
142        build.deliverable_type = deliverable_type
143        build.time = '0'
144        build.size = '0'
145        build.product_version = build_version
146        build.architecture_type = os_architecture
147        build.product = product
148        os_name = ""
149        build.name = '{1}_{2}_{0}.{3}'.format(build_version, product,
150                                               os_architecture, deliverable_type)
151        build.build_number = 0
152        if deliverable_type == "exe":
153            """ /3.0.1/couchbase-server-enterprise_3.0.1-windows_amd64.exe """
154            if not re.match(r'[1-9].[0-9].[0-9]$', build_version):
155                if build_version[:5] in COUCHBASE_RELEASE_FROM_VERSION_3:
156                    arch_type = "amd64"
157                    if "x86_64" not in os_architecture:
158                        arch_type = "x86"
159                    build.url = "{5}{0}/{1}_{4}-windows_{2}.{3}"\
160                        .format(build_version[:build_version.find('-')],
161                        product, arch_type, deliverable_type, build_details[:5],
162                        CB_RELEASE_REPO)
163                else:
164                    if "2.5.2" in build_details[:5]:
165                        build.url = "{5}{0}/{1}_{4}_{2}.setup.{3}"\
166                            .format(build_version[:build_version.find('-')],
167                            product, os_architecture, deliverable_type,
168                            build_details[:5], CB_RELEASE_REPO)
169                    else:
170                        build.url = "{5}{0}/{1}_{2}_{4}.setup.{3}"\
171                            .format(build_version[:build_version.find('-')],
172                            product, os_architecture, deliverable_type,
173                            build_details, CB_RELEASE_REPO)
174            else:
175                if build_version[:5] in COUCHBASE_RELEASE_FROM_VERSION_3:
176                    arch_type = "amd64"
177                    if "x86_64" not in os_architecture:
178                        arch_type = "x86"
179                    build.url = "{5}{0}/{1}_{4}-windows_{2}.{3}"\
180                        .format(build_version, product, arch_type,
181                        deliverable_type, build_details[:5], CB_RELEASE_REPO)
182                else:
183                    build.url = "{5}{0}/{1}_{2}_{4}.setup.{3}"\
184                        .format(build_version, product, os_architecture,
185                        deliverable_type, build_details, CB_RELEASE_REPO)
186            build.url_latest_build = "{4}{0}_{1}_{3}.setup.{2}"\
187                             .format(product, os_architecture, deliverable_type,
188                                            build_details, CB_LATESTBUILDS_REPO)
189        else:
190            """ check match full version x.x.x-xxxx """
191            if not re.match(r'[1-9].[0-9].[0-9]$', build_version):
192                """  in release folder
193                        /3.0.1/couchbase-server-enterprise-3.0.1-centos6.x86_64.rpm
194                        /3.0.1/couchbase-server-enterprise_3.0.1-ubuntu12.04_amd64.deb
195                        /3.0.2/couchbase-server-enterprise-3.0.2-centos6.x86_64.rpm
196                      build release url:
197                               http://builds.hq.northscale.net/releases/3.0.1/
198                      build latestbuilds url:
199                               http://builds.hq.northscale.net/latestbuilds/
200                                  couchbase-server-enterprise_x86_64_3.0.1-1444.rpm
201                """
202                if build_version[:5] in COUCHBASE_RELEASE_FROM_VERSION_3:
203                    if "rpm" in deliverable_type:
204                        build.url = "{5}{0}/{1}-{4}-centos6.{2}.{3}"\
205                                .format(build_version[:build_version.find('-')],
206                                product, os_architecture, deliverable_type,
207                                        build_details[:5], CB_RELEASE_REPO)
208                    elif "deb" in deliverable_type:
209                        os_architecture = "amd64"
210                        os_name = "ubuntu12.04"
211                        if  "ubuntu 14.04" in os_version:
212                            os_name = "ubuntu14.04"
213                        elif "ubuntu 16.04" in os_version:
214                            os_name = "ubuntu16.04"
215                        build.url = "{6}{0}/{1}_{4}-{5}_{2}.{3}"\
216                                .format(build_version[:build_version.find('-')],
217                                 product, os_architecture, deliverable_type,
218                                 build_details[:5], os_name, CB_RELEASE_REPO)
219                else:
220                    if "2.5.2" in build_details[:5]:
221                        build.url = "{5}{0}/{1}_{4}_{2}.{3}"\
222                            .format(build_version[:build_version.find('-')],
223                            product, os_architecture, deliverable_type,
224                                    build_details[:5], CB_RELEASE_REPO)
225                    else:
226                        build.url = "{5}{0}/{1}_{2}_{4}.{3}"\
227                            .format(build_version[:build_version.find('-')],
228                            product, os_architecture, deliverable_type,
229                                        build_details, CB_RELEASE_REPO)
230            else:
231                if build_version[:5] in COUCHBASE_RELEASE_FROM_VERSION_3:
232                    if "rpm" in deliverable_type:
233                        build.url = "{5}{0}/{1}-{4}-centos6.{2}.{3}"\
234                            .format(build_version, product, os_architecture,
235                            deliverable_type, build_details[:5], CB_RELEASE_REPO)
236                    elif "deb" in deliverable_type:
237                        os_architecture = "amd64"
238                        os_name = "ubuntu12.04"
239                        if  "ubuntu 14.04" in os_version:
240                            os_name = "ubuntu14.04"
241                        elif "ubuntu 16.04" in os_version:
242                            os_name = "ubuntu16.04"
243                        build.url = "{6}{0}/{1}_{4}-{5}_{2}.{3}"\
244                            .format(build_version, product, os_architecture,
245                            deliverable_type, build_details[:5], os_name,
246                                                         CB_RELEASE_REPO)
247                        """ http://builds.hq.northscale.net/releases/3.0.1/
248                        couchbase-server-enterprise_3.0.1-ubuntu12.04_amd64.deb """
249                else:
250                    build.url = "{5}{0}/{1}_{2}_{4}.{3}"\
251                        .format(build_version, product, os_architecture,
252                        deliverable_type, build_details, CB_RELEASE_REPO)
253            build.url_latest_build = "{4}{0}_{1}_{3}.{2}"\
254                      .format(product, os_architecture, deliverable_type,
255                               build_details, CB_LATESTBUILDS_REPO)
256        # This points to the Internal s3 account to look for release builds
257        if is_amazon:
258            build.url = 'https://s3.amazonaws.com/packages.couchbase/releases/{0}/{1}_{2}_{0}.{3}'\
259                .format(build_version, product, os_architecture, deliverable_type)
260            build.url = build.url.replace("enterprise", "community")
261            build.name = build.name.replace("enterprise", "community")
262        return build
263
264    def find_couchbase_release_build(self, product, deliverable_type, os_architecture,
265                                    build_version, is_amazon=False, os_version="",
266                                    direct_build_url=None):
267        build_details = build_version
268        if build_version[:5] in COUCHBASE_VERSION_2_WITH_REL:
269            if build_version[-4:] != "-rel":
270                build_details = build_details + "-rel"
271        build = MembaseBuild()
272        build.deliverable_type = deliverable_type
273        build.time = '0'
274        build.size = '0'
275        build.product_version = build_version
276        build.architecture_type = os_architecture
277        build.product = product
278        os_name = ""
279        build.name = '{1}_{2}_{0}.{3}'.format(build_version, product,
280                                               os_architecture, deliverable_type)
281        build.build_number = 0
282        if deliverable_type == "exe":
283            """ /3.0.1/couchbase-server-enterprise_3.0.1-windows_amd64.exe """
284            if not re.match(r'[1-9].[0-9].[0-9]$', build_version):
285                if build_version[:5] in COUCHBASE_RELEASE_FROM_VERSION_3:
286                    arch_type = "amd64"
287                    if "x86_64" not in os_architecture:
288                        arch_type = "x86"
289                    build.url = "{5}{0}/{1}_{4}-windows_{2}.{3}"\
290                            .format(build_version[:build_version.find('-')],
291                            product, arch_type, deliverable_type, build_details[:5],
292                            CB_RELEASE_REPO)
293                else:
294                    if "2.5.2" in build_details[:5]:
295                        build.url = "{5}/{0}/{1}_{4}_{2}.setup.{3}"\
296                            .format(build_version[:build_version.find('-')],
297                            product, os_architecture, deliverable_type,
298                            build_details[:5], CB_RELEASE_REPO)
299                    else:
300                        build.url = "{5}{0}/{1}_{2}_{4}.setup.{3}"\
301                            .format(build_version[:build_version.find('-')],
302                            product, os_architecture, deliverable_type,
303                            build_details, CB_RELEASE_REPO)
304            else:
305                if build_version[:5] in COUCHBASE_RELEASE_FROM_VERSION_3:
306                    arch_type = "amd64"
307                    if "x86_64" not in os_architecture:
308                        arch_type = "x86"
309                    build.url = "{5}{0}/{1}_{4}-windows_{2}.{3}"\
310                        .format(build_version, product, arch_type,
311                         deliverable_type, build_details[:5], CB_RELEASE_REPO)
312                else:
313                    build.url = "{5}{0}/{1}_{2}_{4}.setup.{3}"\
314                        .format(build_version, product, os_architecture,
315                        deliverable_type, build_details, CB_RELEASE_REPO)
316            build.url_latest_build = "{4}{0}_{1}_{3}.setup.{2}"\
317                             .format(product, os_architecture,
318                             deliverable_type, build_details, CB_LATESTBUILDS_REPO)
319        else:
320            """ check match full version x.x.x-xxxx """
321            if not re.match(r'[1-9].[0-9].[0-9]$', build_version):
322                """  in release folder
323                        /3.0.1/couchbase-server-enterprise-3.0.1-centos6.x86_64.rpm
324                        /3.0.1/couchbase-server-enterprise_3.0.1-ubuntu12.04_amd64.deb
325                        /3.0.2/couchbase-server-enterprise-3.0.2-centos6.x86_64.rpm
326                      build release url:
327                               http://builds.hq.northscale.net/releases/3.0.1/
328                      build latestbuilds url:
329                               http://builds.hq.northscale.net/latestbuilds/
330                                  couchbase-server-enterprise_x86_64_3.0.1-1444.rpm
331                """
332                if build_version[:5] in COUCHBASE_RELEASE_FROM_VERSION_3:
333                    if "rpm" in deliverable_type:
334                        if "centos" in os_version.lower():
335                            if "centos 7" in os_version.lower():
336                                os_name = "centos7"
337                            else:
338                                os_name = "centos6"
339                        elif "suse" in os_version.lower():
340                            if "11" in os_version.lower():
341                                os_name = "suse11"
342                            elif "12" in os_version.lower():
343                                os_name = "suse12"
344                        elif "oracle linux" in os_version.lower():
345                            os_name = "oel6"
346                        else:
347                            os_name = "centos6"
348                        build.url = "{6}{0}/{1}-{4}-{5}.{2}.{3}"\
349                                .format(build_version[:build_version.find('-')],
350                                product, os_architecture, deliverable_type,
351                                build_details[:5],os_name, CB_RELEASE_REPO)
352                    elif "deb" in deliverable_type:
353                        os_architecture = "amd64"
354                        os_name = "ubuntu12.04"
355                        if  "ubuntu 14.04" in os_version.lower():
356                            os_name = "ubuntu14.04"
357                        elif "ubuntu 16.04" in os_version.lower():
358                            os_name = "ubuntu16.04"
359                        build.url = "{6}{0}/{1}_{4}-{5}_{2}.{3}"\
360                                .format(build_version[:build_version.find('-')],
361                                 product, os_architecture, deliverable_type,
362                                 build_details[:5], os_name, CB_RELEASE_REPO)
363                else:
364                    if "2.5.2" in build_details[:5]:
365                        if product == "moxi-server" and deliverable_type == "deb":
366                            build.url = "{5}{0}/{1}_{4}_{2}_openssl098.{3}".format(
367                                            build_version[:build_version.find('-')],
368                                            product,
369                                            os_architecture,
370                                            deliverable_type,
371                                            build_details[:5],
372                                            CB_RELEASE_REPO)
373                        else:
374                            build.url = "{5}{0}/{1}_{4}_{2}.{3}".format(
375                                            build_version[:build_version.find('-')],
376                                            product,
377                                            os_architecture,
378                                            deliverable_type,
379                                            build_details[:5],
380                                            CB_RELEASE_REPO)
381                    else:
382                        build.url = "{5}{0}/{1}_{2}_{4}.{3}"\
383                            .format(build_version[:build_version.find('-')],
384                            product, os_architecture, deliverable_type,
385                            build_details, CB_RELEASE_REPO)
386            else:
387                if build_version[:5] in COUCHBASE_RELEASE_FROM_VERSION_3:
388                    if "rpm" in deliverable_type:
389                        if "centos" in os_version.lower():
390                            if "centos 7" in os_version.lower():
391                                os_name = "centos7"
392                            else:
393                                os_name = "centos6"
394                        elif "suse" in os_version.lower():
395                            if "11" in os_version.lower():
396                                os_name = "suse11"
397                            elif "12" in os_version.lower():
398                                os_name = "suse12"
399                        elif "oracle linux" in os_version.lower():
400                            os_name = "oel6"
401                        else:
402                            os_name = "centos6"
403                        build.url = "{6}{0}/{1}-{4}-{5}.{2}.{3}"\
404                                .format(build_version[:build_version.find('-')],
405                                product, os_architecture, deliverable_type,
406                                build_details[:5],os_name, CB_RELEASE_REPO)
407                    elif "deb" in deliverable_type:
408                        os_architecture = "amd64"
409                        os_name = "ubuntu12.04"
410                        if  "ubuntu 14.04" in os_version.lower():
411                            os_name = "ubuntu14.04"
412                        elif "ubuntu 16.04" in os_version.lower():
413                            os_name = "ubuntu16.04"
414                        build.url = "{6}{0}/{1}_{4}-{5}_{2}.{3}"\
415                            .format(build_version, product, os_architecture,
416                            deliverable_type, build_details[:5], os_name,
417                            CB_RELEASE_REPO)
418                else:
419                    build.url = "{5}{0}/{1}_{2}_{4}.{3}".format(build_version,
420                                product, os_architecture, deliverable_type,
421                                build_details, CB_RELEASE_REPO)
422            build.url_latest_build = "{4}{0}_{1}_{3}.{2}".format(product,
423                        os_architecture, deliverable_type, build_details,
424                        CB_LATESTBUILDS_REPO)
425        # This points to the Internal s3 account to look for release builds
426        """ add ce folder in community version from 3.0.2 release """
427        if "community" in product and build_version[:5] not in CE_EE_ON_SAME_FOLDER:
428            build.url = build.url.replace("couchbase-server-community", \
429                                          "ce/couchbase-server-community")
430        if is_amazon:
431            """
432                for centos only
433                         https://s3.amazonaws.com/packages.couchbase.com/releases/
434                         4.0.0/couchbase-server-enterprise-4.0.0-centos6.x86_64.rpm """
435            build.url = "https://s3.amazonaws.com/packages.couchbase.com/releases/"\
436                        "{0}/{1}-{0}-centos6.{2}.{3}" \
437                        .format(build_version, product, os_architecture,
438                                                       deliverable_type)
439            build.url = build.url.replace("enterprise", "community")
440            build.name = build.name.replace("enterprise", "community")
441        if direct_build_url is not None and deliverable_type != "exe":
442            build.url = direct_build_url
443        return build
444
445    def sort_builds_by_version(self, builds):
446        membase_builds = list()
447        for build in builds:
448            if build.product == 'membase-server-enterprise':
449                membase_builds.append(build)
450
451        return sorted(membase_builds,
452                      key=lambda membase_build: membase_build.build_number, reverse=True)
453
454    def sort_builds_by_time(self, builds):
455        membase_builds = list()
456        for build in builds:
457            if build.product == 'membase-server-enterprise':
458                membase_builds.append(build)
459
460        return sorted(membase_builds,
461                      key=lambda membase_build: membase_build.time, reverse=True)
462
463
464    def get_latest_builds(self):
465        return self._get_and_parse_builds('http://builds.hq.northscale.net/latestbuilds')
466
467    def get_sustaining_latest_builds(self):
468        return self._get_and_parse_builds('http://builds.hq.northscale.net/latestbuilds/sustaining')
469
470    def get_all_builds(self, version=None, timeout=None, direct_build_url=None, deliverable_type=None, \
471                       architecture_type=None,edition_type=None, repo=None, toy="", \
472                       distribution_version=None, distribution_type=""):
473        try:
474            latestbuilds, latestchanges = \
475                self._get_and_parse_builds('http://builds.hq.northscale.net/latestbuilds', version=version, \
476                                           timeout=timeout, direct_build_url=direct_build_url, \
477                                           deliverable_type=deliverable_type, architecture_type=architecture_type, \
478                                           edition_type=edition_type,repo=repo, toy=toy, \
479                                           distribution_version=distribution_version, \
480                                           distribution_type=distribution_type)
481        except Exception as e:
482            latestbuilds, latestchanges = \
483                self._get_and_parse_builds('http://packages.northscale.com.s3.amazonaws.com/latestbuilds', \
484                                           version=version, timeout=timeout, direct_build_url=direct_build_url)
485
486        return latestbuilds, latestchanges
487
488
489    #baseurl = 'http://builds.hq.northscale.net/latestbuilds/'
490    def _get_and_parse_builds(self, build_page, version=None, timeout=None, direct_build_url=None, \
491                              deliverable_type=None, architecture_type=None, edition_type=None, \
492                              repo=None, toy="", distribution_version=None, distribution_type=""):
493        builds = []
494        changes = []
495        if direct_build_url is not None and direct_build_url != "":
496            query = BuildQuery()
497            build = query.create_build_info_from_direct_url(direct_build_url)
498            return build, changes
499        elif repo is not None and edition_type is not None and \
500             architecture_type is not None and deliverable_type is not None:
501            query = BuildQuery()
502            build = query.create_build_url(version, deliverable_type, architecture_type, \
503                                           edition_type, repo, toy, distribution_version, \
504                                           distribution_type)
505            return build, changes
506        else:
507            page = None
508            soup = None
509            index_url = '/index.html'
510            if version:
511                if version.find("-") != -1:
512                    index_url = "/index_" + version[:version.find("-")] + ".html"
513                else:
514                    index_url = "/index_" + version + ".html"
515            #try this ten times
516            for _ in range(0, 10):
517                try:
518                    self.log.info("Try collecting build information from url: %s" % (build_page + index_url))
519                    if timeout:
520                        socket.setdefaulttimeout(timeout)
521                    page = urllib2.urlopen(build_page + index_url)
522                    soup = BeautifulSoup.BeautifulSoup(page)
523                    break
524                except:
525                    time.sleep(5)
526            if not page:
527                raise Exception('unable to connect to %s' % (build_page + index_url))
528            query = BuildQuery()
529            for incident in soup('li'):
530                contents = incident.contents
531                build_id = ''
532                build_description = ''
533                for content in contents:
534                    if BeautifulSoup.isString(content):
535                        build_description = content.string
536                    elif content.name == 'a':
537                        build_id = content.string.string
538                try:
539                    if build_id.lower().startswith('changes'):
540                        change = query.create_change_info(build_id, build_description)
541                        change.url = '%s/%s' % (build_page, build_id)
542                        changes.append(change)
543                    else:
544                        build = query.create_build_info(build_id, build_description)
545                        build.url = '%s/%s' % (build_page, build_id)
546                        builds.append(build)
547                except Exception as e:
548                    print "ERROR in creating build/change info for: Build_id: %s , Build_Description: %s" % (build_id, build_description)
549                    print traceback.print_exc(file=sys.stderr)
550                    #raise e : Skipping parsing for this build information,
551                    #Eventually, It will fail with build not found error at install.py:240
552            for build in builds:
553                for change in changes:
554                    if change.build_number == build.product_version:
555                        build.change = change
556                        """ print 'change : ', change.url,change.build_number """
557                        break
558            return builds, changes
559
560    def create_build_info_from_direct_url(self, direct_build_url):
561        if direct_build_url is not None and direct_build_url != "":
562            build = MembaseBuild()
563            build.url = direct_build_url
564            build.toy = ""
565            build_info = direct_build_url.split("/")
566            build_info = build_info[len(build_info)-1]
567            """ windows build name: couchbase_server-enterprise-windows-amd64-3.0.0-892.exe
568                                    couchbase-server-enterprise_3.5.0-952-windows_amd64.exe """
569            build.name = build_info
570            deliverable_type = ["exe", "msi", "rpm", "deb", "zip"]
571            if build_info[-3:] in deliverable_type:
572                build.deliverable_type = build_info[-3:]
573                build_info = build_info[:-4]
574            else:
575                raise Exception('Check your url. Deliverable type %s does not support yet' \
576                                 % (direct_build_url[-3:]))
577            """ build name at this location couchbase-server-enterprise_x86_64_3.0.0-797-rel
578                windows build name: couchbase_server-enterprise-windows-amd64-3.0.0-892 """
579
580            """ Remove the code below when cb name is standardlized (MB-11372) """
581            if "windows" in direct_build_url and build.deliverable_type == "exe" \
582                and build_info not in SHERLOCK_VERSION:
583                build_info = build_info.replace("-windows-amd64-","_x86_64_")
584                build_info = build_info.replace("couchbase_server","couchbase-server")
585            """ End remove here """
586
587            """ sherlock build name
588                centos 6: couchbase-server-enterprise-3.5.0-71-centos6.x86_64
589                debian7:  couchbase-server-enterprise_3.5.0-10-debian7_amd64.deb
590                debian8:  couchbase-server-enterprise_4.5.0-1194-debian8_amd64.deb
591                ubuntu 12.04:
592                    couchbase-server-enterprise_3.5.0-723-ubuntu12.04_amd64.deb
593                mac:
594                    couchbase-server-enterprise_3.5.0-1120-macos_x86_64.zip
595                windows:
596                    couchbase_server-enterprise-windows-amd64-3.5.0-926.exe
597                    couchbase-server-enterprise_3.5.0-952-windows_amd64.exe"""
598
599            if any( x + "-" in build_info for x in COUCHBASE_FROM_VERSION_3):
600                deb_words = ["debian7", "debian8", "ubuntu12.04", "ubuntu14.04",
601                             "ubuntu16.04", "windows", "macos"]
602                if "centos" not in build_info and "suse" not in build_info:
603                    tmp_str = build_info.split("_")
604                    product_version = tmp_str[1].split("-")
605                    product_version = "-".join([i for i in product_version \
606                                                 if i not in deb_words])
607                else:
608                    product_version = build_info.split("-")
609                    product_version = product_version[3] + "-" + product_version[4]
610                if product_version[:5] in testconstants.COUCHBASE_VERSIONS:
611                    build.product_version = product_version
612                    if "centos" not in build_info and "suse" not in build_info:
613                        build_info = build_info.replace("_" + product_version,"")
614                    else:
615                        build_info = build_info.replace("-" + product_version,"")
616                if "x86_64" in build_info:
617                    build.architecture_type = "x86_64"
618                    if "centos" in build_info or "suse" in build_info:
619                        build_info = build_info.replace(".x86_64", "")
620                    elif "macos" in build_info:
621                        build_info = build_info.replace("_x86_64", "")
622                elif "x86" in build_info:
623                    build.architecture_type = "x86"
624                    build_info = build_info.replace(".x86", "")
625                elif "_amd64" in build_info:
626                    build.architecture_type = "x86_64"
627                    build_info = build_info.replace("_amd64", "")
628                elif "-amd64" in build_info:
629                    build.architecture_type = "x86_64"
630                    build_info = build_info.replace("-amd64", "")
631                del_words = ["centos6", "debian7", "debian8", "debian9",
632                             "ubuntu12.04", "ubuntu14.04", "ubuntu16.04",
633                             "windows", "macos", "centos7", "suse11", "suse12"]
634                if build_info.startswith("couchbase-server"):
635                    build.product = build_info.split("-")
636                    build.product = "-".join([i for i in build.product \
637                                                 if i not in del_words])
638                return build
639            product_version = build_info.split("_")
640            product_version = product_version[len(product_version)-1]
641            if product_version[:5] in testconstants.COUCHBASE_VERSIONS:
642                build.product_version = product_version
643                build_info = build_info.replace("_" + product_version,"")
644            else:
645                raise Exception("Check your url. Couchbase server does not have "
646                                       "version %s yet " % (product_version[:5]))
647
648            if "x86_64" in build_info:
649                build.architecture_type = "x86_64"
650                build_info = build_info.replace("_x86_64", "")
651            elif "x86" in build_info:
652                build.architecture_type = "x86"
653                build_info = build_info.replace("_x86", "")
654
655            if build_info.startswith("couchbase-server"):
656                build.product = build_info
657            else:
658                self.fail("unknown server name")
659            return build
660
661    def create_build_url(self, version, deliverable_type, architecture_type,
662                              edition_type, repo, toy, distribution_version,
663                                                         distribution_type):
664        build = MembaseBuild()
665        """
666        version: 3.0.0-xx or 3.0.0-xx-rel
667        deliverable_type: deb
668        distribution_version: ubuntu12 or debian7
669        architecture_type: x86_64
670        edition_type: couchbase-server-enterprise or couchbase-server-community
671        repo: http://builds.hq.northscale.net/latestbuilds/
672        sherlock repo: http://latestbuilds.hq.couchbase.com/couchbase-server/sherlock
673        sherlock build name with extra build number:
674               /684/couchbase-server-enterprise-3.5.0-684-centos6.x86_64.rpm
675               /1454/couchbase-server-enterprise-4.0.0-1454-centos6.x86_64.rpm
676               /1796/couchbase-server-enterprise-4.0.0-1796-oel6.x86_64.rpm
677               /723/couchbase-server-enterprise_3.5.0-723-ubuntu12.04_amd64.deb
678               /723/couchbase-server-enterprise_3.5.0-732-debian7_amd64.deb
679               /1194/couchbase-server-enterprise_4.5.0-1194-debian8_amd64.deb
680               /1120/couchbase-server-enterprise_3.5.0-1120-macos_x86_64.zip
681        toy=Ce
682        build.name = couchbase-server-enterprise_x86_64_3.0.0-xx-rel.deb
683        build.url = http://builds.hq.northscale.net/latestbuilds/
684                              couchbase-server-enterprise_x86_64_3.0.0-xx-rel.deb
685        For toy build: name  =
686            couchbase-server-community_cent58-3.0.0-toy-toyName-x86_64_3.0.0-xx-toy.rpm
687            http://latestbuilds.hq.couchbase.com/couchbase-server/
688                toy-wied/14/couchbase-server-enterprise-1004.0.0-14-centos6.x86_64.rpm
689                toy-nimish/16/couchbase-server-enterprise_1003.5.0-16-windows_amd64.exe
690        For windows build diff - and _ compare to unix build
691                       name = couchbase_server-enterprise-windows-amd64-3.0.0-998.exe
692                              couchbase_server-enterprise-windows-amd64-3.0.2-1603.exe
693                              couchbase_server-enterprise-windows-amd64-3.0.3-1716.exe
694                              couchbase-server-enterprise_3.5.0-952-windows_amd64.exe
695                              couchbase-server-enterprise_3.5.0-1390-windows_x86.exe
696            From build 5.0.0-2924, we don't make any exe build.
697            It will be all in msi
698        """
699        build.toy = "toy-" + toy
700        build.deliverable_type = deliverable_type
701        build.architecture_type = architecture_type
702        build.distribution_version = distribution_version
703        build.distribution_type = distribution_type
704
705        os_name = ""
706        setup = ""
707        build_number = ""
708
709        unix_deliverable_type = ["deb", "rpm", "zip"]
710        if deliverable_type in unix_deliverable_type:
711            if toy == "" and version[:5] not in COUCHBASE_VERSION_2 and \
712                                   version[:5] not in COUCHBASE_VERSION_3:
713                if "rel" not in version and toy == "":
714                    build.product_version = version
715                elif "-rel" in version:
716                    build.product_version = version.replace("-rel", "")
717            elif toy != "":
718                build.product_version = version
719            else:
720                if "rel" not in version and toy == "":
721                    build.product_version = version + "-rel"
722                else:
723                    build.product_version = version
724        if deliverable_type in ["exe", "msi"]:
725            if version[:5] in COUCHBASE_VERSION_2:
726                setup = "setup."
727            else:
728                os_name= "windows-"
729            if "rel" in version and version[:5] not in COUCHBASE_VERSION_2:
730                build.product_version = version.replace("-rel", "")
731            elif "rel" not in version and version[:5] in COUCHBASE_VERSION_2:
732                build.product_version = version + "-rel"
733            else:
734                build.product_version = version
735            if "couchbase-server" in edition_type and version[:5] in WIN_CB_VERSION_3:
736                edition_type = edition_type.replace("couchbase-", "couchbase_")
737            if version[:5] not in COUCHBASE_VERSION_2:
738                if "x86_64" in architecture_type:
739                    build.architecture_type = "amd64"
740                elif "x86" in architecture_type:
741                    build.architecture_type = "x86"
742            """
743                    In spock from build 2924 and later release, we only support
744                    msi installation method on windows
745            """
746            if "-" in version and version.split("-")[0] in COUCHBASE_FROM_SPOCK:
747                deliverable_type = "msi"
748
749        if "deb" in deliverable_type and "centos6" in edition_type:
750            edition_type = edition_type.replace("centos6", "ubuntu_1204")
751        if "debian" in distribution_version:
752            os_name = "debian7_"
753        joint_char = "_"
754        version_join_char = "_"
755        if toy is not "":
756            joint_char = "-"
757        if "exe" in deliverable_type and version[:5] not in COUCHBASE_VERSION_2:
758            joint_char = "-"
759            version_join_char = "-"
760        if toy == "" and version[:5] not in COUCHBASE_VERSION_2 and \
761                                   version[:5] not in COUCHBASE_VERSION_3:
762            """ format for sherlock build name
763            /684/couchbase-server-enterprise-3.5.0-684-centos6.x86_64.rpm
764            /1154/couchbase-server-enterprise-3.5.0-1154-centos7.x86_64.rpm
765            /1454/couchbase-server-enterprise-4.0.0-1454-centos6.x86_64.rpm
766            /1796/couchbase-server-enterprise-4.0.0-1796-oel6.x86_64.rpm
767            /723/couchbase-server-enterprise_3.5.0-723-ubuntu12.04_amd64.deb
768            /723/couchbase-server-enterprise_3.5.0-732-debian7_amd64.deb
769            /795/couchbase_server-enterprise-windows-amd64-3.5.0-795.exe
770            /952/couchbase-server-enterprise_3.5.0-952-windows_amd64.exe
771            /1390/couchbase-server-enterprise_3.5.0-1390-windows_x86.exe
772            /1120/couchbase-server-enterprise_3.5.0-1120-macos_x86_64.zip"""
773            build_number = build.product_version.replace(version[:6],"")
774            """ distribution version:    centos linux release 7.0.1406 (core)
775                distribution version:    centos release 6.5 (final)  """
776            centos_version = "centos6"
777
778            if "centos" in distribution_version or "red hat" in distribution_version:
779                if "centos 7" in distribution_version:
780                    centos_version = "centos7"
781                elif "red hat enterprise linux server release 6" in distribution_version:
782                    centos_version = "centos6"
783                elif "red hat enterprise linux server release 7" in distribution_version:
784                    centos_version = "centos7"
785                build.name = edition_type + "-" + build.product_version + \
786                   "-" + centos_version + "." + build.architecture_type + \
787                   "." + build.deliverable_type
788            elif "suse" in distribution_version:
789                if "suse 12" in distribution_version:
790                    if version[:5] in COUCHBASE_FROM_SPOCK:
791                        suse_version="suse12"
792                        build.distribution_version = "suse12"
793                    else:
794                        self.fail("suse 12 does not support on this version %s "
795                                                                  % version[:5])
796                else:
797                    suse_version="suse11"
798                    build.distribution_version = "suse11"
799                build.name = edition_type + "-" + build.product_version + \
800                   "-" + suse_version + "." + build.architecture_type + \
801                   "." + build.deliverable_type
802            elif "oracle linux" in distribution_version:
803                build.distribution_version = "oracle linux"
804                os_name = "oel6"
805                build.name = edition_type + "-" + build.product_version + \
806                   "-" + os_name + "." + build.architecture_type + \
807                   "." + build.deliverable_type
808            else:
809                os_name = ""
810                joint_char = "-"
811
812                """ sherlock build in unix only support 64-bit """
813                build.architecture_type = "amd64"
814                if  "ubuntu 12.04" in distribution_version:
815                    os_name = "ubuntu12.04"
816                elif "ubuntu 14.04" in distribution_version:
817                    os_name = "ubuntu14.04"
818                elif "ubuntu 16.04" in distribution_version:
819                    os_name = "ubuntu16.04"
820                elif "debian gnu/linux 7" in distribution_version:
821                    build.distribution_version = "debian7"
822                    os_name = "debian7"
823                elif "debian gnu/linux 8" in distribution_version:
824                    build.distribution_version = "debian8"
825                    os_name = "debian8"
826                elif "debian gnu/linux 9" in distribution_version:
827                    build.distribution_version = "debian9"
828                    os_name = "debian9"
829                elif "windows" in distribution_version:
830                    os_name = "windows"
831                    if "x86_64" not in architecture_type:
832                        build.architecture_type = "x86"
833                elif "mac" in distribution_type:
834                    os_name = "macos"
835                    build.architecture_type = "x86_64"
836                build.name = edition_type + "_" + build.product_version + \
837                   joint_char + os_name + "_" +  build.architecture_type + \
838                   "." + build.deliverable_type
839            build.url = repo + build_number + "/" + build.name
840        elif toy is not "":
841            centos_version = "centos6"
842            build_info = version.split("-")
843            build_number = build_info[1]
844            if "centos" in distribution_version:
845                build.name = edition_type + "-" + build.product_version + \
846                   "-" + centos_version + "." + build.architecture_type + \
847                   "." + build.deliverable_type
848            build.url = repo + build.toy + "/" +build_number \
849                        + "/" + build.name
850        elif version[:3] == "3.1":
851            os_name = ""
852            if "suse" in distribution_version:
853                build.distribution_version = "suse11"
854                os_name = "suse11_"
855            elif "centos release 6" in distribution_version:
856                build.distribution_version = "centos6"
857                os_name = "centos6_"
858            elif  "ubuntu 12.04" in distribution_version:
859                os_name = "ubuntu_1204_"
860            elif "debian gnu/linux 7" in distribution_version:
861                build.distribution_version = "debian7"
862                os_name = "debian7_"
863            elif "debian gnu/linux 8" in distribution_version:
864                build.distribution_version = "debian8"
865                os_name = "debian8_"
866            elif "windows" in distribution_version:
867                os_name = "windows-"
868                if "x86_64" not in architecture_type:
869                    build.architecture_type = "x86"
870            elif "mac" in distribution_type:
871                build.architecture_type = "x86_64"
872            build.name = edition_type + joint_char + os_name + \
873                build.architecture_type +  version_join_char + \
874                build.product_version + "." + setup + build.deliverable_type
875            build.url = repo + build.name
876        else:
877            build.name = edition_type + joint_char + os_name + \
878                build.architecture_type +  version_join_char + \
879                build.product_version + "." + setup + build.deliverable_type
880            build.url = repo + build.name
881
882
883        """ reset build.architecture back to x86_64 in windows """
884        build.architecture_type = architecture_type
885        return build
886
887    def create_build_info(self, build_id, build_decription):
888        build = MembaseBuild()
889        build.deliverable_type = self._product_deliverable_type(build_id)
890        build.time = self._product_time(build_decription)
891        build.size = self._product_size(build_decription)
892        build.product_version = self._product_version(build_id)
893        build.architecture_type = self._product_arch_type(build_id)
894        build.product = self._product_name(build_id)
895        build.name = build_id
896        build.build_number = self._build_number(build)
897        build.toy = self._product_toy(build_id)
898        return build
899
900    def create_change_info(self, build_id, build_decription):
901        change = MembaseChange()
902        change.name = build_id.strip()
903        change.build_number = self._change_build_number(build_id)
904        change.time = self._change_time(build_decription)
905        return change
906
907
908    def _product_name(self, build_id):
909        list = build_id.split('_')
910        if "centos6" in build_id:
911            # return couchbase-server-ent/com_centos6
912            return "_".join(list[:2])
913        elif "ubuntu_1204" in build_id:
914            # return couchbase-server-ent/com_ubuntu_1204
915            return "_".join(list[:3])
916        # this should be done w/ more generic rule for toy-split
917        elif "cent54" in build_id:
918            list = build_id.split("-toy")
919            return list[0]
920        else:
921            return list[0]
922        #the first one is the product
923
924    def _product_arch_type(self, build_id):
925        list = build_id.split('_')
926        if '64' in build_id.split('_') or build_id.find('x86_64') != -1:
927            return 'x86_64'
928        elif 'x86' in build_id.split('_'):
929            return 'x86'
930        return ''
931
932
933    def _product_toy(self, build_id):
934        r = re.search("[^_]+-toy-([\w-]*)-x86", build_id)
935        if r:
936            return r.group(1)
937        return ''
938
939    def _change_time(self, build_description):
940        list = build_description.split('/')
941        timestamp = list[1].strip()
942        timestamp = timestamp[:timestamp.index(')')]
943        return datetime.strptime(timestamp, '%a %b %d %H:%M:%S %Y')
944
945    def _change_build_number(self, build_id):
946        list = build_id.split('_')
947        #get list[1] . get rid of .txt
948        build_number = list[1].strip()
949        if re.search('.txt', build_number):
950            build_number = build_number[:build_number.index('.txt')]
951            return build_number
952
953    def _build_number(self, build):
954        #get the first - and then the first - after that
955        first_dash = build.product_version.find('-')
956        if first_dash != -1:
957            second_dash = build.product_version.find('-', first_dash + 1)
958            if second_dash != -1:
959                try:
960                    return int(build.product_version[first_dash + 1:second_dash])
961                except Exception:
962                    return -1
963        return -1
964
965    def _product_version(self, build_id):
966        list = build_id.split('_')
967        version_item = ''
968        for item in list:
969            if re.match(r'[0-2].[0-9].[0-9]-[0-9]+-rel', item):
970                version_item = item
971                if list[-1].endswith('xml'):
972                    break
973                return version_item
974        if version_item == '':
975            for item in list:
976                if item.endswith('.setup.exe') or item.endswith('rpm') or\
977                   item.endswith('deb') or item.endswith('tar.gz') or item.endswith('zip'):
978                    version_item = item
979                    break
980        if version_item != '':
981            if version_item.endswith('.setup.exe'):
982                return version_item[:version_item.index('.setup.exe')]
983            elif version_item.endswith('.tar.gz'):
984                return version_item[:version_item.index('.tar.gz')]
985            elif version_item.endswith('.deb'):
986                return version_item[:version_item.index('.deb')]
987            elif version_item.endswith('.rpm'):
988                return version_item[:version_item.index('.rpm')]
989            elif version_item.endswith('.zip'):
990                return version_item[:version_item.index('.zip')]
991        return ''
992
993    def _product_deliverable_type(self, build_id=''):
994        list = build_id.split('_')
995        version_item = ''
996        for item in list:
997            if item.endswith('.setup.exe') or item.endswith('rpm') or\
998               item.endswith('deb') or item.endswith('tar.gz') or item.endswith('zip'):
999                version_item = item
1000                break
1001        if version_item != '':
1002            if version_item.endswith('.setup.exe'):
1003                return 'exe'
1004            elif version_item.endswith('.tar.gz'):
1005                return 'tar.gz'
1006            elif version_item.endswith('.deb'):
1007                return 'deb'
1008            elif version_item.endswith('.rpm'):
1009                return 'rpm'
1010            elif version_item.endswith('.zip'):
1011                return 'zip'
1012        return ''
1013
1014    def _product_time(self, build_description):
1015        list = build_description.split('/')
1016        timestamp = list[1].strip()
1017        timestamp = timestamp[:timestamp.index(')')]
1018        return datetime.strptime(timestamp, '%a %b %d %H:%M:%S %Y')
1019
1020    def _product_size(self, build_description):
1021        list = build_description.split('/')
1022        filesize = list[0]
1023        filesize = filesize[filesize.index('(') + 1:]
1024        return filesize.strip()
1025
1026#q = BuildQuery()
1027#builds, changes = q.get_latest_builds()
1028#for build in builds:
1029#    print build.product,' ',build.time ,' ',build.deliverable_type,' ',build.product_version ,'',build.size,'',build.architecture_type
1030#    if build.change:
1031#        change = build.change
1032#        print change.name,change.build_number,change.time,change.url
1033
1034#for change in changes:
1035#    print change.name,change.build_number,change.time
1036
1037#builds = q.get_membase_latest_builds()
1038#for build in builds:
1039#    print build.product,' ',build.time ,' ',build.deliverable_type,' ',build.product_version ,'',build.size,'',build.architecture_type
1040
1041