xref: /6.0.3/testrunner/lib/builds/build_query.py (revision 6e673383)
1#this class will contain methods which we
2#use later to
3# map a version # -> rpm url
4from datetime import datetime
5import time
6import urllib2
7import re
8import socket
9import BeautifulSoup
10import testconstants
11import logger
12import traceback
13import sys
14from testconstants import WIN_CB_VERSION_3
15from testconstants import SHERLOCK_VERSION
16from testconstants import COUCHBASE_VERSION_2
17from testconstants import COUCHBASE_VERSION_3
18from testconstants import COUCHBASE_VERSION_2_WITH_REL
19from testconstants import COUCHBASE_RELEASE_FROM_VERSION_3,\
20                          COUCHBASE_RELEASE_FROM_SPOCK
21from testconstants import COUCHBASE_FROM_VERSION_3, COUCHBASE_FROM_SPOCK, \
22                          COUCHBASE_FROM_ALICE, COUCHBASE_FROM_MAD_HATTER, \
23                          COUCHBASE_FROM_601
24from testconstants import CB_RELEASE_REPO
25from testconstants import CB_LATESTBUILDS_REPO
26from testconstants import CE_EE_ON_SAME_FOLDER
27
28
29class MembaseBuild(object):
30    def __init__(self):
31        self.url = ''
32        self.name = ''
33        self.time = ''
34        self.size = ''
35        self.product = ''
36        self.product_version = ''
37        self.build_number = 0
38        self.os = ''
39        self.deliverable_type = ''
40        self.architecture_type = ''
41        self.toy = ''
42        self.change = None  # a MembaseChange
43        self.url_latest_build = ''
44
45    def __repr__(self):
46        return self.__str__()
47
48    #let's also have a json object for all these classes
49    def __str__(self):
50        url = 'url : {0}'.format(self.url)
51        name = 'name : {0}'.format(self.name)
52        product = 'product : {0}'.format(self.product)
53        product_version = 'product_version : {0}'.format(self.product_version)
54        os = 'os : {0}'.format(self.os)
55        deliverable_type = 'deliverable_type : {0}'.format(self.deliverable_type)
56        architecture_type = 'architecture_type : {0}'.format(self.architecture_type)
57        if self.toy:
58            toy = 'toy : {0}'.format(self.toy)
59        else:
60            toy = ''
61        return '{0} {1} {2} {3} {4} {5} {6} {7}'.format(url, name, product, product_version, os, deliverable_type,
62                                                    architecture_type, toy)
63
64
65class MembaseChange(object):
66    def __init__(self):
67        self.url = ''
68        self.name = ''
69        self.time = ''
70        self.build_number = ''
71
72
73class BuildQuery(object):
74    def __init__(self):
75        self.log = logger.Logger.get_logger()
76        pass
77
78    # let's look at buildlatets or latest/sustaining or any other
79    # location
80    def parse_builds(self):
81        #parse build page and create build object
82        pass
83
84    def find_build(self, builds, product, type, arch, version, toy='', openssl='', \
85                   direct_build_url=None, distribution_version=None, \
86                   distribution_type=""):
87        if direct_build_url is None:
88            if not isinstance(builds, list) and builds.url is not None:
89                return builds
90            else:
91                for build in builds:
92                    if build.product_version.find(version) != -1 and product == build.product\
93                       and build.architecture_type == arch and type == build.deliverable_type\
94                       and build.toy == toy:
95                        return build
96        elif direct_build_url != "":
97            if "exe" in builds.deliverable_type:
98                if "rel" in version and version[:5] in WIN_CB_VERSION_3:
99                    version = version.replace("-rel", "")
100            """ direct url only need one build """
101
102            """ if the job trigger with url, no need to check version.
103                remove builds.product_version.find(version) != -1 """
104            if product == builds.product and builds.architecture_type == arch:
105                return builds
106            else:
107                self.log.info("if build not found, url link may not match...")
108        return None
109
110    def find_membase_build(self, builds, product, deliverable_type, os_architecture, build_version, is_amazon=False):
111        if is_amazon:
112            build = BuildQuery().find_build(builds, product, deliverable_type,
113                                            os_architecture, build_version)
114            if build:
115                build.url = build.url.replace("http://builds.hq.northscale.net", \
116                                                  "http://packages.northscale.com.s3.amazonaws.com")
117                build.url = build.url.replace("enterprise", "community")
118                build.name = build.name.replace("enterprise", "community")
119                return build
120
121        for build in builds:
122            if build.product_version.find(build_version) != -1 and product == build.product\
123               and build.architecture_type == os_architecture and deliverable_type == build.deliverable_type:
124                return build
125
126        return None
127
128    def find_membase_build_with_version(self, builds, build_version):
129        for build in builds:
130            if build.product_version == build_version or build.product_version.find(build_version) != -1:
131                #or if it starts with that version ?
132                return build
133        return None
134
135    def find_membase_release_build(self, product, deliverable_type, os_architecture,
136                                    build_version, is_amazon=False, os_version=""):
137        build_details = build_version
138        if build_version[:5] in COUCHBASE_VERSION_2_WITH_REL:
139            if build_version[-4:] != "-rel":
140                build_details = build_details + "-rel"
141        elif build_version.startswith("1.8.0"):
142            build_details = "1.8.0r-55-g80f24f2"
143            product = "couchbase-server-enterprise"
144        build = MembaseBuild()
145        build.deliverable_type = deliverable_type
146        build.time = '0'
147        build.size = '0'
148        build.product_version = build_version
149        build.architecture_type = os_architecture
150        build.product = product
151        os_name = ""
152        build.name = '{1}_{2}_{0}.{3}'.format(build_version, product,
153                                               os_architecture, deliverable_type)
154        build.build_number = 0
155        if deliverable_type == "exe":
156            """ /3.0.1/couchbase-server-enterprise_3.0.1-windows_amd64.exe """
157            if not re.match(r'[1-9].[0-9].[0-9]$', build_version):
158                if build_version[:5] in COUCHBASE_RELEASE_FROM_VERSION_3:
159                    arch_type = "amd64"
160                    if "x86_64" not in os_architecture:
161                        arch_type = "x86"
162                    build.url = "{5}{0}/{1}_{4}-windows_{2}.{3}"\
163                        .format(build_version[:build_version.find('-')],
164                        product, arch_type, deliverable_type, build_details[:5],
165                        CB_RELEASE_REPO)
166                else:
167                    if "2.5.2" in build_details[:5]:
168                        build.url = "{5}{0}/{1}_{4}_{2}.setup.{3}"\
169                            .format(build_version[:build_version.find('-')],
170                            product, os_architecture, deliverable_type,
171                            build_details[:5], CB_RELEASE_REPO)
172                    else:
173                        build.url = "{5}{0}/{1}_{2}_{4}.setup.{3}"\
174                            .format(build_version[:build_version.find('-')],
175                            product, os_architecture, deliverable_type,
176                            build_details, CB_RELEASE_REPO)
177            else:
178                if build_version[:5] in COUCHBASE_RELEASE_FROM_VERSION_3:
179                    arch_type = "amd64"
180                    if "x86_64" not in os_architecture:
181                        arch_type = "x86"
182                    build.url = "{5}{0}/{1}_{4}-windows_{2}.{3}"\
183                        .format(build_version, product, arch_type,
184                        deliverable_type, build_details[:5], CB_RELEASE_REPO)
185                else:
186                    build.url = "{5}{0}/{1}_{2}_{4}.setup.{3}"\
187                        .format(build_version, product, os_architecture,
188                        deliverable_type, build_details, CB_RELEASE_REPO)
189            build.url_latest_build = "{4}{0}_{1}_{3}.setup.{2}"\
190                             .format(product, os_architecture, deliverable_type,
191                                            build_details, CB_LATESTBUILDS_REPO)
192        else:
193            """ check match full version x.x.x-xxxx """
194            if not re.match(r'[1-9].[0-9].[0-9]$', build_version):
195                """  in release folder
196                        /3.0.1/couchbase-server-enterprise-3.0.1-centos6.x86_64.rpm
197                        /3.0.1/couchbase-server-enterprise_3.0.1-ubuntu12.04_amd64.deb
198                        /3.0.2/couchbase-server-enterprise-3.0.2-centos6.x86_64.rpm
199                      build release url:
200                               http://builds.hq.northscale.net/releases/3.0.1/
201                      build latestbuilds url:
202                               http://builds.hq.northscale.net/latestbuilds/
203                                  couchbase-server-enterprise_x86_64_3.0.1-1444.rpm
204                """
205                if build_version[:5] in COUCHBASE_RELEASE_FROM_VERSION_3:
206                    if "rpm" in deliverable_type:
207                        build.url = "{5}{0}/{1}-{4}-centos6.{2}.{3}"\
208                                .format(build_version[:build_version.find('-')],
209                                product, os_architecture, deliverable_type,
210                                        build_details[:5], CB_RELEASE_REPO)
211                    elif "deb" in deliverable_type:
212                        os_architecture = "amd64"
213                        os_name = "ubuntu12.04"
214                        if  "ubuntu 14.04" in os_version:
215                            os_name = "ubuntu14.04"
216                        elif "ubuntu 16.04" in os_version:
217                            os_name = "ubuntu16.04"
218                        build.url = "{6}{0}/{1}_{4}-{5}_{2}.{3}"\
219                                .format(build_version[:build_version.find('-')],
220                                 product, os_architecture, deliverable_type,
221                                 build_details[:5], os_name, CB_RELEASE_REPO)
222                else:
223                    if "2.5.2" in build_details[:5]:
224                        build.url = "{5}{0}/{1}_{4}_{2}.{3}"\
225                            .format(build_version[:build_version.find('-')],
226                            product, os_architecture, deliverable_type,
227                                    build_details[:5], CB_RELEASE_REPO)
228                    else:
229                        build.url = "{5}{0}/{1}_{2}_{4}.{3}"\
230                            .format(build_version[:build_version.find('-')],
231                            product, os_architecture, deliverable_type,
232                                        build_details, CB_RELEASE_REPO)
233            else:
234                if build_version[:5] in COUCHBASE_RELEASE_FROM_VERSION_3:
235                    if "rpm" in deliverable_type:
236                        build.url = "{5}{0}/{1}-{4}-centos6.{2}.{3}"\
237                            .format(build_version, product, os_architecture,
238                            deliverable_type, build_details[:5], CB_RELEASE_REPO)
239                    elif "deb" in deliverable_type:
240                        os_architecture = "amd64"
241                        os_name = "ubuntu12.04"
242                        if  "ubuntu 14.04" in os_version:
243                            os_name = "ubuntu14.04"
244                        elif "ubuntu 16.04" in os_version:
245                            os_name = "ubuntu16.04"
246                        build.url = "{6}{0}/{1}_{4}-{5}_{2}.{3}"\
247                            .format(build_version, product, os_architecture,
248                            deliverable_type, build_details[:5], os_name,
249                                                         CB_RELEASE_REPO)
250                        """ http://builds.hq.northscale.net/releases/3.0.1/
251                        couchbase-server-enterprise_3.0.1-ubuntu12.04_amd64.deb """
252                else:
253                    build.url = "{5}{0}/{1}_{2}_{4}.{3}"\
254                        .format(build_version, product, os_architecture,
255                        deliverable_type, build_details, CB_RELEASE_REPO)
256            build.url_latest_build = "{4}{0}_{1}_{3}.{2}"\
257                      .format(product, os_architecture, deliverable_type,
258                               build_details, CB_LATESTBUILDS_REPO)
259        # This points to the Internal s3 account to look for release builds
260        if is_amazon:
261            build.url = 'https://s3.amazonaws.com/packages.couchbase/releases/{0}/{1}_{2}_{0}.{3}'\
262                .format(build_version, product, os_architecture, deliverable_type)
263            build.url = build.url.replace("enterprise", "community")
264            build.name = build.name.replace("enterprise", "community")
265        return build
266
267    def find_couchbase_release_build(self, product, deliverable_type, os_architecture,
268                                    build_version, is_amazon=False, os_version="",
269                                    direct_build_url=None):
270        build_details = build_version
271        if build_version[:5] in COUCHBASE_VERSION_2_WITH_REL:
272            if build_version[-4:] != "-rel":
273                build_details = build_details + "-rel"
274        build = MembaseBuild()
275        build.deliverable_type = deliverable_type
276        build.time = '0'
277        build.size = '0'
278        build.product_version = build_version
279        build.architecture_type = os_architecture
280        build.product = product
281        os_name = ""
282        build.name = '{1}_{2}_{0}.{3}'.format(build_version, product,
283                                               os_architecture, deliverable_type)
284        build.build_number = 0
285        if deliverable_type == "exe":
286            """ /3.0.1/couchbase-server-enterprise_3.0.1-windows_amd64.exe """
287            if not re.match(r'[1-9].[0-9].[0-9]$', build_version):
288                if build_version[:5] in COUCHBASE_RELEASE_FROM_VERSION_3:
289                    arch_type = "amd64"
290                    if "x86_64" not in os_architecture:
291                        arch_type = "x86"
292                    build.url = "{5}{0}/{1}_{4}-windows_{2}.{3}"\
293                            .format(build_version[:build_version.find('-')],
294                            product, arch_type, deliverable_type, build_details[:5],
295                            CB_RELEASE_REPO)
296                else:
297                    if "2.5.2" in build_details[:5]:
298                        build.url = "{5}/{0}/{1}_{4}_{2}.setup.{3}"\
299                            .format(build_version[:build_version.find('-')],
300                            product, os_architecture, deliverable_type,
301                            build_details[:5], CB_RELEASE_REPO)
302                    else:
303                        build.url = "{5}{0}/{1}_{2}_{4}.setup.{3}"\
304                            .format(build_version[:build_version.find('-')],
305                            product, os_architecture, deliverable_type,
306                            build_details, CB_RELEASE_REPO)
307            else:
308                if build_version[:5] in COUCHBASE_RELEASE_FROM_VERSION_3:
309                    arch_type = "amd64"
310                    if "x86_64" not in os_architecture:
311                        arch_type = "x86"
312                    build.url = "{5}{0}/{1}_{4}-windows_{2}.{3}"\
313                        .format(build_version, product, arch_type,
314                         deliverable_type, build_details[:5], CB_RELEASE_REPO)
315                else:
316                    build.url = "{5}{0}/{1}_{2}_{4}.setup.{3}"\
317                        .format(build_version, product, os_architecture,
318                        deliverable_type, build_details, CB_RELEASE_REPO)
319            build.url_latest_build = "{4}{0}_{1}_{3}.setup.{2}"\
320                             .format(product, os_architecture,
321                             deliverable_type, build_details, CB_LATESTBUILDS_REPO)
322        elif deliverable_type == "msi":
323            if not re.match(r'[1-9].[0-9].[0-9]$', build_version):
324                if build_version[:5] in COUCHBASE_RELEASE_FROM_SPOCK:
325                    arch_type = "amd64"
326                    if "x86_64" not in os_architecture:
327                        arch_type = "x86"
328                    build.url = "{5}{0}/{1}_{4}-windows_{2}.{3}"\
329                            .format(build_version[:build_version.find('-')],
330                            product, arch_type, deliverable_type, build_details[:5],
331                            CB_RELEASE_REPO)
332            else:
333                if build_version[:5] in COUCHBASE_RELEASE_FROM_SPOCK:
334                    arch_type = "amd64"
335                    if "x86_64" not in os_architecture:
336                        arch_type = "x86"
337                    build.url = "{5}{0}/{1}_{4}-windows_{2}.{3}"\
338                        .format(build_version, product, arch_type,
339                         deliverable_type, build_details[:5], CB_RELEASE_REPO)
340        else:
341            """ check match full version x.x.x-xxxx """
342            if not re.match(r'[1-9].[0-9].[0-9]$', build_version):
343                """  in release folder
344                        /3.0.1/couchbase-server-enterprise-3.0.1-centos6.x86_64.rpm
345                        /3.0.1/couchbase-server-enterprise_3.0.1-ubuntu12.04_amd64.deb
346                        /3.0.2/couchbase-server-enterprise-3.0.2-centos6.x86_64.rpm
347                      build release url:
348                               http://builds.hq.northscale.net/releases/3.0.1/
349                      build latestbuilds url:
350                               http://builds.hq.northscale.net/latestbuilds/
351                                  couchbase-server-enterprise_x86_64_3.0.1-1444.rpm
352                """
353                if build_version[:5] in COUCHBASE_RELEASE_FROM_VERSION_3:
354                    if "rpm" in deliverable_type:
355                        if "centos" in os_version.lower():
356                            if "centos 7" in os_version.lower():
357                                os_name = "centos7"
358                            else:
359                                os_name = "centos6"
360                        elif "suse" in os_version.lower():
361                            if "11" in os_version.lower():
362                                os_name = "suse11"
363                            elif "12" in os_version.lower():
364                                os_name = "suse12"
365                        elif "oracle linux" in os_version.lower():
366                            os_name = "oel6"
367                        elif "amazon linux 2" in os_version.lower():
368                            if build_version[:5] in COUCHBASE_FROM_MAD_HATTER or \
369                                            build_version[:5] in COUCHBASE_FROM_601:
370                                os_name = "amzn2"
371                            else:
372                                self.fail("Amazon Linux 2 doesn't support version {0} "\
373                                                              .format(build_version[:5]))
374                        elif "red hat" in os_version.lower():
375                            if "8.0" in os_version.lower():
376                                os_name = "rhel8"
377                        else:
378                            os_name = "centos6"
379                        build.url = "{6}{0}/{1}-{4}-{5}.{2}.{3}"\
380                                .format(build_version[:build_version.find('-')],
381                                product, os_architecture, deliverable_type,
382                                build_details[:5],os_name, CB_RELEASE_REPO)
383                    elif "deb" in deliverable_type:
384                        os_architecture = "amd64"
385                        os_name = "ubuntu12.04"
386                        if  "ubuntu 14.04" in os_version.lower():
387                            os_name = "ubuntu14.04"
388                        elif "ubuntu 16.04" in os_version.lower():
389                            os_name = "ubuntu16.04"
390                        elif "ubuntu 18.04" in os_version.lower():
391                            if build_version[:5] in COUCHBASE_FROM_601:
392                                os_name = "ubuntu18.04"
393                            else:
394                                self.fail("ubuntu 18.04 doesn't support version %s "
395                                                                % build_version[:5])
396                        build.url = "{6}{0}/{1}_{4}-{5}_{2}.{3}"\
397                                .format(build_version[:build_version.find('-')],
398                                 product, os_architecture, deliverable_type,
399                                 build_details[:5], os_name, CB_RELEASE_REPO)
400                else:
401                    if "2.5.2" in build_details[:5]:
402                        if product == "moxi-server" and deliverable_type == "deb":
403                            build.url = "{5}{0}/{1}_{4}_{2}_openssl098.{3}".format(
404                                            build_version[:build_version.find('-')],
405                                            product,
406                                            os_architecture,
407                                            deliverable_type,
408                                            build_details[:5],
409                                            CB_RELEASE_REPO)
410                        else:
411                            build.url = "{5}{0}/{1}_{4}_{2}.{3}".format(
412                                            build_version[:build_version.find('-')],
413                                            product,
414                                            os_architecture,
415                                            deliverable_type,
416                                            build_details[:5],
417                                            CB_RELEASE_REPO)
418                    else:
419                        build.url = "{5}{0}/{1}_{2}_{4}.{3}"\
420                            .format(build_version[:build_version.find('-')],
421                            product, os_architecture, deliverable_type,
422                            build_details, CB_RELEASE_REPO)
423            else:
424                if build_version[:5] in COUCHBASE_RELEASE_FROM_VERSION_3:
425                    if "rpm" in deliverable_type:
426                        if "centos" in os_version.lower():
427                            if "centos 7" in os_version.lower():
428                                os_name = "centos7"
429                            else:
430                                os_name = "centos6"
431                        elif "suse" in os_version.lower():
432                            if "11" in os_version.lower():
433                                os_name = "suse11"
434                            elif "12" in os_version.lower():
435                                os_name = "suse12"
436                        elif "oracle linux" in os_version.lower():
437                            os_name = "oel6"
438                        elif "amazon linux 2" in os_version.lower():
439                            if build_version[:5] in COUCHBASE_FROM_MAD_HATTER or \
440                                            build_version[:5] in COUCHBASE_FROM_601:
441                                os_name = "amzn2"
442                            else:
443                                self.fail("Amazon Linux 2 doesn't support version %s "
444                                          % build_version[:5])
445                        else:
446                            os_name = "centos6"
447                        build.url = "{6}{0}/{1}-{4}-{5}.{2}.{3}"\
448                                .format(build_version[:build_version.find('-')],
449                                product, os_architecture, deliverable_type,
450                                build_details[:5],os_name, CB_RELEASE_REPO)
451                    elif "deb" in deliverable_type:
452                        os_architecture = "amd64"
453                        os_name = "ubuntu12.04"
454                        if  "ubuntu 14.04" in os_version.lower():
455                            os_name = "ubuntu14.04"
456                        elif "ubuntu 16.04" in os_version.lower():
457                            os_name = "ubuntu16.04"
458                        elif "ubuntu 18.04" in os_version.lower():
459                            if build_version[:5] in COUCHBASE_FROM_601:
460                                os_name = "ubuntu18.04"
461                            else:
462                                self.fail("ubuntu 18.04 doesn't support version %s "
463                                                                % build_version[:5])
464                        build.url = "{6}{0}/{1}_{4}-{5}_{2}.{3}"\
465                            .format(build_version, product, os_architecture,
466                            deliverable_type, build_details[:5], os_name,
467                            CB_RELEASE_REPO)
468                else:
469                    build.url = "{5}{0}/{1}_{2}_{4}.{3}".format(build_version,
470                                product, os_architecture, deliverable_type,
471                                build_details, CB_RELEASE_REPO)
472            build.url_latest_build = "{4}{0}_{1}_{3}.{2}".format(product,
473                        os_architecture, deliverable_type, build_details,
474                        CB_LATESTBUILDS_REPO)
475        # This points to the Internal s3 account to look for release builds
476        """ add ce folder in community version from 3.0.2 release """
477        if "community" in product and build_version[:5] not in CE_EE_ON_SAME_FOLDER:
478            build.url = build.url.replace("couchbase-server-community", \
479                                          "ce/couchbase-server-community")
480        if is_amazon:
481            """
482                for centos only
483                         https://s3.amazonaws.com/packages.couchbase.com/releases/
484                         4.0.0/couchbase-server-enterprise-4.0.0-centos6.x86_64.rpm """
485            build.url = "https://s3.amazonaws.com/packages.couchbase.com/releases/"\
486                        "{0}/{1}-{0}-centos6.{2}.{3}" \
487                        .format(build_version, product, os_architecture,
488                                                       deliverable_type)
489            build.url = build.url.replace("enterprise", "community")
490            build.name = build.name.replace("enterprise", "community")
491        if direct_build_url is not None and deliverable_type != "exe":
492            build.url = direct_build_url
493        return build
494
495    def sort_builds_by_version(self, builds):
496        membase_builds = list()
497        for build in builds:
498            if build.product == 'membase-server-enterprise':
499                membase_builds.append(build)
500
501        return sorted(membase_builds,
502                      key=lambda membase_build: membase_build.build_number, reverse=True)
503
504    def sort_builds_by_time(self, builds):
505        membase_builds = list()
506        for build in builds:
507            if build.product == 'membase-server-enterprise':
508                membase_builds.append(build)
509
510        return sorted(membase_builds,
511                      key=lambda membase_build: membase_build.time, reverse=True)
512
513
514    def get_latest_builds(self):
515        return self._get_and_parse_builds('http://builds.hq.northscale.net/latestbuilds')
516
517    def get_sustaining_latest_builds(self):
518        return self._get_and_parse_builds('http://builds.hq.northscale.net/latestbuilds/sustaining')
519
520    def get_all_builds(self, version=None, timeout=None, direct_build_url=None, deliverable_type=None, \
521                       architecture_type=None,edition_type=None, repo=None, toy="", \
522                       distribution_version=None, distribution_type=""):
523        try:
524            latestbuilds, latestchanges = \
525                self._get_and_parse_builds('http://builds.hq.northscale.net/latestbuilds', version=version, \
526                                           timeout=timeout, direct_build_url=direct_build_url, \
527                                           deliverable_type=deliverable_type, architecture_type=architecture_type, \
528                                           edition_type=edition_type,repo=repo, toy=toy, \
529                                           distribution_version=distribution_version, \
530                                           distribution_type=distribution_type)
531        except Exception as e:
532            latestbuilds, latestchanges = \
533                self._get_and_parse_builds('http://packages.northscale.com.s3.amazonaws.com/latestbuilds', \
534                                           version=version, timeout=timeout, direct_build_url=direct_build_url)
535
536        return latestbuilds, latestchanges
537
538
539    #baseurl = 'http://builds.hq.northscale.net/latestbuilds/'
540    def _get_and_parse_builds(self, build_page, version=None, timeout=None, direct_build_url=None, \
541                              deliverable_type=None, architecture_type=None, edition_type=None, \
542                              repo=None, toy="", distribution_version=None, distribution_type=""):
543        builds = []
544        changes = []
545        if direct_build_url is not None and direct_build_url != "":
546            query = BuildQuery()
547            build = query.create_build_info_from_direct_url(direct_build_url)
548            return build, changes
549        elif repo is not None and edition_type is not None and \
550             architecture_type is not None and deliverable_type is not None:
551            query = BuildQuery()
552            build = query.create_build_url(version, deliverable_type, architecture_type, \
553                                           edition_type, repo, toy, distribution_version, \
554                                           distribution_type)
555            return build, changes
556        else:
557            page = None
558            soup = None
559            index_url = '/index.html'
560            if version:
561                if version.find("-") != -1:
562                    index_url = "/index_" + version[:version.find("-")] + ".html"
563                else:
564                    index_url = "/index_" + version + ".html"
565            #try this ten times
566            for _ in range(0, 10):
567                try:
568                    self.log.info("Try collecting build information from url: %s" % (build_page + index_url))
569                    if timeout:
570                        socket.setdefaulttimeout(timeout)
571                    page = urllib2.urlopen(build_page + index_url)
572                    soup = BeautifulSoup.BeautifulSoup(page)
573                    break
574                except:
575                    time.sleep(5)
576            if not page:
577                raise Exception('unable to connect to %s' % (build_page + index_url))
578            query = BuildQuery()
579            for incident in soup('li'):
580                contents = incident.contents
581                build_id = ''
582                build_description = ''
583                for content in contents:
584                    if BeautifulSoup.isString(content):
585                        build_description = content.string
586                    elif content.name == 'a':
587                        build_id = content.string.string
588                try:
589                    if build_id.lower().startswith('changes'):
590                        change = query.create_change_info(build_id, build_description)
591                        change.url = '%s/%s' % (build_page, build_id)
592                        changes.append(change)
593                    else:
594                        build = query.create_build_info(build_id, build_description)
595                        build.url = '%s/%s' % (build_page, build_id)
596                        builds.append(build)
597                except Exception as e:
598                    print "ERROR in creating build/change info for: Build_id: %s , Build_Description: %s" % (build_id, build_description)
599                    print traceback.print_exc(file=sys.stderr)
600                    #raise e : Skipping parsing for this build information,
601                    #Eventually, It will fail with build not found error at install.py:240
602            for build in builds:
603                for change in changes:
604                    if change.build_number == build.product_version:
605                        build.change = change
606                        """ print 'change : ', change.url,change.build_number """
607                        break
608            return builds, changes
609
610    def create_build_info_from_direct_url(self, direct_build_url):
611        if direct_build_url is not None and direct_build_url != "":
612            build = MembaseBuild()
613            build.url = direct_build_url
614            build.toy = ""
615            build_info = direct_build_url.split("/")
616            build_info = build_info[len(build_info)-1]
617            """ windows build name: couchbase_server-enterprise-windows-amd64-3.0.0-892.exe
618                                    couchbase-server-enterprise_3.5.0-952-windows_amd64.exe """
619            build.name = build_info
620            deliverable_type = ["exe", "msi", "rpm", "deb", "zip"]
621            if build_info[-3:] in deliverable_type:
622                build.deliverable_type = build_info[-3:]
623                build_info = build_info[:-4]
624            else:
625                raise Exception('Check your url. Deliverable type %s does not support yet' \
626                                 % (direct_build_url[-3:]))
627            """ build name at this location couchbase-server-enterprise_x86_64_3.0.0-797-rel
628                windows build name: couchbase_server-enterprise-windows-amd64-3.0.0-892 """
629
630            """ Remove the code below when cb name is standardlized (MB-11372) """
631            if "windows" in direct_build_url and build.deliverable_type == "exe" \
632                and build_info not in SHERLOCK_VERSION:
633                build_info = build_info.replace("-windows-amd64-","_x86_64_")
634                build_info = build_info.replace("couchbase_server","couchbase-server")
635            """ End remove here """
636
637            """ sherlock build name
638                centos 6: couchbase-server-enterprise-3.5.0-71-centos6.x86_64
639                debian7:  couchbase-server-enterprise_3.5.0-10-debian7_amd64.deb
640                debian8:  couchbase-server-enterprise_4.5.0-1194-debian8_amd64.deb
641                ubuntu 12.04:
642                    couchbase-server-enterprise_3.5.0-723-ubuntu12.04_amd64.deb
643                mac:
644                    couchbase-server-enterprise_3.5.0-1120-macos_x86_64.zip
645                windows:
646                    couchbase_server-enterprise-windows-amd64-3.5.0-926.exe
647                    couchbase-server-enterprise_3.5.0-952-windows_amd64.exe"""
648
649            if any( x + "-" in build_info for x in COUCHBASE_FROM_VERSION_3):
650                deb_words = ["debian7", "debian8", "ubuntu12.04", "ubuntu14.04",
651                             "ubuntu16.04", "ubuntu18.04", "windows", "macos"]
652                if "centos" not in build_info and "suse" not in build_info:
653                    tmp_str = build_info.split("_")
654                    product_version = tmp_str[1].split("-")
655                    product_version = "-".join([i for i in product_version \
656                                                 if i not in deb_words])
657                else:
658                    product_version = build_info.split("-")
659                    product_version = product_version[3] + "-" + product_version[4]
660                if product_version[:5] in testconstants.COUCHBASE_VERSIONS:
661                    build.product_version = product_version
662                    if "centos" not in build_info and "suse" not in build_info:
663                        build_info = build_info.replace("_" + product_version,"")
664                    else:
665                        build_info = build_info.replace("-" + product_version,"")
666                if "x86_64" in build_info:
667                    build.architecture_type = "x86_64"
668                    if "centos" in build_info or "suse" in build_info:
669                        build_info = build_info.replace(".x86_64", "")
670                    elif "macos" in build_info:
671                        build_info = build_info.replace("_x86_64", "")
672                elif "x86" in build_info:
673                    build.architecture_type = "x86"
674                    build_info = build_info.replace(".x86", "")
675                elif "_amd64" in build_info:
676                    build.architecture_type = "x86_64"
677                    build_info = build_info.replace("_amd64", "")
678                elif "-amd64" in build_info:
679                    build.architecture_type = "x86_64"
680                    build_info = build_info.replace("-amd64", "")
681                del_words = ["centos6", "debian7", "debian8", "debian9",
682                             "ubuntu12.04", "ubuntu14.04", "ubuntu16.04", "ubuntu18.04",
683                             "windows", "macos", "centos7", "suse11", "suse12", "amzn2"]
684                if build_info.startswith("couchbase-server"):
685                    build.product = build_info.split("-")
686                    build.product = "-".join([i for i in build.product \
687                                                 if i not in del_words])
688                return build
689            product_version = build_info.split("_")
690            product_version = product_version[len(product_version)-1]
691            if product_version[:5] in testconstants.COUCHBASE_VERSIONS:
692                build.product_version = product_version
693                build_info = build_info.replace("_" + product_version,"")
694            else:
695                raise Exception("Check your url. Couchbase server does not have "
696                                       "version %s yet " % (product_version[:5]))
697
698            if "x86_64" in build_info:
699                build.architecture_type = "x86_64"
700                build_info = build_info.replace("_x86_64", "")
701            elif "x86" in build_info:
702                build.architecture_type = "x86"
703                build_info = build_info.replace("_x86", "")
704
705            if build_info.startswith("couchbase-server"):
706                build.product = build_info
707            else:
708                self.fail("unknown server name")
709            return build
710
711    def create_build_url(self, version, deliverable_type, architecture_type,
712                              edition_type, repo, toy, distribution_version,
713                                                         distribution_type):
714        build = MembaseBuild()
715        """
716        version: 3.0.0-xx or 3.0.0-xx-rel
717        deliverable_type: deb
718        distribution_version: ubuntu12 or debian7
719        architecture_type: x86_64
720        edition_type: couchbase-server-enterprise or couchbase-server-community
721        repo: http://builds.hq.northscale.net/latestbuilds/
722        sherlock repo: http://latestbuilds.hq.couchbase.com/couchbase-server/sherlock
723        sherlock build name with extra build number:
724               /684/couchbase-server-enterprise-3.5.0-684-centos6.x86_64.rpm
725               /1454/couchbase-server-enterprise-4.0.0-1454-centos6.x86_64.rpm
726               /1796/couchbase-server-enterprise-4.0.0-1796-oel6.x86_64.rpm
727               /723/couchbase-server-enterprise_3.5.0-723-ubuntu12.04_amd64.deb
728               /723/couchbase-server-enterprise_3.5.0-732-debian7_amd64.deb
729               /1194/couchbase-server-enterprise_4.5.0-1194-debian8_amd64.deb
730               /1120/couchbase-server-enterprise_3.5.0-1120-macos_x86_64.zip
731        toy=Ce
732        build.name = couchbase-server-enterprise_x86_64_3.0.0-xx-rel.deb
733        build.url = http://builds.hq.northscale.net/latestbuilds/
734                              couchbase-server-enterprise_x86_64_3.0.0-xx-rel.deb
735        For toy build: name  =
736            couchbase-server-community_cent58-3.0.0-toy-toyName-x86_64_3.0.0-xx-toy.rpm
737            http://latestbuilds.hq.couchbase.com/couchbase-server/
738                toy-wied/14/couchbase-server-enterprise-1004.0.0-14-centos6.x86_64.rpm
739                toy-nimish/16/couchbase-server-enterprise_1003.5.0-16-windows_amd64.exe
740        For windows build diff - and _ compare to unix build
741                       name = couchbase_server-enterprise-windows-amd64-3.0.0-998.exe
742                              couchbase_server-enterprise-windows-amd64-3.0.2-1603.exe
743                              couchbase_server-enterprise-windows-amd64-3.0.3-1716.exe
744                              couchbase-server-enterprise_3.5.0-952-windows_amd64.exe
745                              couchbase-server-enterprise_3.5.0-1390-windows_x86.exe
746            From build 5.0.0-2924, we don't make any exe build.
747            It will be all in msi
748        """
749        build.toy = "toy-" + toy
750        build.deliverable_type = deliverable_type
751        build.architecture_type = architecture_type
752        build.distribution_version = distribution_version
753        build.distribution_type = distribution_type
754
755        os_name = ""
756        setup = ""
757        build_number = ""
758
759        unix_deliverable_type = ["deb", "rpm", "zip"]
760        if deliverable_type in unix_deliverable_type:
761            if toy == "" and version[:5] not in COUCHBASE_VERSION_2 and \
762                                   version[:5] not in COUCHBASE_VERSION_3:
763                if "rel" not in version and toy == "":
764                    build.product_version = version
765                elif "-rel" in version:
766                    build.product_version = version.replace("-rel", "")
767            elif toy != "":
768                build.product_version = version
769            else:
770                if "rel" not in version and toy == "":
771                    build.product_version = version + "-rel"
772                else:
773                    build.product_version = version
774        if deliverable_type in ["exe", "msi"]:
775            if version[:5] in COUCHBASE_VERSION_2:
776                setup = "setup."
777            else:
778                os_name= "windows-"
779            if "rel" in version and version[:5] not in COUCHBASE_VERSION_2:
780                build.product_version = version.replace("-rel", "")
781            elif "rel" not in version and version[:5] in COUCHBASE_VERSION_2:
782                build.product_version = version + "-rel"
783            else:
784                build.product_version = version
785            if "couchbase-server" in edition_type and version[:5] in WIN_CB_VERSION_3:
786                edition_type = edition_type.replace("couchbase-", "couchbase_")
787            if version[:5] not in COUCHBASE_VERSION_2:
788                if "x86_64" in architecture_type:
789                    build.architecture_type = "amd64"
790                elif "x86" in architecture_type:
791                    build.architecture_type = "x86"
792            """
793                    In spock from build 2924 and later release, we only support
794                    msi installation method on windows
795            """
796            if "-" in version and version.split("-")[0] in COUCHBASE_FROM_SPOCK:
797                deliverable_type = "msi"
798
799        if "deb" in deliverable_type and "centos6" in edition_type:
800            edition_type = edition_type.replace("centos6", "ubuntu_1204")
801        if "debian" in distribution_version:
802            os_name = "debian7_"
803        joint_char = "_"
804        version_join_char = "_"
805        if toy is not "":
806            joint_char = "-"
807        if "exe" in deliverable_type and version[:5] not in COUCHBASE_VERSION_2:
808            joint_char = "-"
809            version_join_char = "-"
810        if toy == "" and version[:5] not in COUCHBASE_VERSION_2 and \
811                                   version[:5] not in COUCHBASE_VERSION_3:
812            """ format for sherlock build name
813            /684/couchbase-server-enterprise-3.5.0-684-centos6.x86_64.rpm
814            /1154/couchbase-server-enterprise-3.5.0-1154-centos7.x86_64.rpm
815            /1454/couchbase-server-enterprise-4.0.0-1454-centos6.x86_64.rpm
816            /1796/couchbase-server-enterprise-4.0.0-1796-oel6.x86_64.rpm
817            /723/couchbase-server-enterprise_3.5.0-723-ubuntu12.04_amd64.deb
818            /723/couchbase-server-enterprise_3.5.0-732-debian7_amd64.deb
819            /795/couchbase_server-enterprise-windows-amd64-3.5.0-795.exe
820            /952/couchbase-server-enterprise_3.5.0-952-windows_amd64.exe
821            /1390/couchbase-server-enterprise_3.5.0-1390-windows_x86.exe
822            /1120/couchbase-server-enterprise_3.5.0-1120-macos_x86_64.zip"""
823            build_number = build.product_version.replace(version[:6],"")
824            """ distribution version:    centos linux release 7.0.1406 (core)
825                distribution version:    centos release 6.5 (final)  """
826            rpm_version = "centos6"
827
828            if "centos" in distribution_version or "red hat" in distribution_version:
829                if "centos 7" in distribution_version:
830                    rpm_version = "centos7"
831                elif "red hat enterprise linux server release 6" in distribution_version:
832                    rpm_version = "centos6"
833                elif "red hat enterprise linux server release 7" in distribution_version:
834                    rpm_version = "centos7"
835                elif "red hat enterprise linux release 8.0" in distribution_version:
836                    rpm_version = "rhel8"
837                build.name = edition_type + "-" + build.product_version + \
838                   "-" + rpm_version + "." + build.architecture_type + \
839                   "." + build.deliverable_type
840            elif "suse" in distribution_version:
841                if "suse 12" in distribution_version:
842                    if version[:5] in COUCHBASE_FROM_SPOCK:
843                        suse_version="suse12"
844                        build.distribution_version = "suse12"
845                    else:
846                        self.fail("suse 12 does not support on this version %s "
847                                                                  % version[:5])
848                else:
849                    suse_version="suse11"
850                    build.distribution_version = "suse11"
851                build.name = edition_type + "-" + build.product_version + \
852                   "-" + suse_version + "." + build.architecture_type + \
853                   "." + build.deliverable_type
854            elif "oracle linux" in distribution_version:
855                build.distribution_version = "oracle linux"
856                os_name = "oel6"
857                build.name = edition_type + "-" + build.product_version + \
858                   "-" + os_name + "." + build.architecture_type + \
859                   "." + build.deliverable_type
860            elif "amazon linux release 2" in distribution_version:
861                if version[:5] in COUCHBASE_FROM_MAD_HATTER or \
862                                version[:5] in COUCHBASE_FROM_601:
863                    build.distribution_version = "amazon linux 2"
864                    os_name = "amzn2"
865                    build.name = edition_type + "-" + build.product_version + \
866                                 "-" + os_name + "." + build.architecture_type + \
867                                 "." + build.deliverable_type
868                else:
869                    self.fail("Amazon Linux 2 doesn't support version %s "
870                              % version[:5])
871            else:
872                os_name = ""
873                joint_char = "-"
874
875                """ sherlock build in unix only support 64-bit """
876                build.architecture_type = "amd64"
877                if  "ubuntu 12.04" in distribution_version:
878                    os_name = "ubuntu12.04"
879                elif "ubuntu 14.04" in distribution_version:
880                    os_name = "ubuntu14.04"
881                elif "ubuntu 16.04" in distribution_version:
882                    os_name = "ubuntu16.04"
883                elif "ubuntu 18.04" in distribution_version.lower():
884                    if version[:5] in COUCHBASE_FROM_MAD_HATTER or \
885                        version[:5] in COUCHBASE_FROM_601:
886                        os_name = "ubuntu18.04"
887                    else:
888                        self.fail("ubuntu 18.04 doesn't support version %s "
889                                                              % version[:5])
890                elif "debian gnu/linux 7" in distribution_version:
891                    build.distribution_version = "debian7"
892                    os_name = "debian7"
893                elif "debian gnu/linux 8" in distribution_version:
894                    build.distribution_version = "debian8"
895                    os_name = "debian8"
896                elif "debian gnu/linux 9" in distribution_version:
897                    build.distribution_version = "debian9"
898                    os_name = "debian9"
899                elif "windows" in distribution_version:
900                    os_name = "windows"
901                    if "x86_64" not in architecture_type:
902                        build.architecture_type = "x86"
903                elif "mac" in distribution_type:
904                    os_name = "macos"
905                    build.architecture_type = "x86_64"
906                build.name = edition_type + "_" + build.product_version + \
907                   joint_char + os_name + "_" +  build.architecture_type + \
908                   "." + build.deliverable_type
909            build.url = repo + build_number + "/" + build.name
910        elif toy is not "":
911            centos_version = "centos6"
912            build_info = version.split("-")
913            build_number = build_info[1]
914            if "centos" in distribution_version:
915                build.name = edition_type + "-" + build.product_version + \
916                   "-" + centos_version + "." + build.architecture_type + \
917                   "." + build.deliverable_type
918            build.url = repo + build.toy + "/" +build_number \
919                        + "/" + build.name
920        elif version[:3] == "3.1":
921            os_name = ""
922            if "suse" in distribution_version:
923                build.distribution_version = "suse11"
924                os_name = "suse11_"
925            elif "centos release 6" in distribution_version:
926                build.distribution_version = "centos6"
927                os_name = "centos6_"
928            elif  "ubuntu 12.04" in distribution_version:
929                os_name = "ubuntu_1204_"
930            elif "debian gnu/linux 7" in distribution_version:
931                build.distribution_version = "debian7"
932                os_name = "debian7_"
933            elif "debian gnu/linux 8" in distribution_version:
934                build.distribution_version = "debian8"
935                os_name = "debian8_"
936            elif "windows" in distribution_version:
937                os_name = "windows-"
938                if "x86_64" not in architecture_type:
939                    build.architecture_type = "x86"
940            elif "mac" in distribution_type:
941                build.architecture_type = "x86_64"
942            build.name = edition_type + joint_char + os_name + \
943                build.architecture_type +  version_join_char + \
944                build.product_version + "." + setup + build.deliverable_type
945            build.url = repo + build.name
946        else:
947            build.name = edition_type + joint_char + os_name + \
948                build.architecture_type +  version_join_char + \
949                build.product_version + "." + setup + build.deliverable_type
950            build.url = repo + build.name
951
952
953        """ reset build.architecture back to x86_64 in windows """
954        build.architecture_type = architecture_type
955        return build
956
957    def create_build_info(self, build_id, build_decription):
958        build = MembaseBuild()
959        build.deliverable_type = self._product_deliverable_type(build_id)
960        build.time = self._product_time(build_decription)
961        build.size = self._product_size(build_decription)
962        build.product_version = self._product_version(build_id)
963        build.architecture_type = self._product_arch_type(build_id)
964        build.product = self._product_name(build_id)
965        build.name = build_id
966        build.build_number = self._build_number(build)
967        build.toy = self._product_toy(build_id)
968        return build
969
970    def create_change_info(self, build_id, build_decription):
971        change = MembaseChange()
972        change.name = build_id.strip()
973        change.build_number = self._change_build_number(build_id)
974        change.time = self._change_time(build_decription)
975        return change
976
977
978    def _product_name(self, build_id):
979        list = build_id.split('_')
980        if "centos6" in build_id:
981            # return couchbase-server-ent/com_centos6
982            return "_".join(list[:2])
983        elif "ubuntu_1204" in build_id:
984            # return couchbase-server-ent/com_ubuntu_1204
985            return "_".join(list[:3])
986        # this should be done w/ more generic rule for toy-split
987        elif "cent54" in build_id:
988            list = build_id.split("-toy")
989            return list[0]
990        else:
991            return list[0]
992        #the first one is the product
993
994    def _product_arch_type(self, build_id):
995        list = build_id.split('_')
996        if '64' in build_id.split('_') or build_id.find('x86_64') != -1:
997            return 'x86_64'
998        elif 'x86' in build_id.split('_'):
999            return 'x86'
1000        return ''
1001
1002
1003    def _product_toy(self, build_id):
1004        r = re.search("[^_]+-toy-([\w-]*)-x86", build_id)
1005        if r:
1006            return r.group(1)
1007        return ''
1008
1009    def _change_time(self, build_description):
1010        list = build_description.split('/')
1011        timestamp = list[1].strip()
1012        timestamp = timestamp[:timestamp.index(')')]
1013        return datetime.strptime(timestamp, '%a %b %d %H:%M:%S %Y')
1014
1015    def _change_build_number(self, build_id):
1016        list = build_id.split('_')
1017        #get list[1] . get rid of .txt
1018        build_number = list[1].strip()
1019        if re.search('.txt', build_number):
1020            build_number = build_number[:build_number.index('.txt')]
1021            return build_number
1022
1023    def _build_number(self, build):
1024        #get the first - and then the first - after that
1025        first_dash = build.product_version.find('-')
1026        if first_dash != -1:
1027            second_dash = build.product_version.find('-', first_dash + 1)
1028            if second_dash != -1:
1029                try:
1030                    return int(build.product_version[first_dash + 1:second_dash])
1031                except Exception:
1032                    return -1
1033        return -1
1034
1035    def _product_version(self, build_id):
1036        list = build_id.split('_')
1037        version_item = ''
1038        for item in list:
1039            if re.match(r'[0-2].[0-9].[0-9]-[0-9]+-rel', item):
1040                version_item = item
1041                if list[-1].endswith('xml'):
1042                    break
1043                return version_item
1044        if version_item == '':
1045            for item in list:
1046                if item.endswith('.setup.exe') or item.endswith('rpm') or\
1047                   item.endswith('deb') or item.endswith('tar.gz') or item.endswith('zip'):
1048                    version_item = item
1049                    break
1050        if version_item != '':
1051            if version_item.endswith('.setup.exe'):
1052                return version_item[:version_item.index('.setup.exe')]
1053            elif version_item.endswith('.tar.gz'):
1054                return version_item[:version_item.index('.tar.gz')]
1055            elif version_item.endswith('.deb'):
1056                return version_item[:version_item.index('.deb')]
1057            elif version_item.endswith('.rpm'):
1058                return version_item[:version_item.index('.rpm')]
1059            elif version_item.endswith('.zip'):
1060                return version_item[:version_item.index('.zip')]
1061        return ''
1062
1063    def _product_deliverable_type(self, build_id=''):
1064        list = build_id.split('_')
1065        version_item = ''
1066        for item in list:
1067            if item.endswith('.setup.exe') or item.endswith('rpm') or\
1068               item.endswith('deb') or item.endswith('tar.gz') or item.endswith('zip'):
1069                version_item = item
1070                break
1071        if version_item != '':
1072            if version_item.endswith('.setup.exe'):
1073                return 'exe'
1074            elif version_item.endswith('.tar.gz'):
1075                return 'tar.gz'
1076            elif version_item.endswith('.deb'):
1077                return 'deb'
1078            elif version_item.endswith('.rpm'):
1079                return 'rpm'
1080            elif version_item.endswith('.zip'):
1081                return 'zip'
1082        return ''
1083
1084    def _product_time(self, build_description):
1085        list = build_description.split('/')
1086        timestamp = list[1].strip()
1087        timestamp = timestamp[:timestamp.index(')')]
1088        return datetime.strptime(timestamp, '%a %b %d %H:%M:%S %Y')
1089
1090    def _product_size(self, build_description):
1091        list = build_description.split('/')
1092        filesize = list[0]
1093        filesize = filesize[filesize.index('(') + 1:]
1094        return filesize.strip()
1095
1096#q = BuildQuery()
1097#builds, changes = q.get_latest_builds()
1098#for build in builds:
1099#    print build.product,' ',build.time ,' ',build.deliverable_type,' ',build.product_version ,'',build.size,'',build.architecture_type
1100#    if build.change:
1101#        change = build.change
1102#        print change.name,change.build_number,change.time,change.url
1103
1104#for change in changes:
1105#    print change.name,change.build_number,change.time
1106
1107#builds = q.get_membase_latest_builds()
1108#for build in builds:
1109#    print build.product,' ',build.time ,' ',build.deliverable_type,' ',build.product_version ,'',build.size,'',build.architecture_type
1110
1111