xref: /5.5.2/couchdb/test/ci/dotest (revision abb37c58)
1#!/bin/bash
2
3source ~/.cienv
4
5echo '</pre><h3>Testing</h3><pre>'
6
7if [ "$WORKSPACE" = "" ]; then
8  echo 'WORKSPACE not set'
9  exit 2
10fi
11
12if [ "$MODE" = "" ]; then
13  echo 'MODE not set'
14  exit 2
15fi
16
17if [ "$TS" = "" ]; then
18  TS="adhoc"
19fi
20
21start_cluster() {
22  echo "Starting server"
23  for a in {1..3}; do
24    echo "Starting server: attempt $a"
25    stop_cluster
26    cd $WORKSPACE/ns_server
27    ./cluster_run -n4 1>$WORKSPACE/run.log 2>&1 &
28    disown
29    for i in {1..120}; do
30      grep -qs 'Couchbase Server has started' logs/n_0/info.log && \
31        grep -qs 'Couchbase Server has started' logs/n_1/info.log && \
32          wget -qO- http://localhost:9000/ &>/dev/null &&
33            wget -qO- http://localhost:9001/ &>/dev/null &&
34              ok_run=1 && break
35      sleep 3
36    done
37    sleep 30
38    test "$ok_run" && break
39  done
40  test "$ok_run" || error_exit "Server startup failed even after 3 tries"
41}
42
43stop_cluster() {
44  pkill -f cluster_
45  pkill -f $WORKSPACE/install/bin
46  pkill -f testrunner
47  pkill -f $WORKSPACE/install
48  pkill -f testrunner
49  killall beam.smp epmd memcached eventing-producer eventing-consumer python memcached.json 1>/dev/null 2>&1
50  sleep 10
51  killall -9 beam.smp epmd memcached eventing-producer eventing-consumer python memcached.json 1>/dev/null 2>&1
52  cd $WORKSPACE/ns_server
53  make dataclean 1>/dev/null 2>&1
54  sudo rm -f /tmp/core*
55  sleep 10
56}
57
58collect_logs() {
59  if [ "$phase" = "" ]; then phase=unknown; fi
60  tar -C $WORKSPACE --transform "s/^/logs-$phase-/" -uf $WORKSPACE/logs.tar ns_server/logs 1>/dev/null 2>&1
61  tar -C $WORKSPACE --transform "s/^/logs-$phase-/" -uf $WORKSPACE/logs.tar testrunner/logs 1>/dev/null 2>&1
62  tar -C $WORKSPACE --transform "s/^/logs-$phase-/" -uf $WORKSPACE/logs.tar test.log 1>/dev/null 2>&1
63  tar -C $WORKSPACE --transform "s/^/logs-$phase-/" -uf $WORKSPACE/logs.tar run.log 1>/dev/null 2>&1
64}
65
66error_email() {
67  if [ ! -f ~/.cigreen ]; then return; fi
68  rm ~/.cigreen
69
70  if test "`find ~ -maxdepth 1 -name .cinotify -mmin -1440`"; then return; fi
71  touch ~/.cinotify
72
73  if [ "$WATCHERS" = "" ]; then return; fi
74  cat /var/www/views-current.html | \
75     sendemail -q \
76       -s west.smtp.mx.exch029.serverdata.net:25 \
77       -f 'Eventing CI <community_admin@couchbase.com>' \
78       -t $WATCHERS \
79       -u 'Eventing CI just broke'
80}
81
82error_exit() {
83  echo "</pre><h4>Testing Failed: $1</h4><pre>"
84  echo "$2"
85  collect_logs
86  stop_cluster
87  error_email
88  exit 2
89}
90
91# Setup workspace
92echo "Testing mode: $MODE"
93stop_cluster
94rm -f $WORKSPACE/logs.tar.* $WORKSPACE/test.log $WORKSPACE/run.log /tmp/core-*
95> $WORKSPACE/test.log
96
97# Test the base
98echo "</pre><h4>Simple Test</h4><pre>"
99export phase=sanity
100cd $WORKSPACE/testrunner
101(timeout 1h make simple-test 2>&1 | ts) | tee -a $WORKSPACE/test.log | egrep 'pass|fail|summary|ok$'
102
103# Functional tests
104export phase=functional
105start_cluster
106ulimit -c 2097152
107sudo bash -c "echo /tmp/core-%e.$TS.%p > /proc/sys/kernel/core_pattern"
108perl -pi -e 's/num_files, 10/num_files, 50/' $WORKSPACE/install/etc/couchbase/static_config
109(timeout 1h echo "No functional tests defined" 2>&1 | ts) | tee -a $WORKSPACE/test.log | egrep 'pass|fail|summary|ok$'
110collect_logs
111stop_cluster
112
113# Integration tests
114echo "</pre><h4>Integration tests</h4><pre>"
115export phase=integration
116cd $WORKSPACE/testrunner
117(timeout 2h "No integration tests defined" 2>&1 | ts) | tee -a $WORKSPACE/test.log
118collect_logs
119
120# Verify results
121> /tmp/fail.log
122cd $WORKSPACE/couchdb
123grep 'make:\s\*\*\*\s\[test-view.*\]\s' $WORKSPACE/test.log >> /tmp/fail.log
124grep '\sfail\s*$' $WORKSPACE/test.log >> /tmp/fail.log
125grep 'FAIL\s*github.com/couchbase' $WORKSPACE/test.log >> /tmp/fail.log
126grep -- '--- FAIL:' $WORKSPACE/test.log >> /tmp/fail.log
127grep '^\s*2i' $WORKSPACE/test.log | grep 'fail\s*$' >> /tmp/fail.log
128skiplist="`cat test/ci/skip.txt`"
129for tst in $skiplist; do
130  echo "$tst" | grep -qs '^\s*#' && continue
131  echo "$tst" | grep -qs '^\s*$' && continue
132  grep -v "$tst" /tmp/fail.log > /tmp/fail-out.log
133  mv /tmp/fail-out.log /tmp/fail.log
134done
135faillog="`cat /tmp/fail.log`"
136if [ "$faillog" != "" ]; then error_exit "Required test failed" "$faillog"; fi
137echo "</pre><h4>Testing Succeeded</h4><pre>"
138if [ "$skiplist" != "" ]; then
139  echo "But we ignored these tests:"
140  echo "$skiplist"
141fi
142echo
143
144# Note versions
145cd $WORKSPACE/goproj/src/github.com/couchbase/eventing
146git rev-parse HEAD > ~/eventing.good
147cd $WORKSPACE/goproj/src/github.com/couchbase/indexing
148git rev-parse HEAD > ~/indexing.good
149cd $WORKSPACE/goproj/src/github.com/couchbase/query
150git rev-parse HEAD > ~/query.good
151cd $WORKSPACE/goproj/src/github.com/couchbase/gometa
152git rev-parse HEAD > ~/gometa.good
153cd $WORKSPACE/forestdb
154git rev-parse HEAD > ~/forestdb.good
155cd $WORKSPACE/ns_server
156git rev-parse HEAD > ~/ns_server.good
157cd $WORKSPACE/couchdb
158git rev-parse HEAD > ~/couchdb.good
159cd $WORKSPACE/couchstore
160git rev-parse HEAD > ~/couchstore.good
161cd $WORKSPACE/goproj/src/github.com/couchbase/plasma
162git rev-parse HEAD > ~/plasma.good
163cd $WORKSPACE/goproj/src/github.com/couchbase/nitro
164git rev-parse HEAD > ~/nitro.good
165cd $WORKSPACE/testrunner
166git rev-parse HEAD > ~/testrunner.good
167cd $WORKSPACE/kv_engine
168git rev-parse HEAD > ~/kv_engine.good
169
170touch ~/.cigreen
171exit 0
172
173