1from seriesly import Seriesly
2from optparse import OptionParser
3import sys
4sys.path.append(".")
5import testcfg as cfg
6import datetime
7import pandas as pd
8import pygal
9import os
10import shutil
11
12conn = Seriesly(cfg.SERIESLY_IP, 3133)
13
14"""
15" retrieve timeseries data from seriesly
16"""
17def getDBData(db):
18  db=conn[db]
19  data = db.get_all()
20  return (data, None)[len(data) == 0]
21
22"""
23" sort data by its timestamp keys
24"""
25def sortDBData(data):
26
27  sorted_data = []
28  keys = []
29  if(data):
30    keys = sorted(data.iterkeys())
31
32  for ts in keys:
33    sorted_data.append(data[ts])
34
35  return keys, sorted_data
36
37def getSortedDBData(db):
38  return sortDBData(getDBData(db))
39
40"""
41" create datetime index array by converting ts strings
42"""
43def indexFromKeys(keys):
44  return [datetime.datetime.strptime(ts[:ts.index('.')],"%Y-%m-%dT%H:%M:%S") for ts in keys]
45
46"""
47" make a timeseries dataframe
48"""
49def _createDataframe(index, data):
50
51  df = None
52
53  try:
54
55    if(data):
56      df = pd.DataFrame(data)
57      df.index = index
58
59  except ValueError as ex:
60    print "unable to create dataframe: has incorrect format"
61    raise Exception(ex)
62
63  return df
64
65"""
66" get data from seriesly and convert to a 2d timeseries dataframe rows=ts, columns=stats
67"""
68def createDataframe(db):
69  df = None
70  data = getDBData(db)
71
72  if data:
73    index, data = getSortedDBData(db)
74    df = _createDataframe(index, data)
75  else:
76    print "WARNING: stat db %s is empty!" % db
77
78  return df
79
80
81"""
82" plot stats per-phase and save to html
83"""
84def plot_phases(ns_dataframe, test_id = "simple"):
85
86  phase_dataframe = None
87  columns = ns_dataframe.columns
88  event_idx, _ = getSortedDBData('event')
89
90  # plot each phase
91  for i in xrange(len(event_idx)):
92    if i == 0:
93      phase_dataframe = ns_dataframe[ns_dataframe.index < event_idx[i+1]]
94    elif i == len(event_idx) - 1:
95      phase_dataframe = ns_dataframe[ns_dataframe.index > event_idx[i]]
96    else:
97      phase_dataframe = ns_dataframe[ (ns_dataframe.index < event_idx[i+1]) &\
98        (ns_dataframe.index > event_idx[i])]
99
100    ph_html = "%s/%s_phase%s.html" % (test_id, test_id, i)
101    ph_cvs = "%s/%s_phase%s.cvs" % (test_id, test_id, i)
102    f = open(ph_html,"w")
103    print ph_html
104
105
106    for column in columns:
107
108      # filter out no-data columns
109      if all([v == 0 for v in phase_dataframe[column].values]):
110        continue
111
112      # plot phase data and filter 0's values
113      chart=pygal.Line(stroke=False, print_values=False, human_readable=True)
114      chart.add(column, filter(lambda x: x > 0, (phase_dataframe[column].values)))
115
116      # write out chart html
117      res = chart.render_response()
118      f.write(res.data)
119
120    # write out phase data to cvs
121    phase_dataframe.to_csv(ph_cvs)
122
123
124    f.close()
125
126def get_testid():
127  test_id = None
128  evdata = getDBData('event')
129  if not evdata:
130    return
131
132  evkeys = evdata.keys()
133  if(len(evkeys) > 0):
134    onephase = evdata[evkeys[0]].values()[0]
135    if 'name' in onephase:
136      test_id = str(onephase['name'])
137
138  return test_id
139
140def mkdir(path):
141  if not os.path.exists(path):
142      os.makedirs(path)
143  else:
144      shutil.rmtree(path)
145      os.makedirs(path)
146
147def prepare_env():
148  test_id = get_testid()
149  if test_id is None:
150    raise Exception("testid missing from event-db")
151
152  path = "%s" % test_id
153  mkdir(path)
154  return test_id
155
156
157def parse_args():
158    """Parse CLI arguments"""
159    usage = "usage: %prog bucket1,bucket2\n\n" + \
160            "Example: python tools/plotter.py default,saslbucket"
161
162    parser = OptionParser(usage)
163    options, args = parser.parse_args()
164
165    if len(args) < 1 :
166        parser.print_help()
167        sys.exit()
168
169    return options, args
170
171
172def main():
173  options, args = parse_args()
174  buckets = args[0].split(",")
175
176  test_id = prepare_env()
177
178  for bucket in buckets:
179    ns_dataframe = createDataframe('ns_serverdefault%s' % bucket)
180
181    if ns_dataframe:
182      ns_dataframe.to_excel('%s/%s.xlsx' % (test_id, test_id), sheet_name=test_id)
183      plot_phases(ns_dataframe, test_id)
184
185if __name__ == "__main__":
186    main()
187