1286eea9cSTommie McAfeefrom seriesly import Seriesly
2286eea9cSTommie McAfeefrom optparse import OptionParser
3286eea9cSTommie McAfeeimport sys
4286eea9cSTommie McAfeesys.path.append(".")
5286eea9cSTommie McAfeeimport testcfg as cfg
6286eea9cSTommie McAfeeimport datetime
7286eea9cSTommie McAfeeimport pandas as pd
8286eea9cSTommie McAfeeimport pygal
9286eea9cSTommie McAfeeimport os
10286eea9cSTommie McAfeeimport shutil
11286eea9cSTommie McAfee
12286eea9cSTommie McAfeeconn = Seriesly(cfg.SERIESLY_IP, 3133)
13286eea9cSTommie McAfee
14286eea9cSTommie McAfee"""
15286eea9cSTommie McAfee" retrieve timeseries data from seriesly
16286eea9cSTommie McAfee"""
17286eea9cSTommie McAfeedef getDBData(db):
18286eea9cSTommie McAfee  db=conn[db]
19286eea9cSTommie McAfee  data = db.get_all()
20286eea9cSTommie McAfee  return (data, None)[len(data) == 0]
21286eea9cSTommie McAfee
22286eea9cSTommie McAfee"""
23286eea9cSTommie McAfee" sort data by its timestamp keys
24286eea9cSTommie McAfee"""
25286eea9cSTommie McAfeedef sortDBData(data):
26286eea9cSTommie McAfee
27286eea9cSTommie McAfee  sorted_data = []
28286eea9cSTommie McAfee  keys = []
29286eea9cSTommie McAfee  if(data):
30286eea9cSTommie McAfee    keys = sorted(data.iterkeys())
31286eea9cSTommie McAfee
32286eea9cSTommie McAfee  for ts in keys:
33286eea9cSTommie McAfee    sorted_data.append(data[ts])
34286eea9cSTommie McAfee
35286eea9cSTommie McAfee  return keys, sorted_data
36286eea9cSTommie McAfee
37286eea9cSTommie McAfeedef getSortedDBData(db):
38286eea9cSTommie McAfee  return sortDBData(getDBData(db))
39286eea9cSTommie McAfee
40286eea9cSTommie McAfee"""
41286eea9cSTommie McAfee" create datetime index array by converting ts strings
42286eea9cSTommie McAfee"""
43286eea9cSTommie McAfeedef indexFromKeys(keys):
44286eea9cSTommie McAfee  return [datetime.datetime.strptime(ts[:ts.index('.')],"%Y-%m-%dT%H:%M:%S") for ts in keys]
45286eea9cSTommie McAfee
46286eea9cSTommie McAfee"""
47286eea9cSTommie McAfee" make a timeseries dataframe
48286eea9cSTommie McAfee"""
49286eea9cSTommie McAfeedef _createDataframe(index, data):
50286eea9cSTommie McAfee
51286eea9cSTommie McAfee  df = None
52286eea9cSTommie McAfee
53286eea9cSTommie McAfee  try:
54286eea9cSTommie McAfee
55286eea9cSTommie McAfee    if(data):
56286eea9cSTommie McAfee      df = pd.DataFrame(data)
57286eea9cSTommie McAfee      df.index = index
58286eea9cSTommie McAfee
59286eea9cSTommie McAfee  except ValueError as ex:
60286eea9cSTommie McAfee    print "unable to create dataframe: has incorrect format"
61286eea9cSTommie McAfee    raise Exception(ex)
62286eea9cSTommie McAfee
63286eea9cSTommie McAfee  return df
64286eea9cSTommie McAfee
65286eea9cSTommie McAfee"""
66286eea9cSTommie McAfee" get data from seriesly and convert to a 2d timeseries dataframe rows=ts, columns=stats
67286eea9cSTommie McAfee"""
68286eea9cSTommie McAfeedef createDataframe(db):
69286eea9cSTommie McAfee  df = None
70286eea9cSTommie McAfee  data = getDBData(db)
71286eea9cSTommie McAfee
72286eea9cSTommie McAfee  if data:
73286eea9cSTommie McAfee    index, data = getSortedDBData(db)
74286eea9cSTommie McAfee    df = _createDataframe(index, data)
75286eea9cSTommie McAfee  else:
76286eea9cSTommie McAfee    print "WARNING: stat db %s is empty!" % db
77286eea9cSTommie McAfee
78286eea9cSTommie McAfee  return df
79286eea9cSTommie McAfee
80286eea9cSTommie McAfee
81286eea9cSTommie McAfee"""
82286eea9cSTommie McAfee" plot stats per-phase and save to html
83286eea9cSTommie McAfee"""
84286eea9cSTommie McAfeedef plot_phases(ns_dataframe, test_id = "simple"):
85286eea9cSTommie McAfee
86286eea9cSTommie McAfee  phase_dataframe = None
87286eea9cSTommie McAfee  columns = ns_dataframe.columns
88286eea9cSTommie McAfee  event_idx, _ = getSortedDBData('event')
89286eea9cSTommie McAfee
90286eea9cSTommie McAfee  # plot each phase
91286eea9cSTommie McAfee  for i in xrange(len(event_idx)):
92286eea9cSTommie McAfee    if i == 0:
93286eea9cSTommie McAfee      phase_dataframe = ns_dataframe[ns_dataframe.index < event_idx[i+1]]
94286eea9cSTommie McAfee    elif i == len(event_idx) - 1:
95286eea9cSTommie McAfee      phase_dataframe = ns_dataframe[ns_dataframe.index > event_idx[i]]
96286eea9cSTommie McAfee    else:
97286eea9cSTommie McAfee      phase_dataframe = ns_dataframe[ (ns_dataframe.index < event_idx[i+1]) &\
98286eea9cSTommie McAfee        (ns_dataframe.index > event_idx[i])]
99286eea9cSTommie McAfee
100286eea9cSTommie McAfee    ph_html = "%s/%s_phase%s.html" % (test_id, test_id, i)
101286eea9cSTommie McAfee    ph_cvs = "%s/%s_phase%s.cvs" % (test_id, test_id, i)
102286eea9cSTommie McAfee    f = open(ph_html,"w")
103286eea9cSTommie McAfee    print ph_html
104286eea9cSTommie McAfee
105286eea9cSTommie McAfee
106286eea9cSTommie McAfee    for column in columns:
107286eea9cSTommie McAfee
108286eea9cSTommie McAfee      # filter out no-data columns
109286eea9cSTommie McAfee      if all([v == 0 for v in phase_dataframe[column].values]):
110286eea9cSTommie McAfee        continue
111286eea9cSTommie McAfee
112286eea9cSTommie McAfee      # plot phase data and filter 0's values
113286eea9cSTommie McAfee      chart=pygal.Line(stroke=False, print_values=False, human_readable=True)
114286eea9cSTommie McAfee      chart.add(column, filter(lambda x: x > 0, (phase_dataframe[column].values)))
115286eea9cSTommie McAfee
116286eea9cSTommie McAfee      # write out chart html
117286eea9cSTommie McAfee      res = chart.render_response()
118286eea9cSTommie McAfee      f.write(res.data)
119286eea9cSTommie McAfee
120286eea9cSTommie McAfee    # write out phase data to cvs
121286eea9cSTommie McAfee    phase_dataframe.to_csv(ph_cvs)
122286eea9cSTommie McAfee
123286eea9cSTommie McAfee
124286eea9cSTommie McAfee    f.close()
125286eea9cSTommie McAfee
126286eea9cSTommie McAfeedef get_testid():
127286eea9cSTommie McAfee  test_id = None
128286eea9cSTommie McAfee  evdata = getDBData('event')
129286eea9cSTommie McAfee  if not evdata:
130286eea9cSTommie McAfee    return
131286eea9cSTommie McAfee
132286eea9cSTommie McAfee  evkeys = evdata.keys()
133286eea9cSTommie McAfee  if(len(evkeys) > 0):
134286eea9cSTommie McAfee    onephase = evdata[evkeys[0]].values()[0]
135286eea9cSTommie McAfee    if 'name' in onephase:
136286eea9cSTommie McAfee      test_id = str(onephase['name'])
137286eea9cSTommie McAfee
138286eea9cSTommie McAfee  return test_id
139286eea9cSTommie McAfee
140286eea9cSTommie McAfeedef mkdir(path):
141286eea9cSTommie McAfee  if not os.path.exists(path):
142286eea9cSTommie McAfee      os.makedirs(path)
143286eea9cSTommie McAfee  else:
144286eea9cSTommie McAfee      shutil.rmtree(path)
145286eea9cSTommie McAfee      os.makedirs(path)
146286eea9cSTommie McAfee
147286eea9cSTommie McAfeedef prepare_env():
148286eea9cSTommie McAfee  test_id = get_testid()
149286eea9cSTommie McAfee  if test_id is None:
150286eea9cSTommie McAfee    raise Exception("testid missing from event-db")
151286eea9cSTommie McAfee
152286eea9cSTommie McAfee  path = "%s" % test_id
153286eea9cSTommie McAfee  mkdir(path)
154286eea9cSTommie McAfee  return test_id
155286eea9cSTommie McAfee
156286eea9cSTommie McAfee
157286eea9cSTommie McAfeedef parse_args():
158286eea9cSTommie McAfee    """Parse CLI arguments"""
159286eea9cSTommie McAfee    usage = "usage: %prog bucket1,bucket2\n\n" + \
160286eea9cSTommie McAfee            "Example: python tools/plotter.py default,saslbucket"
161286eea9cSTommie McAfee
162286eea9cSTommie McAfee    parser = OptionParser(usage)
163286eea9cSTommie McAfee    options, args = parser.parse_args()
164286eea9cSTommie McAfee
165286eea9cSTommie McAfee    if len(args) < 1 :
166286eea9cSTommie McAfee        parser.print_help()
167286eea9cSTommie McAfee        sys.exit()
168286eea9cSTommie McAfee
169286eea9cSTommie McAfee    return options, args
170286eea9cSTommie McAfee
171286eea9cSTommie McAfee
172286eea9cSTommie McAfeedef main():
173286eea9cSTommie McAfee  options, args = parse_args()
174286eea9cSTommie McAfee  buckets = args[0].split(",")
175286eea9cSTommie McAfee
176286eea9cSTommie McAfee  test_id = prepare_env()
177286eea9cSTommie McAfee
178286eea9cSTommie McAfee  for bucket in buckets:
179286eea9cSTommie McAfee    ns_dataframe = createDataframe('ns_serverdefault%s' % bucket)
180286eea9cSTommie McAfee
181286eea9cSTommie McAfee    if ns_dataframe:
182286eea9cSTommie McAfee      ns_dataframe.to_excel('%s/%s.xlsx' % (test_id, test_id), sheet_name=test_id)
183286eea9cSTommie McAfee      plot_phases(ns_dataframe, test_id)
184286eea9cSTommie McAfee
185286eea9cSTommie McAfeeif __name__ == "__main__":
186286eea9cSTommie McAfee    main()