blob: b26eb56ae00ea71ed68f6d8bd0d037fd210069cb [file] [log] [blame]
bensong@google.com0d9e6de2013-10-15 00:08:14 +00001'''
2Created on May 16, 2011
3
4@author: bungeman
5'''
6import bench_util
7import getopt
8import httplib
9import itertools
10import json
11import os
12import re
13import sys
14import urllib
15import urllib2
16import xml.sax.saxutils
17
18# Maximum expected number of characters we expect in an svn revision.
19MAX_SVN_REV_LENGTH = 5
20
21def usage():
22 """Prints simple usage information."""
23
24 print '-a <representation_alg> bench representation algorithm to use. '
25 print ' Defaults to "25th". See bench_util.py for details.'
26 print '-b <builder> name of the builder whose bench data we are checking.'
27 print '-d <dir> a directory containing bench_<revision>_<scalar> files.'
28 print '-e <file> file containing expected bench builder values/ranges.'
29 print ' Will raise exception if actual bench values are out of range.'
30 print ' See bench_expectations_<builder>.txt for data format / examples.'
31 print '-r <revision> the git commit hash or svn revision for checking '
32 print ' bench values.'
33
34
35class Label:
36 """The information in a label.
37
38 (str, str, str, str, {str:str})"""
39 def __init__(self, bench, config, time_type, settings):
40 self.bench = bench
41 self.config = config
42 self.time_type = time_type
43 self.settings = settings
44
45 def __repr__(self):
46 return "Label(%s, %s, %s, %s)" % (
47 str(self.bench),
48 str(self.config),
49 str(self.time_type),
50 str(self.settings),
51 )
52
53 def __str__(self):
54 return "%s_%s_%s_%s" % (
55 str(self.bench),
56 str(self.config),
57 str(self.time_type),
58 str(self.settings),
59 )
60
61 def __eq__(self, other):
62 return (self.bench == other.bench and
63 self.config == other.config and
64 self.time_type == other.time_type and
65 self.settings == other.settings)
66
67 def __hash__(self):
68 return (hash(self.bench) ^
69 hash(self.config) ^
70 hash(self.time_type) ^
71 hash(frozenset(self.settings.iteritems())))
72
73def parse_dir(directory, default_settings, revision, rep):
74 """Parses bench data from bench logs files.
75 revision can be either svn revision or git commit hash.
76 """
77 revision_data_points = [] # list of BenchDataPoint
78 file_list = os.listdir(directory)
79 file_list.sort()
80 for bench_file in file_list:
81 scalar_type = None
82 # Scalar type, if any, is in the bench filename after revision
83 if (len(revision) > MAX_SVN_REV_LENGTH and
84 bench_file.startswith('bench_' + revision + '_')):
85 # The revision is GIT commit hash.
86 scalar_type = bench_file[len(revision) + len('bench_') + 1:]
87 elif (bench_file.startswith('bench_r' + revision + '_') and
88 revision.isdigit()):
89 # The revision is SVN number
90 scalar_type = bench_file[len(revision) + len('bench_r') + 1:]
91 else:
92 continue
93
94 file_handle = open(directory + '/' + bench_file, 'r')
95
96 default_settings['scalar'] = scalar_type
97 revision_data_points.extend(
98 bench_util.parse(default_settings, file_handle, rep))
99 file_handle.close()
100 return revision_data_points
101
102def create_bench_dict(revision_data_points):
103 """Convert current revision data into a dictionary of line data.
104
105 Args:
106 revision_data_points: a list of bench data points
107
108 Returns:
109 a dictionary of this form:
110 keys = Label objects
111 values = the corresponding bench value
112 """
113 bench_dict = {}
114 for point in revision_data_points:
115 point_name = Label(point.bench,point.config,point.time_type,
116 point.settings)
117 if point_name not in bench_dict:
118 bench_dict[point_name] = point.time
119 else:
120 raise Exception('Duplicate expectation entry: ' + str(point_name))
121
122 return bench_dict
123
124def read_expectations(expectations, filename):
125 """Reads expectations data from file and put in expectations dict."""
126 for expectation in open(filename).readlines():
127 elements = expectation.strip().split(',')
128 if not elements[0] or elements[0].startswith('#'):
129 continue
130 if len(elements) != 5:
131 raise Exception("Invalid expectation line format: %s" %
132 expectation)
133 bench_entry = elements[0] + ',' + elements[1]
134 if bench_entry in expectations:
135 raise Exception("Dup entries for bench expectation %s" %
136 bench_entry)
137 # [<Bench_BmpConfig_TimeType>,<Platform-Alg>] -> (LB, UB)
138 expectations[bench_entry] = (float(elements[-2]),
139 float(elements[-1]))
140
141def check_expectations(lines, expectations, revision, key_suffix):
142 """Check if there are benches in the given revising out of range.
143 """
144 # The platform for this bot, to pass to the dashboard plot.
145 platform = key_suffix[ : key_suffix.rfind('-')]
146 exceptions = []
147 for line in lines:
148 line_str = str(line)
149 line_str = line_str[ : line_str.find('_{')]
150 bench_platform_key = line_str + ',' + key_suffix
151 if bench_platform_key not in expectations:
152 continue
153 this_bench_value = lines[line]
154 this_min, this_max = expectations[bench_platform_key]
155 if this_bench_value < this_min or this_bench_value > this_max:
156 exception = 'Bench %s value %s out of range [%s, %s].' % (
157 bench_platform_key, this_bench_value, this_min, this_max)
158 exceptions.append(exception)
159 if exceptions:
160 raise Exception('Bench values out of range:\n' +
161 '\n'.join(exceptions))
162
163def main():
164 """Parses command line and checks bench expectations."""
165 try:
166 opts, _ = getopt.getopt(sys.argv[1:],
167 "a:b:d:e:r:",
168 "default-setting=")
169 except getopt.GetoptError, err:
170 print str(err)
171 usage()
172 sys.exit(2)
173
174 directory = None
175 bench_expectations = {}
176 rep = '25th' # bench representation algorithm, default to 25th
177 rev = None # git commit hash or svn revision number
178 bot = None
179
180 try:
181 for option, value in opts:
182 if option == "-a":
183 rep = value
184 elif option == "-b":
185 bot = value
186 elif option == "-d":
187 directory = value
188 elif option == "-e":
189 read_expectations(bench_expectations, value)
190 elif option == "-r":
191 rev = value
192 else:
193 usage()
194 assert False, "unhandled option"
195 except ValueError:
196 usage()
197 sys.exit(2)
198
199 if directory is None or bot is None or rev is None:
200 usage()
201 sys.exit(2)
202
203 platform_and_alg = bot + '-' + rep
204
205 data_points = parse_dir(directory,
206 {}, # Sets default settings to empty.
207 rev,
208 rep)
209
210 bench_dict = create_bench_dict(data_points)
211
212 if bench_expectations:
213 check_expectations(bench_dict, bench_expectations, rev,
214 platform_and_alg)
215
216
217if __name__ == "__main__":
218 main()