bensong@google.com | 0d9e6de | 2013-10-15 00:08:14 +0000 | [diff] [blame^] | 1 | ''' |
| 2 | Created on May 16, 2011 |
| 3 | |
| 4 | @author: bungeman |
| 5 | ''' |
| 6 | import bench_util |
| 7 | import getopt |
| 8 | import httplib |
| 9 | import itertools |
| 10 | import json |
| 11 | import os |
| 12 | import re |
| 13 | import sys |
| 14 | import urllib |
| 15 | import urllib2 |
| 16 | import xml.sax.saxutils |
| 17 | |
| 18 | # Maximum expected number of characters we expect in an svn revision. |
| 19 | MAX_SVN_REV_LENGTH = 5 |
| 20 | |
| 21 | def usage(): |
| 22 | """Prints simple usage information.""" |
| 23 | |
| 24 | print '-a <representation_alg> bench representation algorithm to use. ' |
| 25 | print ' Defaults to "25th". See bench_util.py for details.' |
| 26 | print '-b <builder> name of the builder whose bench data we are checking.' |
| 27 | print '-d <dir> a directory containing bench_<revision>_<scalar> files.' |
| 28 | print '-e <file> file containing expected bench builder values/ranges.' |
| 29 | print ' Will raise exception if actual bench values are out of range.' |
| 30 | print ' See bench_expectations_<builder>.txt for data format / examples.' |
| 31 | print '-r <revision> the git commit hash or svn revision for checking ' |
| 32 | print ' bench values.' |
| 33 | |
| 34 | |
| 35 | class Label: |
| 36 | """The information in a label. |
| 37 | |
| 38 | (str, str, str, str, {str:str})""" |
| 39 | def __init__(self, bench, config, time_type, settings): |
| 40 | self.bench = bench |
| 41 | self.config = config |
| 42 | self.time_type = time_type |
| 43 | self.settings = settings |
| 44 | |
| 45 | def __repr__(self): |
| 46 | return "Label(%s, %s, %s, %s)" % ( |
| 47 | str(self.bench), |
| 48 | str(self.config), |
| 49 | str(self.time_type), |
| 50 | str(self.settings), |
| 51 | ) |
| 52 | |
| 53 | def __str__(self): |
| 54 | return "%s_%s_%s_%s" % ( |
| 55 | str(self.bench), |
| 56 | str(self.config), |
| 57 | str(self.time_type), |
| 58 | str(self.settings), |
| 59 | ) |
| 60 | |
| 61 | def __eq__(self, other): |
| 62 | return (self.bench == other.bench and |
| 63 | self.config == other.config and |
| 64 | self.time_type == other.time_type and |
| 65 | self.settings == other.settings) |
| 66 | |
| 67 | def __hash__(self): |
| 68 | return (hash(self.bench) ^ |
| 69 | hash(self.config) ^ |
| 70 | hash(self.time_type) ^ |
| 71 | hash(frozenset(self.settings.iteritems()))) |
| 72 | |
| 73 | def parse_dir(directory, default_settings, revision, rep): |
| 74 | """Parses bench data from bench logs files. |
| 75 | revision can be either svn revision or git commit hash. |
| 76 | """ |
| 77 | revision_data_points = [] # list of BenchDataPoint |
| 78 | file_list = os.listdir(directory) |
| 79 | file_list.sort() |
| 80 | for bench_file in file_list: |
| 81 | scalar_type = None |
| 82 | # Scalar type, if any, is in the bench filename after revision |
| 83 | if (len(revision) > MAX_SVN_REV_LENGTH and |
| 84 | bench_file.startswith('bench_' + revision + '_')): |
| 85 | # The revision is GIT commit hash. |
| 86 | scalar_type = bench_file[len(revision) + len('bench_') + 1:] |
| 87 | elif (bench_file.startswith('bench_r' + revision + '_') and |
| 88 | revision.isdigit()): |
| 89 | # The revision is SVN number |
| 90 | scalar_type = bench_file[len(revision) + len('bench_r') + 1:] |
| 91 | else: |
| 92 | continue |
| 93 | |
| 94 | file_handle = open(directory + '/' + bench_file, 'r') |
| 95 | |
| 96 | default_settings['scalar'] = scalar_type |
| 97 | revision_data_points.extend( |
| 98 | bench_util.parse(default_settings, file_handle, rep)) |
| 99 | file_handle.close() |
| 100 | return revision_data_points |
| 101 | |
| 102 | def create_bench_dict(revision_data_points): |
| 103 | """Convert current revision data into a dictionary of line data. |
| 104 | |
| 105 | Args: |
| 106 | revision_data_points: a list of bench data points |
| 107 | |
| 108 | Returns: |
| 109 | a dictionary of this form: |
| 110 | keys = Label objects |
| 111 | values = the corresponding bench value |
| 112 | """ |
| 113 | bench_dict = {} |
| 114 | for point in revision_data_points: |
| 115 | point_name = Label(point.bench,point.config,point.time_type, |
| 116 | point.settings) |
| 117 | if point_name not in bench_dict: |
| 118 | bench_dict[point_name] = point.time |
| 119 | else: |
| 120 | raise Exception('Duplicate expectation entry: ' + str(point_name)) |
| 121 | |
| 122 | return bench_dict |
| 123 | |
| 124 | def read_expectations(expectations, filename): |
| 125 | """Reads expectations data from file and put in expectations dict.""" |
| 126 | for expectation in open(filename).readlines(): |
| 127 | elements = expectation.strip().split(',') |
| 128 | if not elements[0] or elements[0].startswith('#'): |
| 129 | continue |
| 130 | if len(elements) != 5: |
| 131 | raise Exception("Invalid expectation line format: %s" % |
| 132 | expectation) |
| 133 | bench_entry = elements[0] + ',' + elements[1] |
| 134 | if bench_entry in expectations: |
| 135 | raise Exception("Dup entries for bench expectation %s" % |
| 136 | bench_entry) |
| 137 | # [<Bench_BmpConfig_TimeType>,<Platform-Alg>] -> (LB, UB) |
| 138 | expectations[bench_entry] = (float(elements[-2]), |
| 139 | float(elements[-1])) |
| 140 | |
| 141 | def check_expectations(lines, expectations, revision, key_suffix): |
| 142 | """Check if there are benches in the given revising out of range. |
| 143 | """ |
| 144 | # The platform for this bot, to pass to the dashboard plot. |
| 145 | platform = key_suffix[ : key_suffix.rfind('-')] |
| 146 | exceptions = [] |
| 147 | for line in lines: |
| 148 | line_str = str(line) |
| 149 | line_str = line_str[ : line_str.find('_{')] |
| 150 | bench_platform_key = line_str + ',' + key_suffix |
| 151 | if bench_platform_key not in expectations: |
| 152 | continue |
| 153 | this_bench_value = lines[line] |
| 154 | this_min, this_max = expectations[bench_platform_key] |
| 155 | if this_bench_value < this_min or this_bench_value > this_max: |
| 156 | exception = 'Bench %s value %s out of range [%s, %s].' % ( |
| 157 | bench_platform_key, this_bench_value, this_min, this_max) |
| 158 | exceptions.append(exception) |
| 159 | if exceptions: |
| 160 | raise Exception('Bench values out of range:\n' + |
| 161 | '\n'.join(exceptions)) |
| 162 | |
| 163 | def main(): |
| 164 | """Parses command line and checks bench expectations.""" |
| 165 | try: |
| 166 | opts, _ = getopt.getopt(sys.argv[1:], |
| 167 | "a:b:d:e:r:", |
| 168 | "default-setting=") |
| 169 | except getopt.GetoptError, err: |
| 170 | print str(err) |
| 171 | usage() |
| 172 | sys.exit(2) |
| 173 | |
| 174 | directory = None |
| 175 | bench_expectations = {} |
| 176 | rep = '25th' # bench representation algorithm, default to 25th |
| 177 | rev = None # git commit hash or svn revision number |
| 178 | bot = None |
| 179 | |
| 180 | try: |
| 181 | for option, value in opts: |
| 182 | if option == "-a": |
| 183 | rep = value |
| 184 | elif option == "-b": |
| 185 | bot = value |
| 186 | elif option == "-d": |
| 187 | directory = value |
| 188 | elif option == "-e": |
| 189 | read_expectations(bench_expectations, value) |
| 190 | elif option == "-r": |
| 191 | rev = value |
| 192 | else: |
| 193 | usage() |
| 194 | assert False, "unhandled option" |
| 195 | except ValueError: |
| 196 | usage() |
| 197 | sys.exit(2) |
| 198 | |
| 199 | if directory is None or bot is None or rev is None: |
| 200 | usage() |
| 201 | sys.exit(2) |
| 202 | |
| 203 | platform_and_alg = bot + '-' + rep |
| 204 | |
| 205 | data_points = parse_dir(directory, |
| 206 | {}, # Sets default settings to empty. |
| 207 | rev, |
| 208 | rep) |
| 209 | |
| 210 | bench_dict = create_bench_dict(data_points) |
| 211 | |
| 212 | if bench_expectations: |
| 213 | check_expectations(bench_dict, bench_expectations, rev, |
| 214 | platform_and_alg) |
| 215 | |
| 216 | |
| 217 | if __name__ == "__main__": |
| 218 | main() |