bensong@google.com | 0d9e6de | 2013-10-15 00:08:14 +0000 | [diff] [blame] | 1 | ''' |
| 2 | Created on May 16, 2011 |
| 3 | |
| 4 | @author: bungeman |
| 5 | ''' |
| 6 | import bench_util |
| 7 | import getopt |
| 8 | import httplib |
| 9 | import itertools |
| 10 | import json |
| 11 | import os |
| 12 | import re |
| 13 | import sys |
| 14 | import urllib |
| 15 | import urllib2 |
| 16 | import xml.sax.saxutils |
| 17 | |
| 18 | # Maximum expected number of characters we expect in an svn revision. |
| 19 | MAX_SVN_REV_LENGTH = 5 |
| 20 | |
commit-bot@chromium.org | 557e3cd | 2014-02-20 15:58:32 +0000 | [diff] [blame] | 21 | # Indices for getting elements from bench expectation files. |
| 22 | # See bench_expectations_<builder>.txt for details. |
| 23 | EXPECTED_IDX = -3 |
| 24 | LB_IDX = -2 |
| 25 | UB_IDX = -1 |
| 26 | |
| 27 | # Indices of the tuple of dictionaries containing slower and faster alerts. |
| 28 | SLOWER = 0 |
| 29 | FASTER = 1 |
| 30 | |
bensong@google.com | 0d9e6de | 2013-10-15 00:08:14 +0000 | [diff] [blame] | 31 | def usage(): |
| 32 | """Prints simple usage information.""" |
| 33 | |
| 34 | print '-a <representation_alg> bench representation algorithm to use. ' |
| 35 | print ' Defaults to "25th". See bench_util.py for details.' |
| 36 | print '-b <builder> name of the builder whose bench data we are checking.' |
| 37 | print '-d <dir> a directory containing bench_<revision>_<scalar> files.' |
| 38 | print '-e <file> file containing expected bench builder values/ranges.' |
| 39 | print ' Will raise exception if actual bench values are out of range.' |
| 40 | print ' See bench_expectations_<builder>.txt for data format / examples.' |
| 41 | print '-r <revision> the git commit hash or svn revision for checking ' |
| 42 | print ' bench values.' |
| 43 | |
| 44 | |
| 45 | class Label: |
| 46 | """The information in a label. |
| 47 | |
| 48 | (str, str, str, str, {str:str})""" |
| 49 | def __init__(self, bench, config, time_type, settings): |
| 50 | self.bench = bench |
| 51 | self.config = config |
| 52 | self.time_type = time_type |
| 53 | self.settings = settings |
| 54 | |
| 55 | def __repr__(self): |
| 56 | return "Label(%s, %s, %s, %s)" % ( |
| 57 | str(self.bench), |
| 58 | str(self.config), |
| 59 | str(self.time_type), |
| 60 | str(self.settings), |
| 61 | ) |
| 62 | |
| 63 | def __str__(self): |
| 64 | return "%s_%s_%s_%s" % ( |
| 65 | str(self.bench), |
| 66 | str(self.config), |
| 67 | str(self.time_type), |
| 68 | str(self.settings), |
| 69 | ) |
| 70 | |
| 71 | def __eq__(self, other): |
| 72 | return (self.bench == other.bench and |
| 73 | self.config == other.config and |
| 74 | self.time_type == other.time_type and |
| 75 | self.settings == other.settings) |
| 76 | |
| 77 | def __hash__(self): |
| 78 | return (hash(self.bench) ^ |
| 79 | hash(self.config) ^ |
| 80 | hash(self.time_type) ^ |
| 81 | hash(frozenset(self.settings.iteritems()))) |
| 82 | |
bensong@google.com | 0d9e6de | 2013-10-15 00:08:14 +0000 | [diff] [blame] | 83 | def create_bench_dict(revision_data_points): |
| 84 | """Convert current revision data into a dictionary of line data. |
| 85 | |
| 86 | Args: |
| 87 | revision_data_points: a list of bench data points |
| 88 | |
| 89 | Returns: |
| 90 | a dictionary of this form: |
| 91 | keys = Label objects |
| 92 | values = the corresponding bench value |
| 93 | """ |
| 94 | bench_dict = {} |
| 95 | for point in revision_data_points: |
| 96 | point_name = Label(point.bench,point.config,point.time_type, |
| 97 | point.settings) |
| 98 | if point_name not in bench_dict: |
| 99 | bench_dict[point_name] = point.time |
| 100 | else: |
| 101 | raise Exception('Duplicate expectation entry: ' + str(point_name)) |
| 102 | |
| 103 | return bench_dict |
| 104 | |
| 105 | def read_expectations(expectations, filename): |
| 106 | """Reads expectations data from file and put in expectations dict.""" |
| 107 | for expectation in open(filename).readlines(): |
| 108 | elements = expectation.strip().split(',') |
| 109 | if not elements[0] or elements[0].startswith('#'): |
| 110 | continue |
| 111 | if len(elements) != 5: |
| 112 | raise Exception("Invalid expectation line format: %s" % |
| 113 | expectation) |
| 114 | bench_entry = elements[0] + ',' + elements[1] |
| 115 | if bench_entry in expectations: |
| 116 | raise Exception("Dup entries for bench expectation %s" % |
| 117 | bench_entry) |
commit-bot@chromium.org | 557e3cd | 2014-02-20 15:58:32 +0000 | [diff] [blame] | 118 | # [<Bench_BmpConfig_TimeType>,<Platform-Alg>] -> (LB, UB, EXPECTED) |
| 119 | expectations[bench_entry] = (float(elements[LB_IDX]), |
| 120 | float(elements[UB_IDX]), |
| 121 | float(elements[EXPECTED_IDX])) |
bensong@google.com | 0d9e6de | 2013-10-15 00:08:14 +0000 | [diff] [blame] | 122 | |
commit-bot@chromium.org | 557e3cd | 2014-02-20 15:58:32 +0000 | [diff] [blame] | 123 | def check_expectations(lines, expectations, key_suffix): |
| 124 | """Check if any bench results are outside of expected range. |
| 125 | |
| 126 | For each input line in lines, checks the expectations dictionary to see if |
| 127 | the bench is out of the given range. |
| 128 | |
| 129 | Args: |
| 130 | lines: dictionary mapping Label objects to the bench values. |
| 131 | expectations: dictionary returned by read_expectations(). |
| 132 | key_suffix: string of <Platform>-<Alg> containing the bot platform and the |
| 133 | bench representation algorithm. |
| 134 | |
| 135 | Returns: |
| 136 | No return value. |
| 137 | |
| 138 | Raises: |
| 139 | Exception containing bench data that are out of range, if any. |
bensong@google.com | 0d9e6de | 2013-10-15 00:08:14 +0000 | [diff] [blame] | 140 | """ |
| 141 | # The platform for this bot, to pass to the dashboard plot. |
| 142 | platform = key_suffix[ : key_suffix.rfind('-')] |
commit-bot@chromium.org | 557e3cd | 2014-02-20 15:58:32 +0000 | [diff] [blame] | 143 | # Tuple of dictionaries recording exceptions that are slower and faster, |
| 144 | # respectively. Each dictionary maps off_ratio (ratio of actual to expected) |
| 145 | # to a list of corresponding exception messages. |
| 146 | exceptions = ({}, {}) |
bensong@google.com | 0d9e6de | 2013-10-15 00:08:14 +0000 | [diff] [blame] | 147 | for line in lines: |
| 148 | line_str = str(line) |
| 149 | line_str = line_str[ : line_str.find('_{')] |
| 150 | bench_platform_key = line_str + ',' + key_suffix |
| 151 | if bench_platform_key not in expectations: |
| 152 | continue |
| 153 | this_bench_value = lines[line] |
commit-bot@chromium.org | 557e3cd | 2014-02-20 15:58:32 +0000 | [diff] [blame] | 154 | this_min, this_max, this_expected = expectations[bench_platform_key] |
bensong@google.com | 0d9e6de | 2013-10-15 00:08:14 +0000 | [diff] [blame] | 155 | if this_bench_value < this_min or this_bench_value > this_max: |
commit-bot@chromium.org | 557e3cd | 2014-02-20 15:58:32 +0000 | [diff] [blame] | 156 | off_ratio = this_bench_value / this_expected |
| 157 | exception = 'Bench %s out of range [%s, %s] (%s vs %s, %s%%).' % ( |
| 158 | bench_platform_key, this_min, this_max, this_bench_value, |
| 159 | this_expected, (off_ratio - 1) * 100) |
| 160 | if off_ratio > 1: # Bench is slower. |
| 161 | exceptions[SLOWER].setdefault(off_ratio, []).append(exception) |
| 162 | else: |
| 163 | exceptions[FASTER].setdefault(off_ratio, []).append(exception) |
| 164 | outputs = [] |
| 165 | for i in [SLOWER, FASTER]: |
| 166 | if exceptions[i]: |
| 167 | ratios = exceptions[i].keys() |
| 168 | ratios.sort(reverse=True) |
| 169 | li = [] |
| 170 | for ratio in ratios: |
| 171 | li.extend(exceptions[i][ratio]) |
| 172 | header = '%s benches got slower (sorted by %% difference):' % len(li) |
| 173 | if i == FASTER: |
| 174 | header = header.replace('slower', 'faster') |
| 175 | outputs.extend(['', header] + li) |
| 176 | |
| 177 | if outputs: |
| 178 | raise Exception('\n'.join(outputs)) |
bensong@google.com | 0d9e6de | 2013-10-15 00:08:14 +0000 | [diff] [blame] | 179 | |
commit-bot@chromium.org | b1bcb21 | 2014-03-17 21:16:29 +0000 | [diff] [blame^] | 180 | |
bensong@google.com | 0d9e6de | 2013-10-15 00:08:14 +0000 | [diff] [blame] | 181 | def main(): |
| 182 | """Parses command line and checks bench expectations.""" |
| 183 | try: |
| 184 | opts, _ = getopt.getopt(sys.argv[1:], |
| 185 | "a:b:d:e:r:", |
| 186 | "default-setting=") |
| 187 | except getopt.GetoptError, err: |
| 188 | print str(err) |
| 189 | usage() |
| 190 | sys.exit(2) |
| 191 | |
| 192 | directory = None |
| 193 | bench_expectations = {} |
| 194 | rep = '25th' # bench representation algorithm, default to 25th |
| 195 | rev = None # git commit hash or svn revision number |
| 196 | bot = None |
| 197 | |
| 198 | try: |
| 199 | for option, value in opts: |
| 200 | if option == "-a": |
| 201 | rep = value |
| 202 | elif option == "-b": |
| 203 | bot = value |
| 204 | elif option == "-d": |
| 205 | directory = value |
| 206 | elif option == "-e": |
| 207 | read_expectations(bench_expectations, value) |
| 208 | elif option == "-r": |
| 209 | rev = value |
| 210 | else: |
| 211 | usage() |
| 212 | assert False, "unhandled option" |
| 213 | except ValueError: |
| 214 | usage() |
| 215 | sys.exit(2) |
| 216 | |
| 217 | if directory is None or bot is None or rev is None: |
| 218 | usage() |
| 219 | sys.exit(2) |
| 220 | |
| 221 | platform_and_alg = bot + '-' + rep |
| 222 | |
commit-bot@chromium.org | b1bcb21 | 2014-03-17 21:16:29 +0000 | [diff] [blame^] | 223 | data_points = bench_util.parse_skp_bench_data(directory, rev, rep) |
bensong@google.com | 0d9e6de | 2013-10-15 00:08:14 +0000 | [diff] [blame] | 224 | |
| 225 | bench_dict = create_bench_dict(data_points) |
| 226 | |
| 227 | if bench_expectations: |
commit-bot@chromium.org | 557e3cd | 2014-02-20 15:58:32 +0000 | [diff] [blame] | 228 | check_expectations(bench_dict, bench_expectations, platform_and_alg) |
bensong@google.com | 0d9e6de | 2013-10-15 00:08:14 +0000 | [diff] [blame] | 229 | |
| 230 | |
| 231 | if __name__ == "__main__": |
| 232 | main() |