Eric Boren | a1db799 | 2021-03-25 09:04:43 -0400 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | # |
| 3 | # Copyright 2011 Google Inc. |
| 4 | # |
| 5 | # Use of this source code is governed by a BSD-style license that can be |
| 6 | # found in the LICENSE file. |
bensong@google.com | 0d9e6de | 2013-10-15 00:08:14 +0000 | [diff] [blame] | 7 | |
Eric Boren | a1db799 | 2021-03-25 09:04:43 -0400 | [diff] [blame] | 8 | |
| 9 | from __future__ import print_function |
bensong@google.com | 0d9e6de | 2013-10-15 00:08:14 +0000 | [diff] [blame] | 10 | import bench_util |
| 11 | import getopt |
| 12 | import httplib |
| 13 | import itertools |
| 14 | import json |
| 15 | import os |
| 16 | import re |
| 17 | import sys |
| 18 | import urllib |
| 19 | import urllib2 |
| 20 | import xml.sax.saxutils |
| 21 | |
| 22 | # Maximum expected number of characters we expect in an svn revision. |
| 23 | MAX_SVN_REV_LENGTH = 5 |
| 24 | |
commit-bot@chromium.org | 557e3cd | 2014-02-20 15:58:32 +0000 | [diff] [blame] | 25 | # Indices for getting elements from bench expectation files. |
| 26 | # See bench_expectations_<builder>.txt for details. |
| 27 | EXPECTED_IDX = -3 |
| 28 | LB_IDX = -2 |
| 29 | UB_IDX = -1 |
| 30 | |
| 31 | # Indices of the tuple of dictionaries containing slower and faster alerts. |
| 32 | SLOWER = 0 |
| 33 | FASTER = 1 |
| 34 | |
commit-bot@chromium.org | 9d439a7 | 2014-05-13 17:38:17 +0000 | [diff] [blame] | 35 | # URL prefix for the bench dashboard page. Showing recent 15 days of data. |
| 36 | DASHBOARD_URL_PREFIX = 'http://go/skpdash/#15' |
| 37 | |
bensong@google.com | 0d9e6de | 2013-10-15 00:08:14 +0000 | [diff] [blame] | 38 | def usage(): |
| 39 | """Prints simple usage information.""" |
| 40 | |
Eric Boren | a1db799 | 2021-03-25 09:04:43 -0400 | [diff] [blame] | 41 | print('-a <representation_alg> bench representation algorithm to use. ') |
| 42 | print(' Defaults to "25th". See bench_util.py for details.') |
| 43 | print('-b <builder> name of the builder whose bench data we are checking.') |
| 44 | print('-d <dir> a directory containing bench_<revision>_<scalar> files.') |
| 45 | print('-e <file> file containing expected bench builder values/ranges.') |
| 46 | print(' Will raise exception if actual bench values are out of range.') |
| 47 | print(' See bench_expectations_<builder>.txt for data format / examples.') |
| 48 | print('-r <revision> the git commit hash or svn revision for checking ') |
| 49 | print(' bench values.') |
bensong@google.com | 0d9e6de | 2013-10-15 00:08:14 +0000 | [diff] [blame] | 50 | |
| 51 | |
| 52 | class Label: |
| 53 | """The information in a label. |
| 54 | |
| 55 | (str, str, str, str, {str:str})""" |
| 56 | def __init__(self, bench, config, time_type, settings): |
| 57 | self.bench = bench |
| 58 | self.config = config |
| 59 | self.time_type = time_type |
| 60 | self.settings = settings |
| 61 | |
| 62 | def __repr__(self): |
| 63 | return "Label(%s, %s, %s, %s)" % ( |
| 64 | str(self.bench), |
| 65 | str(self.config), |
| 66 | str(self.time_type), |
| 67 | str(self.settings), |
| 68 | ) |
| 69 | |
| 70 | def __str__(self): |
| 71 | return "%s_%s_%s_%s" % ( |
| 72 | str(self.bench), |
| 73 | str(self.config), |
| 74 | str(self.time_type), |
| 75 | str(self.settings), |
| 76 | ) |
| 77 | |
| 78 | def __eq__(self, other): |
| 79 | return (self.bench == other.bench and |
| 80 | self.config == other.config and |
| 81 | self.time_type == other.time_type and |
| 82 | self.settings == other.settings) |
| 83 | |
| 84 | def __hash__(self): |
| 85 | return (hash(self.bench) ^ |
| 86 | hash(self.config) ^ |
| 87 | hash(self.time_type) ^ |
| 88 | hash(frozenset(self.settings.iteritems()))) |
| 89 | |
bensong@google.com | 0d9e6de | 2013-10-15 00:08:14 +0000 | [diff] [blame] | 90 | def create_bench_dict(revision_data_points): |
| 91 | """Convert current revision data into a dictionary of line data. |
| 92 | |
| 93 | Args: |
| 94 | revision_data_points: a list of bench data points |
| 95 | |
| 96 | Returns: |
| 97 | a dictionary of this form: |
| 98 | keys = Label objects |
| 99 | values = the corresponding bench value |
| 100 | """ |
| 101 | bench_dict = {} |
| 102 | for point in revision_data_points: |
| 103 | point_name = Label(point.bench,point.config,point.time_type, |
| 104 | point.settings) |
| 105 | if point_name not in bench_dict: |
| 106 | bench_dict[point_name] = point.time |
| 107 | else: |
| 108 | raise Exception('Duplicate expectation entry: ' + str(point_name)) |
| 109 | |
| 110 | return bench_dict |
| 111 | |
| 112 | def read_expectations(expectations, filename): |
| 113 | """Reads expectations data from file and put in expectations dict.""" |
| 114 | for expectation in open(filename).readlines(): |
| 115 | elements = expectation.strip().split(',') |
| 116 | if not elements[0] or elements[0].startswith('#'): |
| 117 | continue |
| 118 | if len(elements) != 5: |
| 119 | raise Exception("Invalid expectation line format: %s" % |
| 120 | expectation) |
| 121 | bench_entry = elements[0] + ',' + elements[1] |
| 122 | if bench_entry in expectations: |
| 123 | raise Exception("Dup entries for bench expectation %s" % |
| 124 | bench_entry) |
commit-bot@chromium.org | 557e3cd | 2014-02-20 15:58:32 +0000 | [diff] [blame] | 125 | # [<Bench_BmpConfig_TimeType>,<Platform-Alg>] -> (LB, UB, EXPECTED) |
| 126 | expectations[bench_entry] = (float(elements[LB_IDX]), |
| 127 | float(elements[UB_IDX]), |
| 128 | float(elements[EXPECTED_IDX])) |
bensong@google.com | 0d9e6de | 2013-10-15 00:08:14 +0000 | [diff] [blame] | 129 | |
commit-bot@chromium.org | 557e3cd | 2014-02-20 15:58:32 +0000 | [diff] [blame] | 130 | def check_expectations(lines, expectations, key_suffix): |
| 131 | """Check if any bench results are outside of expected range. |
| 132 | |
| 133 | For each input line in lines, checks the expectations dictionary to see if |
| 134 | the bench is out of the given range. |
| 135 | |
| 136 | Args: |
| 137 | lines: dictionary mapping Label objects to the bench values. |
| 138 | expectations: dictionary returned by read_expectations(). |
| 139 | key_suffix: string of <Platform>-<Alg> containing the bot platform and the |
| 140 | bench representation algorithm. |
| 141 | |
| 142 | Returns: |
| 143 | No return value. |
| 144 | |
| 145 | Raises: |
| 146 | Exception containing bench data that are out of range, if any. |
bensong@google.com | 0d9e6de | 2013-10-15 00:08:14 +0000 | [diff] [blame] | 147 | """ |
| 148 | # The platform for this bot, to pass to the dashboard plot. |
| 149 | platform = key_suffix[ : key_suffix.rfind('-')] |
commit-bot@chromium.org | 557e3cd | 2014-02-20 15:58:32 +0000 | [diff] [blame] | 150 | # Tuple of dictionaries recording exceptions that are slower and faster, |
| 151 | # respectively. Each dictionary maps off_ratio (ratio of actual to expected) |
| 152 | # to a list of corresponding exception messages. |
| 153 | exceptions = ({}, {}) |
bensong@google.com | 0d9e6de | 2013-10-15 00:08:14 +0000 | [diff] [blame] | 154 | for line in lines: |
| 155 | line_str = str(line) |
| 156 | line_str = line_str[ : line_str.find('_{')] |
commit-bot@chromium.org | 9d439a7 | 2014-05-13 17:38:17 +0000 | [diff] [blame] | 157 | # Extracts bench and config from line_str, which is in the format |
| 158 | # <bench-picture-name>.skp_<config>_ |
| 159 | bench, config = line_str.strip('_').split('.skp_') |
bensong@google.com | 0d9e6de | 2013-10-15 00:08:14 +0000 | [diff] [blame] | 160 | bench_platform_key = line_str + ',' + key_suffix |
| 161 | if bench_platform_key not in expectations: |
| 162 | continue |
| 163 | this_bench_value = lines[line] |
commit-bot@chromium.org | 557e3cd | 2014-02-20 15:58:32 +0000 | [diff] [blame] | 164 | this_min, this_max, this_expected = expectations[bench_platform_key] |
bensong@google.com | 0d9e6de | 2013-10-15 00:08:14 +0000 | [diff] [blame] | 165 | if this_bench_value < this_min or this_bench_value > this_max: |
commit-bot@chromium.org | 557e3cd | 2014-02-20 15:58:32 +0000 | [diff] [blame] | 166 | off_ratio = this_bench_value / this_expected |
| 167 | exception = 'Bench %s out of range [%s, %s] (%s vs %s, %s%%).' % ( |
| 168 | bench_platform_key, this_min, this_max, this_bench_value, |
| 169 | this_expected, (off_ratio - 1) * 100) |
commit-bot@chromium.org | 9d439a7 | 2014-05-13 17:38:17 +0000 | [diff] [blame] | 170 | exception += '\n' + '~'.join([ |
| 171 | DASHBOARD_URL_PREFIX, bench, platform, config]) |
commit-bot@chromium.org | 557e3cd | 2014-02-20 15:58:32 +0000 | [diff] [blame] | 172 | if off_ratio > 1: # Bench is slower. |
| 173 | exceptions[SLOWER].setdefault(off_ratio, []).append(exception) |
| 174 | else: |
| 175 | exceptions[FASTER].setdefault(off_ratio, []).append(exception) |
| 176 | outputs = [] |
| 177 | for i in [SLOWER, FASTER]: |
| 178 | if exceptions[i]: |
| 179 | ratios = exceptions[i].keys() |
| 180 | ratios.sort(reverse=True) |
| 181 | li = [] |
| 182 | for ratio in ratios: |
| 183 | li.extend(exceptions[i][ratio]) |
| 184 | header = '%s benches got slower (sorted by %% difference):' % len(li) |
| 185 | if i == FASTER: |
| 186 | header = header.replace('slower', 'faster') |
| 187 | outputs.extend(['', header] + li) |
| 188 | |
| 189 | if outputs: |
commit-bot@chromium.org | cb3f079 | 2014-03-28 17:55:51 +0000 | [diff] [blame] | 190 | # Directly raising Exception will have stderr outputs tied to the line |
| 191 | # number of the script, so use sys.stderr.write() instead. |
| 192 | # Add a trailing newline to supress new line checking errors. |
| 193 | sys.stderr.write('\n'.join(['Exception:'] + outputs + ['\n'])) |
| 194 | exit(1) |
bensong@google.com | 0d9e6de | 2013-10-15 00:08:14 +0000 | [diff] [blame] | 195 | |
commit-bot@chromium.org | b1bcb21 | 2014-03-17 21:16:29 +0000 | [diff] [blame] | 196 | |
bensong@google.com | 0d9e6de | 2013-10-15 00:08:14 +0000 | [diff] [blame] | 197 | def main(): |
| 198 | """Parses command line and checks bench expectations.""" |
| 199 | try: |
| 200 | opts, _ = getopt.getopt(sys.argv[1:], |
| 201 | "a:b:d:e:r:", |
| 202 | "default-setting=") |
Eric Boren | a1db799 | 2021-03-25 09:04:43 -0400 | [diff] [blame] | 203 | except getopt.GetoptError as err: |
| 204 | print(str(err)) |
bensong@google.com | 0d9e6de | 2013-10-15 00:08:14 +0000 | [diff] [blame] | 205 | usage() |
| 206 | sys.exit(2) |
| 207 | |
| 208 | directory = None |
| 209 | bench_expectations = {} |
| 210 | rep = '25th' # bench representation algorithm, default to 25th |
| 211 | rev = None # git commit hash or svn revision number |
| 212 | bot = None |
| 213 | |
| 214 | try: |
| 215 | for option, value in opts: |
| 216 | if option == "-a": |
| 217 | rep = value |
| 218 | elif option == "-b": |
| 219 | bot = value |
| 220 | elif option == "-d": |
| 221 | directory = value |
| 222 | elif option == "-e": |
| 223 | read_expectations(bench_expectations, value) |
| 224 | elif option == "-r": |
| 225 | rev = value |
| 226 | else: |
| 227 | usage() |
| 228 | assert False, "unhandled option" |
| 229 | except ValueError: |
| 230 | usage() |
| 231 | sys.exit(2) |
| 232 | |
| 233 | if directory is None or bot is None or rev is None: |
| 234 | usage() |
| 235 | sys.exit(2) |
| 236 | |
| 237 | platform_and_alg = bot + '-' + rep |
| 238 | |
commit-bot@chromium.org | b1bcb21 | 2014-03-17 21:16:29 +0000 | [diff] [blame] | 239 | data_points = bench_util.parse_skp_bench_data(directory, rev, rep) |
bensong@google.com | 0d9e6de | 2013-10-15 00:08:14 +0000 | [diff] [blame] | 240 | |
| 241 | bench_dict = create_bench_dict(data_points) |
| 242 | |
| 243 | if bench_expectations: |
commit-bot@chromium.org | 557e3cd | 2014-02-20 15:58:32 +0000 | [diff] [blame] | 244 | check_expectations(bench_dict, bench_expectations, platform_and_alg) |
bensong@google.com | 0d9e6de | 2013-10-15 00:08:14 +0000 | [diff] [blame] | 245 | |
| 246 | |
| 247 | if __name__ == "__main__": |
| 248 | main() |