bensong@google.com | fa1d4ea | 2012-11-27 17:30:26 +0000 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| 3 | # Use of this source code is governed by a BSD-style license that can be found |
| 4 | # in the LICENSE file. |
| 5 | |
| 6 | """ Analyze recent SkPicture bench data, and output suggested ranges. |
| 7 | |
| 8 | The outputs can be edited and pasted to bench_expectations.txt to trigger |
| 9 | buildbot alerts if the actual benches are out of range. Details are documented |
| 10 | in the .txt file. |
| 11 | |
| 12 | Currently the easiest way to update bench_expectations.txt is to delete all skp |
| 13 | bench lines, run this script, and redirect outputs (">>") to be added to the |
| 14 | .txt file. |
| 15 | TODO(bensong): find a better way for updating the bench lines in place. |
| 16 | |
| 17 | Note: since input data are stored in Google Storage, you will need to set up |
| 18 | the corresponding library. |
| 19 | See http://developers.google.com/storage/docs/gspythonlibrary for details. |
| 20 | """ |
| 21 | |
| 22 | __author__ = 'bensong@google.com (Ben Chen)' |
| 23 | |
| 24 | import bench_util |
| 25 | import boto |
| 26 | import cStringIO |
| 27 | import optparse |
| 28 | import re |
| 29 | import shutil |
| 30 | |
| 31 | from oauth2_plugin import oauth2_plugin |
| 32 | |
| 33 | |
| 34 | # Ratios for calculating suggested picture bench upper and lower bounds. |
bensong@google.com | dc2dd2e | 2012-11-27 21:52:32 +0000 | [diff] [blame] | 35 | BENCH_UB = 1.1 # Allow for 10% room for normal variance on the up side. |
| 36 | BENCH_LB = 0.85 |
bensong@google.com | fa1d4ea | 2012-11-27 17:30:26 +0000 | [diff] [blame] | 37 | |
bensong@google.com | cd63fb5 | 2012-11-30 04:42:59 +0000 | [diff] [blame] | 38 | # Further allow for a fixed amount of noise. This is especially useful for |
| 39 | # benches of smaller absolute value. Keeping this value small will not affect |
| 40 | # performance tunings. |
| 41 | BENCH_ALLOWED_NOISE = 10 |
| 42 | |
bensong@google.com | fa1d4ea | 2012-11-27 17:30:26 +0000 | [diff] [blame] | 43 | # List of platforms to track. |
| 44 | PLATFORMS = ['Mac_Float_Bench_32', |
| 45 | 'Nexus10_4-1_Float_Bench_32', |
| 46 | 'Shuttle_Ubuntu12_ATI5770_Float_Bench_32', |
| 47 | ] |
| 48 | |
| 49 | # Filter for configs of no interest. They are old config names replaced by more |
| 50 | # specific ones. |
| 51 | CONFIGS_TO_FILTER = ['gpu', 'raster'] |
| 52 | |
| 53 | # Template for gsutil uri. |
| 54 | GOOGLE_STORAGE_URI_SCHEME = 'gs' |
| 55 | URI_BUCKET = 'chromium-skia-gm' |
| 56 | |
| 57 | # Constants for optparse. |
| 58 | USAGE_STRING = 'USAGE: %s [options]' |
| 59 | HOWTO_STRING = """ |
| 60 | Feel free to revise PLATFORMS for your own needs. The default is the most common |
| 61 | combination that we care most about. Platforms that did not run bench_pictures |
| 62 | in the given revision range will not have corresponding outputs. |
| 63 | Please check http://go/skpbench to choose a range that fits your needs. |
bensong@google.com | cd63fb5 | 2012-11-30 04:42:59 +0000 | [diff] [blame] | 64 | BENCH_UB, BENCH_LB and BENCH_ALLOWED_NOISE can be changed to expand or narrow |
| 65 | the permitted bench ranges without triggering buidbot alerts. |
bensong@google.com | fa1d4ea | 2012-11-27 17:30:26 +0000 | [diff] [blame] | 66 | """ |
| 67 | HELP_STRING = """ |
| 68 | Outputs expectation picture bench ranges for the latest revisions for the given |
| 69 | revision range. For instance, --rev_range=6000:6000 will return only bench |
| 70 | ranges for the bots that ran bench_pictures at rev 6000; --rev-range=6000:7000 |
| 71 | may have multiple bench data points for each bench configuration, and the code |
| 72 | returns bench data for the latest revision of all available (closer to 7000). |
| 73 | """ + HOWTO_STRING |
| 74 | |
| 75 | OPTION_REVISION_RANGE = '--rev-range' |
| 76 | OPTION_REVISION_RANGE_SHORT = '-r' |
| 77 | # Bench bench representation algorithm flag. |
| 78 | OPTION_REPRESENTATION_ALG = '--algorithm' |
| 79 | OPTION_REPRESENTATION_ALG_SHORT = '-a' |
| 80 | |
| 81 | # List of valid representation algorithms. |
| 82 | REPRESENTATION_ALGS = ['avg', 'min', 'med', '25th'] |
| 83 | |
| 84 | def OutputSkpBenchExpectations(rev_min, rev_max, representation_alg): |
| 85 | """Reads skp bench data from google storage, and outputs expectations. |
| 86 | |
| 87 | Ignores data with revisions outside [rev_min, rev_max] integer range. For |
| 88 | bench data with multiple revisions, we use higher revisions to calculate |
| 89 | expected bench values. |
| 90 | Uses the provided representation_alg for calculating bench representations. |
| 91 | """ |
| 92 | expectation_dic = {} |
| 93 | uri = boto.storage_uri(URI_BUCKET, GOOGLE_STORAGE_URI_SCHEME) |
| 94 | for obj in uri.get_bucket(): |
| 95 | # Filters out non-skp-bench files. |
| 96 | if (not obj.name.startswith('perfdata/Skia_') or |
| 97 | obj.name.find('_data_skp_') < 0): |
| 98 | continue |
| 99 | # Ignores uninterested platforms. |
| 100 | platform = obj.name.split('/')[1][5:] # Removes "Skia_" prefix. |
| 101 | if platform not in PLATFORMS: |
| 102 | continue |
| 103 | # Filters by revision. |
| 104 | for rev in range(rev_min, rev_max + 1): |
| 105 | if '_r%s_' % rev not in obj.name: |
| 106 | continue |
| 107 | |
| 108 | contents = cStringIO.StringIO() |
| 109 | obj.get_file(contents) |
| 110 | for point in bench_util.parse('', contents.getvalue().split('\n'), |
| 111 | representation_alg): |
| 112 | if point.config in CONFIGS_TO_FILTER: |
| 113 | continue |
| 114 | # TODO(bensong): the filtering below is only needed during skp generation |
| 115 | # system transitioning. Change it once the new system (bench name starts |
| 116 | # with http) is stable for the switch-over, and delete it once we |
| 117 | # deprecate the old ones. |
| 118 | if point.bench.startswith('http'): |
| 119 | continue |
| 120 | |
| 121 | key = '%s_%s_%s,%s-%s' % (point.bench, point.config, point.time_type, |
| 122 | platform, representation_alg) |
| 123 | # It is fine to have later revisions overwrite earlier benches, since we |
| 124 | # only use the latest bench within revision range to set expectations. |
| 125 | expectation_dic[key] = point.time |
| 126 | keys = expectation_dic.keys() |
| 127 | keys.sort() |
| 128 | for key in keys: |
| 129 | bench_val = expectation_dic[key] |
| 130 | # Prints out expectation lines. |
bensong@google.com | cd63fb5 | 2012-11-30 04:42:59 +0000 | [diff] [blame] | 131 | print '%s,%.3f,%.3f,%.3f' % (key, bench_val, |
| 132 | bench_val * BENCH_LB - BENCH_ALLOWED_NOISE, |
| 133 | bench_val * BENCH_UB + BENCH_ALLOWED_NOISE) |
bensong@google.com | fa1d4ea | 2012-11-27 17:30:26 +0000 | [diff] [blame] | 134 | |
| 135 | def main(): |
| 136 | """Parses flags and outputs expected Skia picture bench results.""" |
| 137 | parser = optparse.OptionParser(USAGE_STRING % '%prog' + HELP_STRING) |
| 138 | parser.add_option(OPTION_REVISION_RANGE_SHORT, OPTION_REVISION_RANGE, |
| 139 | dest='rev_range', |
| 140 | help='(Mandatory) revision range separated by ":", e.g., 6000:6005') |
| 141 | parser.add_option(OPTION_REPRESENTATION_ALG_SHORT, OPTION_REPRESENTATION_ALG, |
| 142 | dest='alg', default='25th', |
| 143 | help=('Bench representation algorithm. One of ' |
| 144 | '%s. Default to "25th".' % str(REPRESENTATION_ALGS))) |
| 145 | (options, args) = parser.parse_args() |
| 146 | if options.rev_range: |
| 147 | range_match = re.search('(\d+)\:(\d+)', options.rev_range) |
| 148 | if not range_match: |
| 149 | parser.error('Wrong format for rev-range [%s]' % options.rev_range) |
| 150 | else: |
| 151 | rev_min = int(range_match.group(1)) |
| 152 | rev_max = int(range_match.group(2)) |
| 153 | OutputSkpBenchExpectations(rev_min, rev_max, options.alg) |
| 154 | else: |
| 155 | parser.error('Please provide mandatory flag %s' % OPTION_REVISION_RANGE) |
| 156 | |
| 157 | |
| 158 | if '__main__' == __name__: |
| 159 | main() |