Adds microbench range generation option; renames file appropriately. (SkipBuildbotRuns)
Review URL: https://codereview.chromium.org/16858015
git-svn-id: http://skia.googlecode.com/svn/trunk@9601 2bbb7eff-a529-9590-31e7-b0007b416f81
diff --git a/bench/gen_bench_ranges.py b/bench/gen_bench_ranges.py
new file mode 100755
index 0000000..93be53c
--- /dev/null
+++ b/bench/gen_bench_ranges.py
@@ -0,0 +1,181 @@
+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be found
+# in the LICENSE file.
+
+""" Analyze recent SkPicture or Microbench data, and output suggested ranges.
+
+The outputs can be edited and pasted to bench_expectations.txt to trigger
+buildbot alerts if the actual benches are out of range. Details are documented
+in the .txt file.
+
+Currently the easiest way to batch update bench_expectations.txt is to delete
+all bench lines, run this script, and redirect outputs (">>") to be added to the
+.txt file.
+You can also just manually change a few lines of interest, of course.
+
+Note: since input data are stored in Google Storage, you will need to set up
+the corresponding library.
+See http://developers.google.com/storage/docs/gspythonlibrary for details.
+"""
+
+__author__ = 'bensong@google.com (Ben Chen)'
+
+import bench_util
+import boto
+import cStringIO
+import optparse
+import re
+import shutil
+
+from oauth2_plugin import oauth2_plugin
+
+
+# Ratios for calculating suggested picture bench upper and lower bounds.
+BENCH_UB = 1.1 # Allow for 10% room for normal variance on the up side.
+BENCH_LB = 0.9
+
+# Further allow for a fixed amount of noise. This is especially useful for
+# benches of smaller absolute value. Keeping this value small will not affect
+# performance tunings.
+BENCH_ALLOWED_NOISE = 10
+
+# Name prefix for benchmark builders.
+BENCH_BUILDER_PREFIX = 'Perf-'
+
+# List of platforms to track. Feel free to change it to meet your needs.
+PLATFORMS = ['Perf-Mac10.8-MacMini4.1-GeForce320M-x86-Release',
+ 'Perf-Android-Nexus7-Tegra3-Arm7-Release',
+ 'Perf-Ubuntu12-ShuttleA-ATI5770-x86-Release',
+ 'Perf-Win7-ShuttleA-HD2000-x86-Release',
+ ]
+
+# Filter for configs of no interest. They are old config names replaced by more
+# specific ones.
+CONFIGS_TO_FILTER = ['gpu', 'raster']
+
+# Template for gsutil uri.
+GOOGLE_STORAGE_URI_SCHEME = 'gs'
+URI_BUCKET = 'chromium-skia-gm'
+
+# Constants for optparse.
+USAGE_STRING = 'USAGE: %s [options]'
+HOWTO_STRING = """
+Feel free to revise PLATFORMS for your own needs. The default is the most common
+combination that we care most about. Platforms that did not run bench_pictures
+or benchmain in the given revision range will not have corresponding outputs.
+Please check http://go/skpbench to choose a range that fits your needs.
+BENCH_UB, BENCH_LB and BENCH_ALLOWED_NOISE can be changed to expand or narrow
+the permitted bench ranges without triggering buidbot alerts.
+"""
+HELP_STRING = """
+Outputs expectation picture bench ranges for the latest revisions for the given
+revision range. For instance, --rev_range=6000:6000 will return only bench
+ranges for the bots that ran benches at rev 6000; --rev-range=6000:7000
+may have multiple bench data points for each bench configuration, and the code
+returns bench data for the latest revision of all available (closer to 7000).
+""" + HOWTO_STRING
+
+OPTION_REVISION_RANGE = '--rev-range'
+OPTION_REVISION_RANGE_SHORT = '-r'
+# Bench representation algorithm flag.
+OPTION_REPRESENTATION_ALG = '--algorithm'
+OPTION_REPRESENTATION_ALG_SHORT = '-a'
+# Bench type to examine. Either 'micro' or 'skp'.
+OPTION_BENCH_TYPE = '--bench-type'
+OPTION_BENCH_TYPE_SHORT = '-b'
+
+# List of valid bench types.
+BENCH_TYPES = ['micro', 'skp']
+# List of valid representation algorithms.
+REPRESENTATION_ALGS = ['avg', 'min', 'med', '25th']
+
+def OutputBenchExpectations(bench_type, rev_min, rev_max, representation_alg):
+ """Reads bench data from google storage, and outputs expectations.
+
+ Ignores data with revisions outside [rev_min, rev_max] integer range. For
+ bench data with multiple revisions, we use higher revisions to calculate
+ expected bench values.
+ bench_type is either 'micro' or 'skp', according to the flag '-b'.
+ Uses the provided representation_alg for calculating bench representations.
+ """
+ if bench_type not in BENCH_TYPES:
+ raise Exception('Not valid bench_type! (%s)' % BENCH_TYPES)
+ expectation_dic = {}
+ uri = boto.storage_uri(URI_BUCKET, GOOGLE_STORAGE_URI_SCHEME)
+ for obj in uri.get_bucket():
+ # Filters out non-bench files.
+ if ((not obj.name.startswith('perfdata/%s' % BENCH_BUILDER_PREFIX) and
+ not obj.name.startswith(
+ 'playback/perfdata/%s' % BENCH_BUILDER_PREFIX)) or
+ obj.name.find('_data') < 0):
+ continue
+ if ((bench_type == 'micro' and obj.name.find('_data_skp_') > 0) or
+ (bench_type == 'skp' and obj.name.find('_skp_') < 0)):
+ # Skips wrong bench type.
+ continue
+ # Ignores uninterested platforms.
+ platform = obj.name.split('/')[1]
+ if not platform.startswith(BENCH_BUILDER_PREFIX):
+ platform = obj.name.split('/')[2]
+ if not platform.startswith(BENCH_BUILDER_PREFIX):
+ continue # Ignores non-platform object
+ if platform not in PLATFORMS:
+ continue
+ # Filters by revision.
+ to_filter = True
+ for rev in range(rev_min, rev_max + 1):
+ if '_r%s_' % rev in obj.name:
+ to_filter = False
+ break
+ if to_filter:
+ continue
+ contents = cStringIO.StringIO()
+ obj.get_file(contents)
+ for point in bench_util.parse('', contents.getvalue().split('\n'),
+ representation_alg):
+ if point.config in CONFIGS_TO_FILTER:
+ continue
+
+ key = '%s_%s_%s,%s-%s' % (point.bench, point.config, point.time_type,
+ platform, representation_alg)
+ # It is fine to have later revisions overwrite earlier benches, since we
+ # only use the latest bench within revision range to set expectations.
+ expectation_dic[key] = point.time
+ keys = expectation_dic.keys()
+ keys.sort()
+ for key in keys:
+ bench_val = expectation_dic[key]
+ # Prints out expectation lines.
+ print '%s,%.3f,%.3f,%.3f' % (key, bench_val,
+ bench_val * BENCH_LB - BENCH_ALLOWED_NOISE,
+ bench_val * BENCH_UB + BENCH_ALLOWED_NOISE)
+
+def main():
+ """Parses flags and outputs expected Skia bench results."""
+ parser = optparse.OptionParser(USAGE_STRING % '%prog' + HELP_STRING)
+ parser.add_option(OPTION_REVISION_RANGE_SHORT, OPTION_REVISION_RANGE,
+ dest='rev_range',
+ help='(Mandatory) revision range separated by ":", e.g., 6000:6005')
+ parser.add_option(OPTION_BENCH_TYPE_SHORT, OPTION_BENCH_TYPE,
+ dest='bench_type', default='skp',
+ help=('Bench type, either "skp" or "micro". Default to "skp".'))
+ parser.add_option(OPTION_REPRESENTATION_ALG_SHORT, OPTION_REPRESENTATION_ALG,
+ dest='alg', default='25th',
+ help=('Bench representation algorithm. One of '
+ '%s. Default to "25th".' % str(REPRESENTATION_ALGS)))
+ (options, args) = parser.parse_args()
+ if options.rev_range:
+ range_match = re.search('(\d+)\:(\d+)', options.rev_range)
+ if not range_match:
+ parser.error('Wrong format for rev-range [%s]' % options.rev_range)
+ else:
+ rev_min = int(range_match.group(1))
+ rev_max = int(range_match.group(2))
+ OutputBenchExpectations(options.bench_type, rev_min, rev_max, options.alg)
+ else:
+ parser.error('Please provide mandatory flag %s' % OPTION_REVISION_RANGE)
+
+
+if '__main__' == __name__:
+ main()