Adds a script to help analyze bench ranges to add/change in bench/bench_expectations.txt
git-svn-id: http://skia.googlecode.com/svn/trunk@5824 2bbb7eff-a529-9590-31e7-b0007b416f81
diff --git a/bench/bench_analyze.py b/bench/bench_analyze.py
new file mode 100755
index 0000000..74751cb
--- /dev/null
+++ b/bench/bench_analyze.py
@@ -0,0 +1,247 @@
+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be found
+# in the LICENSE file.
+
+""" Analyze recent bench data from graphs, and output suggested ranges.
+
+This script reads and parses Skia benchmark values from the xhtml files
+generated by bench_graph_svg.py, and outputs an html file containing suggested
+bench ranges to use in bench_expectations.txt, with analytical plots.
+"""
+
+__author__ = 'bensong@google.com (Ben Chen)'
+
+import getopt
+import math
+import re
+import sys
+import urllib
+from datetime import datetime
+
+
+# Constants for calculating suggested bench ranges.
+WINDOW = 5 # Moving average sliding window size.
+# We use moving average as expected bench value, and calculate average Variance
+# of bench from the moving average. Set range to be [X_UB * Variance above
+# moving average, X_LB * Variance below moving average] of latest revision.
+X_UB = 4.0
+X_LB = 5.0
+
+# List of platforms.
+PLATFORMS = ['GalaxyNexus_4-1_Float_Release',
+ 'Mac_Float_NoDebug_32',
+ 'Mac_Float_NoDebug_64',
+ 'MacMiniLion_Float_NoDebug_32',
+ 'MacMiniLion_Float_NoDebug_64',
+ 'Nexus7_4-1_Float_Release',
+ 'Shuttle_Ubuntu12_ATI5770_Float_Release_64',
+ 'Shuttle_Win7_Intel_Float_Release_32',
+ 'Shuttle_Win7_Intel_Float_Release_64',
+ 'Xoom_4-1_Float_Release'
+ ]
+
+# List of bench representation algorithms. Flag "-a" is chosen from the list.
+ALGS = ['25th', 'avg', 'med', 'min']
+
+# Regular expressions for parsing bench/revision values.
+HEIGHT_RE = 'height (\d+\.\d+) corresponds to bench value (\d+\.\d+).-->'
+REV_RE = '<rect id="(\d+)" x="(\d+\.\d+)" y="' # Revision corresponding x.
+LINE_RE = '<polyline id="(.*)".*points="(.*)"/>' # Bench value lines.
+
+# Bench graph url pattern.
+INPUT_URL_TEMPLATE = ('http://chromium-skia-gm.commondatastorage.googleapis.com'
+ '/graph-Skia_%s-2.xhtml')
+
+# Output HTML elements and templates.
+HTML_HEAD = ('<html><head><title>Skia Bench Expected Ranges</title>'
+ '<script type="text/javascript" src="https://skia.googlecode.com/'
+ 'svn/buildbot/dygraph-combined.js"></script></head><body>Please '
+ 'adjust values as appropriate and update benches to monitor in '
+ 'bench/bench_expectations.txt.<br><br>')
+HTML_SUFFIX = '</body></html>'
+GRAPH_PREFIX = ('<br>%s<br><div id="%s" style="width:400px;height:200px"></div>'
+ '<script type="text/javascript">g%s=new Dygraph('
+ 'document.getElementById("%s"),"rev,bench,alert\\n')
+GRAPH_SUFFIX = ('",{customBars: true,"alert":{strokeWidth:0.0,drawPoints:true,'
+ 'pointSize:4,highlightCircleSize:6}});</script>')
+
+
+def Usage():
+ """Prints flag usage information."""
+ print '-a <representation-algorithm>: defaults to "25th".'
+ print ' If set, must be one of the list element in ALGS defined above.'
+ print '-b <bench-prefix>: prefix of matching bench names to analyze.'
+ print ' Only include benchmarks whose names start with this string.'
+ print ' Cannot be empty, because there are too many benches overall.'
+ print '-o <file>: html output filename. Output to STDOUT if not set.'
+ print '-p <platform-prefix>: prefix of platform names to analyze.'
+ print ' PLATFORMS has list of matching candidates. Matches all if not set.'
+
+def GetBenchValues(page, bench_prefix):
+ """Returns a dict of matching bench values from the given xhtml page.
+ Args:
+ page: substring used to construct the specific bench graph URL to fetch.
+ bench_prefix: only benches starting with this string will be included.
+
+ Returns:
+ a dict mapping benchmark name and revision combinations to bench values.
+ """
+ height = None
+ max_bench = None
+ height_scale = None
+ revisions = []
+ x_axes = [] # For calculating corresponding revisions.
+ val_dic = {} # dict[bench_name][revision] -> bench_value
+
+ lines = urllib.urlopen(INPUT_URL_TEMPLATE % page).readlines()
+ for line in lines:
+ height_match = re.search(HEIGHT_RE, line)
+ if height_match:
+ height = float(height_match.group(1))
+ max_bench = float(height_match.group(2))
+ height_scale = max_bench / height
+
+ rev_match = re.search(REV_RE, line)
+ if rev_match:
+ revisions.append(int(rev_match.group(1)))
+ x_axes.append(float(rev_match.group(2)))
+
+ line_match = re.search(LINE_RE, line)
+ if not line_match:
+ continue
+ bench = line_match.group(1)
+ bench = bench[:bench.find('_{')]
+ if not bench.startswith(bench_prefix):
+ continue
+ if bench not in val_dic:
+ val_dic[bench] = {}
+
+ vals = line_match.group(2).strip().split(' ')
+ if len(vals) < WINDOW: # Too few bench data points; skip.
+ continue
+ for val in vals:
+ x, y = [float(i) for i in val.split(',')]
+ for i in range(len(x_axes)):
+ if x <= x_axes[i]: # Found corresponding bench revision.
+ break
+ val_dic[bench][revisions[i]] = float(
+ '%.3f' % ((height - y) * height_scale))
+
+ return val_dic
+
+def CreateBenchOutput(page, bench, val_dic):
+ """Returns output for the given page and bench data in dict.
+ Args:
+ page: substring of bench graph webpage, to indicate the bench platform.
+ bench: name of the benchmark to process.
+ val_dic: dict[bench_name][revision] -> bench_value.
+
+ Returns:
+ string of html/javascript as part of the whole script output for the bench.
+ """
+ revs = val_dic[bench].keys()
+ revs.sort()
+ # Uses moving average to calculate expected bench variance, then sets
+ # expectations and ranges accordingly.
+ variances = []
+ moving_avgs = []
+ points = []
+ for rev in revs:
+ points.append(val_dic[bench][rev])
+ if len(points) >= WINDOW:
+ moving_avgs.append(sum(points[-WINDOW:]) / WINDOW)
+ variances.append(abs(points[-1] - moving_avgs[-1]))
+ else: # For the first WINDOW-1 points, cannot calculate moving average.
+ moving_avgs.append(points[-1]) # Uses actual value as estimates.
+ variances.append(0)
+ if len(variances) >= WINDOW:
+ for i in range(WINDOW - 1):
+ # Backfills estimated variances for the first WINDOW-1 points.
+ variances[i] = variances[WINDOW - 1]
+
+ avg_var = sum(variances) / len(variances)
+ for val in variances: # Removes outlier variances. Only does one iter.
+ if val > min(X_LB, X_UB) * avg_var:
+ variances.remove(val)
+ avg_var = sum(variances) / len(variances)
+
+ graph_id = '%s_%s' % (bench, page.replace('-', '_'))
+ expectations = '%s,%s,%.2f,%.2f,%.2f' % (bench, page, moving_avgs[-1],
+ moving_avgs[-1] - X_LB * avg_var,
+ moving_avgs[-1] + X_UB * avg_var)
+ out = GRAPH_PREFIX % (expectations, graph_id, graph_id, graph_id)
+ for i in range(len(revs)):
+ out += '%s,%.2f;%.2f;%.2f,' % (revs[i], moving_avgs[i] - X_LB * avg_var,
+ points[i], moving_avgs[i] + X_UB * avg_var)
+ if (points[i] > moving_avgs[i] + X_UB * avg_var or
+ points[i] < moving_avgs[i] - X_LB * avg_var): # Mark as alert point.
+ out += '%.2f;%.2f;%.2f\\n' % (points[i], points[i], points[i])
+ else:
+ out += 'NaN;NaN;NaN\\n'
+
+ return out
+
+def main():
+ """Parses flags and outputs analysis results."""
+ try:
+ opts, _ = getopt.getopt(sys.argv[1:], 'a:b:o:p:')
+ except getopt.GetoptError, err:
+ Usage()
+ sys.exit(2)
+
+ alg = '25th'
+ bench_prefix = None
+ out_file = None
+ platform_prefix = ''
+ for option, value in opts:
+ if option == '-a':
+ if value not in ALGS:
+ raise Exception('Invalid flag -a (%s): must be set to one of %s.' %
+ (value, str(ALGS)))
+ alg = value
+ elif option == '-b':
+ bench_prefix = value
+ elif option == '-o':
+ out_file = value
+ elif option == '-p':
+ platform_prefix = value
+ else:
+ Usage()
+ raise Exception('Error handling flags.')
+
+ if not bench_prefix:
+ raise Exception('Must provide nonempty Flag -b (bench name prefix).')
+
+ pages = []
+ for platform in PLATFORMS:
+ if not platform.startswith(platform_prefix):
+ continue
+ pages.append('%s-%s' % (platform, alg))
+
+ if not pages: # No matching platform found.
+ raise Exception('Flag -p (platform prefix: %s) does not match any of %s.' %
+ (platform_prefix, str(PLATFORMS)))
+
+ body = ''
+ # Iterates through bench graph xhtml pages for oututting matching benches.
+ for page in pages:
+ bench_value_dict = GetBenchValues(page, bench_prefix)
+ for bench in bench_value_dict:
+ body += CreateBenchOutput(page, bench, bench_value_dict) + GRAPH_SUFFIX
+
+ if not body:
+ raise Exception('No bench outputs. Most likely there are no matching bench'
+ ' prefix (%s) in Flags -b for platforms %s.\nPlease also '
+ 'check if the bench graph URLs are valid at %s.' % (
+ bench_prefix, str(PLATFORMS), INPUT_URL_TEMPLATE))
+ if out_file:
+ f = open(out_file, 'w+')
+ f.write(HTML_HEAD + body + HTML_SUFFIX)
+ f.close()
+ else:
+ print HTML_HEAD + body + HTML_SUFFIX
+
+
+if '__main__' == __name__:
+ main()