Gitiles
Code Review Sign In
gerrit-public.fairphone.software / platform / external / skia / f53d0a998b59f922c7e04c94310a83df76bbc400 / . / bench / bench_analyze.py
blob: 74751cbc0f8aa2f17d4fff81034fc0a93ffe8a9b [file] [log] [blame]
bensong@google.com43859d52012-10-05 14:02:33 +0000[diff] [blame]1#!/usr/bin/env python
2# Copyright (c) 2012 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be found
4# in the LICENSE file.
5
6""" Analyze recent bench data from graphs, and output suggested ranges.
7
8This script reads and parses Skia benchmark values from the xhtml files
9generated by bench_graph_svg.py, and outputs an html file containing suggested
10bench ranges to use in bench_expectations.txt, with analytical plots.
11"""
12
13__author__ = 'bensong@google.com (Ben Chen)'
14
15import getopt
16import math
17import re
18import sys
19import urllib
20from datetime import datetime
21
22
23# Constants for calculating suggested bench ranges.
24WINDOW = 5 # Moving average sliding window size.
25# We use moving average as expected bench value, and calculate average Variance
26# of bench from the moving average. Set range to be [X_UB * Variance above
27# moving average, X_LB * Variance below moving average] of latest revision.
28X_UB = 4.0
29X_LB = 5.0
30
31# List of platforms.
32PLATFORMS = ['GalaxyNexus_4-1_Float_Release',
33 'Mac_Float_NoDebug_32',
34 'Mac_Float_NoDebug_64',
35 'MacMiniLion_Float_NoDebug_32',
36 'MacMiniLion_Float_NoDebug_64',
37 'Nexus7_4-1_Float_Release',
38 'Shuttle_Ubuntu12_ATI5770_Float_Release_64',
39 'Shuttle_Win7_Intel_Float_Release_32',
40 'Shuttle_Win7_Intel_Float_Release_64',
41 'Xoom_4-1_Float_Release'
42 ]
43
44# List of bench representation algorithms. Flag "-a" is chosen from the list.
45ALGS = ['25th', 'avg', 'med', 'min']
46
47# Regular expressions for parsing bench/revision values.
48HEIGHT_RE = 'height (\d+\.\d+) corresponds to bench value (\d+\.\d+).-->'
49REV_RE = '<rect id="(\d+)" x="(\d+\.\d+)" y="' # Revision corresponding x.
50LINE_RE = '<polyline id="(.*)".*points="(.*)"/>' # Bench value lines.
51
52# Bench graph url pattern.
53INPUT_URL_TEMPLATE = ('http://chromium-skia-gm.commondatastorage.googleapis.com'
54 '/graph-Skia_%s-2.xhtml')
55
56# Output HTML elements and templates.
57HTML_HEAD = ('<html><head><title>Skia Bench Expected Ranges</title>'
58 '<script type="text/javascript" src="https://skia.googlecode.com/'
59 'svn/buildbot/dygraph-combined.js"></script></head><body>Please '
60 'adjust values as appropriate and update benches to monitor in '
61 'bench/bench_expectations.txt.<br><br>')
62HTML_SUFFIX = '</body></html>'
63GRAPH_PREFIX = ('<br>%s<br><div id="%s" style="width:400px;height:200px"></div>'
64 '<script type="text/javascript">g%s=new Dygraph('
65 'document.getElementById("%s"),"rev,bench,alert\\n')
66GRAPH_SUFFIX = ('",{customBars: true,"alert":{strokeWidth:0.0,drawPoints:true,'
67 'pointSize:4,highlightCircleSize:6}});</script>')
68
69
70def Usage():
71 """Prints flag usage information."""
72 print '-a <representation-algorithm>: defaults to "25th".'
73 print ' If set, must be one of the list element in ALGS defined above.'
74 print '-b <bench-prefix>: prefix of matching bench names to analyze.'
75 print ' Only include benchmarks whose names start with this string.'
76 print ' Cannot be empty, because there are too many benches overall.'
77 print '-o <file>: html output filename. Output to STDOUT if not set.'
78 print '-p <platform-prefix>: prefix of platform names to analyze.'
79 print ' PLATFORMS has list of matching candidates. Matches all if not set.'
80
81def GetBenchValues(page, bench_prefix):
82 """Returns a dict of matching bench values from the given xhtml page.
83 Args:
84 page: substring used to construct the specific bench graph URL to fetch.
85 bench_prefix: only benches starting with this string will be included.
86
87 Returns:
88 a dict mapping benchmark name and revision combinations to bench values.
89 """
90 height = None
91 max_bench = None
92 height_scale = None
93 revisions = []
94 x_axes = [] # For calculating corresponding revisions.
95 val_dic = {} # dict[bench_name][revision] -> bench_value
96
97 lines = urllib.urlopen(INPUT_URL_TEMPLATE % page).readlines()
98 for line in lines:
99 height_match = re.search(HEIGHT_RE, line)
100 if height_match:
101 height = float(height_match.group(1))
102 max_bench = float(height_match.group(2))
103 height_scale = max_bench / height
104
105 rev_match = re.search(REV_RE, line)
106 if rev_match:
107 revisions.append(int(rev_match.group(1)))
108 x_axes.append(float(rev_match.group(2)))
109
110 line_match = re.search(LINE_RE, line)
111 if not line_match:
112 continue
113 bench = line_match.group(1)
114 bench = bench[:bench.find('_{')]
115 if not bench.startswith(bench_prefix):
116 continue
117 if bench not in val_dic:
118 val_dic[bench] = {}
119
120 vals = line_match.group(2).strip().split(' ')
121 if len(vals) < WINDOW: # Too few bench data points; skip.
122 continue
123 for val in vals:
124 x, y = [float(i) for i in val.split(',')]
125 for i in range(len(x_axes)):
126 if x <= x_axes[i]: # Found corresponding bench revision.
127 break
128 val_dic[bench][revisions[i]] = float(
129 '%.3f' % ((height - y) * height_scale))
130
131 return val_dic
132
133def CreateBenchOutput(page, bench, val_dic):
134 """Returns output for the given page and bench data in dict.
135 Args:
136 page: substring of bench graph webpage, to indicate the bench platform.
137 bench: name of the benchmark to process.
138 val_dic: dict[bench_name][revision] -> bench_value.
139
140 Returns:
141 string of html/javascript as part of the whole script output for the bench.
142 """
143 revs = val_dic[bench].keys()
144 revs.sort()
145 # Uses moving average to calculate expected bench variance, then sets
146 # expectations and ranges accordingly.
147 variances = []
148 moving_avgs = []
149 points = []
150 for rev in revs:
151 points.append(val_dic[bench][rev])
152 if len(points) >= WINDOW:
153 moving_avgs.append(sum(points[-WINDOW:]) / WINDOW)
154 variances.append(abs(points[-1] - moving_avgs[-1]))
155 else: # For the first WINDOW-1 points, cannot calculate moving average.
156 moving_avgs.append(points[-1]) # Uses actual value as estimates.
157 variances.append(0)
158 if len(variances) >= WINDOW:
159 for i in range(WINDOW - 1):
160 # Backfills estimated variances for the first WINDOW-1 points.
161 variances[i] = variances[WINDOW - 1]
162
163 avg_var = sum(variances) / len(variances)
164 for val in variances: # Removes outlier variances. Only does one iter.
165 if val > min(X_LB, X_UB) * avg_var:
166 variances.remove(val)
167 avg_var = sum(variances) / len(variances)
168
169 graph_id = '%s_%s' % (bench, page.replace('-', '_'))
170 expectations = '%s,%s,%.2f,%.2f,%.2f' % (bench, page, moving_avgs[-1],
171 moving_avgs[-1] - X_LB * avg_var,
172 moving_avgs[-1] + X_UB * avg_var)
173 out = GRAPH_PREFIX % (expectations, graph_id, graph_id, graph_id)
174 for i in range(len(revs)):
175 out += '%s,%.2f;%.2f;%.2f,' % (revs[i], moving_avgs[i] - X_LB * avg_var,
176 points[i], moving_avgs[i] + X_UB * avg_var)
177 if (points[i] > moving_avgs[i] + X_UB * avg_var or
178 points[i] < moving_avgs[i] - X_LB * avg_var): # Mark as alert point.
179 out += '%.2f;%.2f;%.2f\\n' % (points[i], points[i], points[i])
180 else:
181 out += 'NaN;NaN;NaN\\n'
182
183 return out
184
185def main():
186 """Parses flags and outputs analysis results."""
187 try:
188 opts, _ = getopt.getopt(sys.argv[1:], 'a:b:o:p:')
189 except getopt.GetoptError, err:
190 Usage()
191 sys.exit(2)
192
193 alg = '25th'
194 bench_prefix = None
195 out_file = None
196 platform_prefix = ''
197 for option, value in opts:
198 if option == '-a':
199 if value not in ALGS:
200 raise Exception('Invalid flag -a (%s): must be set to one of %s.' %
201 (value, str(ALGS)))
202 alg = value
203 elif option == '-b':
204 bench_prefix = value
205 elif option == '-o':
206 out_file = value
207 elif option == '-p':
208 platform_prefix = value
209 else:
210 Usage()
211 raise Exception('Error handling flags.')
212
213 if not bench_prefix:
214 raise Exception('Must provide nonempty Flag -b (bench name prefix).')
215
216 pages = []
217 for platform in PLATFORMS:
218 if not platform.startswith(platform_prefix):
219 continue
220 pages.append('%s-%s' % (platform, alg))
221
222 if not pages: # No matching platform found.
223 raise Exception('Flag -p (platform prefix: %s) does not match any of %s.' %
224 (platform_prefix, str(PLATFORMS)))
225
226 body = ''
227 # Iterates through bench graph xhtml pages for oututting matching benches.
228 for page in pages:
229 bench_value_dict = GetBenchValues(page, bench_prefix)
230 for bench in bench_value_dict:
231 body += CreateBenchOutput(page, bench, bench_value_dict) + GRAPH_SUFFIX
232
233 if not body:
234 raise Exception('No bench outputs. Most likely there are no matching bench'
235 ' prefix (%s) in Flags -b for platforms %s.\nPlease also '
236 'check if the bench graph URLs are valid at %s.' % (
237 bench_prefix, str(PLATFORMS), INPUT_URL_TEMPLATE))
238 if out_file:
239 f = open(out_file, 'w+')
240 f.write(HTML_HEAD + body + HTML_SUFFIX)
241 f.close()
242 else:
243 print HTML_HEAD + body + HTML_SUFFIX
244
245
246if '__main__' == __name__:
247 main()