Change build_benchmarks to use --rebuild
Initial version of comparing benchmarks runs looking at the results.txt files.
PRESUBMIT=passed
R=raymes,asharif
DELTA=168 (168 added, 0 deleted, 0 changed)
OCL=49427-p2
RCL=49479-p2
RDATE=2011/03/05 22:02:50
P4 change: 42617806
diff --git a/v14/compare_benchmarks.py b/v14/compare_benchmarks.py
new file mode 100755
index 0000000..45ffa8d
--- /dev/null
+++ b/v14/compare_benchmarks.py
@@ -0,0 +1,168 @@
+#!/usr/bin/python2.6
+#
+# Copyright 2010 Google Inc. All Rights Reserved.
+
+"""Script to compare ChromeOS benchmarks
+
+Inputs:
+ <perflab-output directory 1 - baseline>
+ <perflab-output directory 2 - results>
+ --csv - comma separated results
+
+This script doesn't really know much about benchmarks. It simply looks for
+similarly names directories and a results.txt file, and compares
+the results and presents it, along with a geometric mean.
+
+"""
+
+__author__ = "bjanakiraman@google.com (Bhaskar Janakiraman)"
+
+import optparse
+import os
+import re
+import sys
+import glob
+import math
+import run_tests
+import image_chromeos
+from utils import command_executer
+from utils import utils
+from utils import logger
+
+BENCHDIRS = "%s/default/default/*/gcc-4.4.3-glibc-2.11.1-grte-k8-opt/ref/*/results.txt"
+
+
+# Common initializations
+cmd_executer = command_executer.GetCommandExecuter()
+
+
+def Usage(parser, message):
+ print "ERROR: " + message
+ parser.print_help()
+ sys.exit(0)
+
+
+def GetStats(file):
+ """Return stats from file"""
+ f = open(file, 'r')
+ pairs = []
+ for l in f:
+ line = l.strip()
+ # Look for match lines like the following:
+ # METRIC isolated TotalTime_ms (down, scalar) trial_run_0: ['1524.4']
+ # METRIC isolated isolated_walltime (down, scalar) trial_run_0: ['167.407445192']
+ m = re.match(r"METRIC\s+isolated\s+(\S+).*\['(\d+(?:\.\d+)?)'\]", line)
+ if not m:
+ continue
+ metric = m.group(1)
+ if re.match(r"isolated_walltime", metric):
+ continue
+
+ value = float(m.group(2))
+ pairs.append((metric, value))
+
+ return dict(pairs)
+
+def PrintDash(n):
+ tmpstr = ''
+ for i in range(n):
+ tmpstr += '-'
+ print tmpstr
+
+
+def PrintHeaderCSV(hdr):
+ tmpstr = ''
+ for i in range(len(hdr)):
+ if tmpstr != '':
+ tmpstr += ','
+ tmpstr += hdr[i]
+ print tmpstr
+
+
+def PrintHeader(hdr):
+ tot_len = len(hdr)
+ PrintDash(tot_len * 15)
+
+ tmpstr = ''
+ for i in range(len(hdr)):
+ tmpstr += "%15.15s" % hdr[i]
+
+ print tmpstr
+ PrintDash(tot_len * 15)
+
+def Main(argv):
+ """Compare Benchmarks."""
+ # Common initializations
+
+ parser = optparse.OptionParser()
+ parser.add_option("-c", "--csv", dest="csv_output",
+ action="store_true", default=False,
+ help="Output in csv form.")
+
+ (options, args) = parser.parse_args(argv[1:])
+
+ # validate args
+ if len(args) != 2:
+ Usage(parser, "Needs <baseline output dir> <results output dir>")
+
+ base_dir = args[0]
+ res_dir = args[1]
+
+ # find res benchmarks that have results
+ resbenches_glob = BENCHDIRS % res_dir
+ resbenches = glob.glob(resbenches_glob)
+
+ basebenches_glob = BENCHDIRS % base_dir
+ basebenches = glob.glob(basebenches_glob)
+
+ to_compare = []
+ for resbench in resbenches:
+ tmp = resbench.replace(res_dir, base_dir, 1)
+ if tmp in basebenches:
+ to_compare.append((resbench, tmp))
+
+ for (resfile, basefile) in to_compare:
+ stats = GetStats(resfile)
+ basestats = GetStats(basefile)
+ # Print a header
+ # benchname (remove results.txt), basetime, restime, %speed-up
+ hdr = []
+ benchname = re.split('/', resfile)[-2:-1][0]
+ benchname = benchname.replace('chromeos__', '',1)
+ hdr.append(benchname)
+ hdr.append('basetime')
+ hdr.append('restime')
+ hdr.append('%speed up')
+ if options.csv_output:
+ PrintHeaderCSV(hdr)
+ else:
+ PrintHeader(hdr)
+
+ # For geomean computations
+ prod = 1.0
+ count = 0
+ for key in stats.keys():
+ if key in basestats.keys():
+ # ignore very small values.
+ if stats[key] < 0.01:
+ continue
+ count = count + 1
+ prod = prod * (stats[key]/basestats[key])
+ speedup = (basestats[key] - stats[key])/basestats[key]
+ speedup = speedup * 100.0
+ if options.csv_output:
+ print "%s,%f,%f,%f" % (key, basestats[key], stats[key],speedup)
+ else:
+ print "%15.15s%15.2f%15.2f%14.2f%%" % (key, basestats[key], stats[key],speedup)
+
+ prod = math.exp(1.0/count * math.log(prod))
+ prod = (1.0 - prod) * 100
+ if options.csv_output:
+ print "%s,,,%f" % ('Geomean', prod)
+ else:
+ print "%15.15s%15.15s%15.15s%14.2f%%" % ('Geomean', '', '', prod)
+ print
+
+
+if __name__ == "__main__":
+ Main(sys.argv)