compare_benchmarks.py

#!/usr/bin/python
#
# Copyright 2010 Google Inc. All Rights Reserved.

"""Script to compare ChromeOS benchmarks

Inputs:
    <perflab-output directory 1 - baseline> 
    <perflab-output directory 2 - results> 
    --csv - comma separated results

This script doesn't really know much about benchmarks. It simply looks for
similarly names directories and a results.txt file, and compares
the results and presents it, along with a geometric mean.

"""

__author__ = "bjanakiraman@google.com (Bhaskar Janakiraman)"

import glob
import math
import optparse
import os
import re
import sys

import image_chromeos
import run_tests
from utils import command_executer
from utils import logger

BENCHDIRS = "%s/default/default/*/gcc-4.4.3-glibc-2.11.1-grte-k8-opt/ref/*/results.txt"


# Common initializations
cmd_executer = command_executer.GetCommandExecuter()


def Usage(parser, message):
  print "ERROR: " + message
  parser.print_help()
  sys.exit(0)


def GetStats(file):
  """Return stats from file"""
  f = open(file, 'r')
  pairs = []
  for l in f:
    line = l.strip()
    # Look for match lines like the following:
    #       METRIC isolated TotalTime_ms (down, scalar) trial_run_0: ['1524.4']
    #       METRIC isolated isolated_walltime (down, scalar) trial_run_0: ['167.407445192']
    m = re.match(r"METRIC\s+isolated\s+(\S+).*\['(\d+(?:\.\d+)?)'\]", line)
    if not m:
      continue
    metric = m.group(1)
    if re.match(r"isolated_walltime", metric):
      continue

    value = float(m.group(2))
    pairs.append((metric, value))

  return dict(pairs)

def PrintDash(n):
  tmpstr = ''
  for i in range(n):
    tmpstr += '-'
  print tmpstr


def PrintHeaderCSV(hdr):
  tmpstr = ''
  for i in range(len(hdr)):
    if tmpstr != '':
      tmpstr += ','
    tmpstr += hdr[i]
  print tmpstr


def PrintHeader(hdr):
  tot_len = len(hdr)
  PrintDash(tot_len * 15)

  tmpstr = ''
  for i in range(len(hdr)):
    tmpstr += "%15.15s" % hdr[i]

  print tmpstr
  PrintDash(tot_len * 15)

def Main(argv):
  """Compare Benchmarks."""
  # Common initializations

  parser = optparse.OptionParser()
  parser.add_option("-c", "--csv", dest="csv_output",
                    action="store_true", default=False,
                    help="Output in csv form.")

  (options, args) = parser.parse_args(argv[1:])

  # validate args
  if len(args) != 2:
    Usage(parser, "Needs <baseline output dir> <results output dir>")

  base_dir = args[0]
  res_dir = args[1]

  # find res benchmarks that have results
  resbenches_glob = BENCHDIRS % res_dir
  resbenches = glob.glob(resbenches_glob)

  basebenches_glob = BENCHDIRS % base_dir
  basebenches = glob.glob(basebenches_glob)

  to_compare = []
  for resbench in resbenches:
    tmp = resbench.replace(res_dir, base_dir, 1)
    if tmp in basebenches:
      to_compare.append((resbench, tmp))

  for (resfile, basefile) in to_compare:
    stats = GetStats(resfile)
    basestats = GetStats(basefile)
    # Print a header
    # benchname (remove results.txt), basetime, restime, %speed-up
    hdr = []
    benchname = re.split('/', resfile)[-2:-1][0]
    benchname = benchname.replace('chromeos__', '',1)
    hdr.append(benchname)
    hdr.append('basetime')
    hdr.append('restime')
    hdr.append('%speed up')
    if options.csv_output:
      PrintHeaderCSV(hdr)
    else:
      PrintHeader(hdr)

    # For geomean computations
    prod = 1.0
    count = 0
    for key in stats.keys():
      if key in basestats.keys():
        # ignore very small values. 
        if stats[key] < 0.01:
          continue
        count = count + 1
        prod = prod * (stats[key]/basestats[key])
        speedup = (basestats[key] - stats[key])/basestats[key]
        speedup = speedup * 100.0
        if options.csv_output:
          print "%s,%f,%f,%f" % (key, basestats[key], stats[key],speedup)
        else:
          print "%15.15s%15.2f%15.2f%14.2f%%" % (key, basestats[key], stats[key],speedup)

    prod = math.exp(1.0/count * math.log(prod))
    prod = (1.0 - prod) * 100
    if options.csv_output:
      print "%s,,,%f" % ('Geomean', prod)
    else:
      print "%15.15s%15.15s%15.15s%14.2f%%" % ('Geomean', '', '',  prod)
      print
  return 0


if __name__ == "__main__":
  retval = Main(sys.argv)
  sys.exit(retval)