compare_benchmarks.py

#!/usr/bin/python
#
# Copyright 2010 Google Inc. All Rights Reserved.
"""Script to compare ChromeOS benchmarks

Inputs:
    <perflab-output directory 1 - baseline> 
    <perflab-output directory 2 - results> 
    --csv - comma separated results

This script doesn't really know much about benchmarks. It simply looks for
similarly names directories and a results.txt file, and compares
the results and presents it, along with a geometric mean.

"""

__author__ = 'bjanakiraman@google.com (Bhaskar Janakiraman)'

import glob
import math
import optparse
import os
import re
import sys

import image_chromeos
import run_tests
from utils import command_executer
from utils import logger

BENCHDIRS = '%s/default/default/*/gcc-4.4.3-glibc-2.11.1-grte-k8-opt/ref/*/results.txt'

# Common initializations
cmd_executer = command_executer.GetCommandExecuter()


def Usage(parser, message):
  print 'ERROR: ' + message
  parser.print_help()
  sys.exit(0)


def GetStats(file):
  """Return stats from file"""
  f = open(file, 'r')
  pairs = []
  for l in f:
    line = l.strip()
    # Look for match lines like the following:
    #       METRIC isolated TotalTime_ms (down, scalar) trial_run_0: ['1524.4']
    #       METRIC isolated isolated_walltime (down, scalar) trial_run_0: ['167.407445192']
    m = re.match(r"METRIC\s+isolated\s+(\S+).*\['(\d+(?:\.\d+)?)'\]", line)
    if not m:
      continue
    metric = m.group(1)
    if re.match(r'isolated_walltime', metric):
      continue

    value = float(m.group(2))
    pairs.append((metric, value))

  return dict(pairs)


def PrintDash(n):
  tmpstr = ''
  for i in range(n):
    tmpstr += '-'
  print tmpstr


def PrintHeaderCSV(hdr):
  tmpstr = ''
  for i in range(len(hdr)):
    if tmpstr != '':
      tmpstr += ','
    tmpstr += hdr[i]
  print tmpstr


def PrintHeader(hdr):
  tot_len = len(hdr)
  PrintDash(tot_len * 15)

  tmpstr = ''
  for i in range(len(hdr)):
    tmpstr += '%15.15s' % hdr[i]

  print tmpstr
  PrintDash(tot_len * 15)


def Main(argv):
  """Compare Benchmarks."""
  # Common initializations

  parser = optparse.OptionParser()
  parser.add_option('-c',
                    '--csv',
                    dest='csv_output',
                    action='store_true',
                    default=False,
                    help='Output in csv form.')

  (options, args) = parser.parse_args(argv[1:])

  # validate args
  if len(args) != 2:
    Usage(parser, 'Needs <baseline output dir> <results output dir>')

  base_dir = args[0]
  res_dir = args[1]

  # find res benchmarks that have results
  resbenches_glob = BENCHDIRS % res_dir
  resbenches = glob.glob(resbenches_glob)

  basebenches_glob = BENCHDIRS % base_dir
  basebenches = glob.glob(basebenches_glob)

  to_compare = []
  for resbench in resbenches:
    tmp = resbench.replace(res_dir, base_dir, 1)
    if tmp in basebenches:
      to_compare.append((resbench, tmp))

  for (resfile, basefile) in to_compare:
    stats = GetStats(resfile)
    basestats = GetStats(basefile)
    # Print a header
    # benchname (remove results.txt), basetime, restime, %speed-up
    hdr = []
    benchname = re.split('/', resfile)[-2:-1][0]
    benchname = benchname.replace('chromeos__', '', 1)
    hdr.append(benchname)
    hdr.append('basetime')
    hdr.append('restime')
    hdr.append('%speed up')
    if options.csv_output:
      PrintHeaderCSV(hdr)
    else:
      PrintHeader(hdr)

    # For geomean computations
    prod = 1.0
    count = 0
    for key in stats.keys():
      if key in basestats.keys():
        # ignore very small values.
        if stats[key] < 0.01:
          continue
        count = count + 1
        prod = prod * (stats[key] / basestats[key])
        speedup = (basestats[key] - stats[key]) / basestats[key]
        speedup = speedup * 100.0
        if options.csv_output:
          print '%s,%f,%f,%f' % (key, basestats[key], stats[key], speedup)
        else:
          print '%15.15s%15.2f%15.2f%14.2f%%' % (key, basestats[key],
                                                 stats[key], speedup)

    prod = math.exp(1.0 / count * math.log(prod))
    prod = (1.0 - prod) * 100
    if options.csv_output:
      print '%s,,,%f' % ('Geomean', prod)
    else:
      print '%15.15s%15.15s%15.15s%14.2f%%' % ('Geomean', '', '', prod)
      print
  return 0


if __name__ == '__main__':
  retval = Main(sys.argv)
  sys.exit(retval)