run_benchmarks.py

#!/usr/bin/python
#
# Copyright 2010 Google Inc. All Rights Reserved.
"""Script to run ChromeOS benchmarks

Inputs:
    chromeos_root
    board
    [chromeos/cpu/<benchname>|chromeos/browser/[pagecycler|sunspider]|chromeos/startup]
    hostname/IP of Chromeos machine

    chromeos/cpu/<benchname>
       - Read run script rules from bench.mk perflab-bin, copy benchmark to
       host, run
       and return results.

    chromeos/startup
       - Re-image host with image in perflab-bin
       - Call run_tests to run startup test, gather results.
       - Restore host back to what it was.

    chromeos/browser/*
       - Call build_chromebrowser to build image with new browser
       - Copy image to perflab-bin

"""

__author__ = 'bjanakiraman@google.com (Bhaskar Janakiraman)'

import optparse
import os
import re
import sys

import image_chromeos
import run_tests
from utils import command_executer
from utils import logger

KNOWN_BENCHMARKS = [
    'chromeos/startup', 'chromeos/browser/pagecycler',
    'chromeos/browser/sunspider', 'chromeos/browser/v8bench',
    'chromeos/cpu/bikjmp'
]

name_map = {
    'pagecycler': 'Page',
    'sunspider': 'SunSpider',
    'v8bench': 'V8Bench',
    'startup': 'BootPerfServer'
}

# Run command template

# Common initializations
cmd_executer = command_executer.GetCommandExecuter()


def Usage(parser, message):
  print 'ERROR: ' + message
  parser.print_help()
  sys.exit(0)


def RunBrowserBenchmark(chromeos_root, board, bench, workdir, machine):
  """Run browser benchmarks.

  Args:
    chromeos_root: ChromeOS src dir
    board: Board being tested
    bench: Name of benchmark (chromeos/browser/*)
    workdir: Directory containing benchmark directory
    machine: name of chromeos machine
  """
  benchname = re.split('/', bench)[2]
  benchdir = '%s/%s' % (workdir, benchname)
  benchname = name_map[benchname]
  retval = run_tests.RunRemoteTests(chromeos_root, machine, board, benchname)
  return retval


def RunStartupBenchmark(chromeos_root, board, bench, workdir, machine):
  """Run browser benchmarks.

  Args:
    chromeos_root: ChromeOS src dir
    board: Board being tested
    bench: Name of benchmark (chromeos/browser/*)
    workdir: Directory containing benchmark directory
    machine: name of chromeos machine
  """
  benchname = 'startup'
  benchdir = '%s/%s' % (workdir, benchname)
  benchname = name_map[benchname]
  retval = run_tests.RunRemoteTests(chromeos_root, machine, board, benchname)
  return retval


def RunCpuBenchmark(chromeos_root, bench, workdir, machine):
  """Run CPU benchmark.

  Args:
    bench: Name of benchmark
    workdir: directory containing benchmark directory
    machine: name of chromeos machine

  Returns:
    status: 0 on success
  """

  benchname = re.split('/', bench)[2]
  benchdir = '%s/%s' % (workdir, benchname)

  # Delete any existing run directories on machine.
  # Since this has exclusive access to the machine,
  # we do not worry about duplicates.
  args = 'rm -rf /tmp/%s' % benchname
  retval = cmd_executer.CrosRunCommand(args,
                                       chromeos_root=chromeos_root,
                                       machine=machine)
  if retval:
    return retval

  # Copy benchmark directory.
  retval = cmd_executer.CopyFiles(benchdir,
                                  '/tmp/' + benchname,
                                  chromeos_root=chromeos_root,
                                  dest_machine=machine,
                                  dest_cros=True)
  if retval:
    return retval

  # Parse bench.mk to extract run flags.

  benchmk_file = open('%s/bench.mk' % benchdir, 'r')
  for line in benchmk_file:
    line.rstrip()
    if re.match('^run_cmd', line):
      line = re.sub('^run_cmd.*\${PERFLAB_PATH}', './out', line)
      line = re.sub('\${PERFLAB_INPUT}', './data', line)
      run_cmd = line
      break

  # Execute on remote machine
  # Capture output and process it.
  sshargs = "'cd /tmp/%s;" % benchname
  sshargs += "time -p %s'" % run_cmd
  cmd_executer.CrosRunCommand(sshargs,
                              chromeos_root=chromeos_root,
                              machine=machine)

  return retval


def Main(argv):
  """Build ChromeOS."""
  # Common initializations

  parser = optparse.OptionParser()
  parser.add_option('-c',
                    '--chromeos_root',
                    dest='chromeos_root',
                    help='Target directory for ChromeOS installation.')
  parser.add_option('-m',
                    '--machine',
                    dest='machine',
                    help='The chromeos host machine.')
  parser.add_option('--workdir',
                    dest='workdir',
                    default='./perflab-bin',
                    help='Work directory for perflab outputs.')
  parser.add_option('--board',
                    dest='board',
                    help='ChromeOS target board, e.g. x86-generic')

  (options, args) = parser.parse_args(argv[1:])

  # validate args
  for arg in args:
    if arg not in KNOWN_BENCHMARKS:
      logger.GetLogger().LogFatal('Bad benchmark %s specified' % arg)

  if options.chromeos_root is None:
    Usage(parser, '--chromeos_root must be set')

  if options.board is None:
    Usage(parser, '--board must be set')

  if options.machine is None:
    Usage(parser, '--machine must be set')

  found_err = 0
  retval = 0
  for arg in args:
    # CPU benchmarks
    comps = re.split('/', arg)
    if re.match('chromeos/cpu', arg):
      benchname = comps[2]
      print 'RUNNING %s' % benchname
      retval = RunCpuBenchmark(options.chromeos_root, arg, options.workdir,
                               options.machine)
      if not found_err:
        found_err = retval
    elif re.match('chromeos/startup', arg):
      benchname = comps[1]
      image_args = [
          os.path.dirname(os.path.abspath(__file__)) + '/image_chromeos.py',
          '--chromeos_root=' + options.chromeos_root,
          '--remote=' + options.machine, '--image=' + options.workdir + '/' +
          benchname + '/chromiumos_image.bin'
      ]
      logger.GetLogger().LogOutput('Reimaging machine %s' % options.machine)
      image_chromeos.Main(image_args)

      logger.GetLogger().LogOutput('Running %s' % arg)
      retval = RunStartupBenchmark(options.chromeos_root, options.board, arg,
                                   options.workdir, options.machine)
      if not found_err:
        found_err = retval
    elif re.match('chromeos/browser', arg):
      benchname = comps[2]
      image_args = [
          os.path.dirname(os.path.abspath(__file__)) + '/image_chromeos.py',
          '--chromeos_root=' + options.chromeos_root,
          '--remote=' + options.machine, '--image=' + options.workdir + '/' +
          benchname + '/chromiumos_image.bin'
      ]
      logger.GetLogger().LogOutput('Reimaging machine %s' % options.machine)
      image_chromeos.Main(image_args)

      logger.GetLogger().LogOutput('Running %s' % arg)
      retval = RunBrowserBenchmark(options.chromeos_root, options.board, arg,
                                   options.workdir, options.machine)
      if not found_err:
        found_err = retval

  return found_err


if __name__ == '__main__':
  retval = Main(sys.argv)
  sys.exit(retval)