crosperf/suite_runner.py

# Copyright (c) 2013~2015 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""SuiteRunner defines the interface from crosperf to test script."""

from __future__ import print_function

import os
import time
import shlex

from cros_utils import command_executer
import test_flag

TEST_THAT_PATH = '/usr/bin/test_that'
CHROME_MOUNT_DIR = '/tmp/chrome_root'


def GetProfilerArgs(profiler_args):
  # Remove "--" from in front of profiler args.
  args_list = shlex.split(profiler_args)
  new_list = []
  for arg in args_list:
    if arg[0:2] == '--':
      arg = arg[2:]
    new_list.append(arg)
  args_list = new_list

  # Remove "perf_options=" from middle of profiler args.
  new_list = []
  for arg in args_list:
    idx = arg.find('perf_options=')
    if idx != -1:
      prefix = arg[0:idx]
      suffix = arg[idx + len('perf_options=') + 1:-1]
      new_arg = prefix + "'" + suffix + "'"
      new_list.append(new_arg)
    else:
      new_list.append(arg)
  args_list = new_list

  return ' '.join(args_list)


class SuiteRunner(object):
  """This defines the interface from crosperf to test script."""

  def __init__(self,
               logger_to_use=None,
               log_level='verbose',
               cmd_exec=None,
               cmd_term=None):
    self._logger = logger_to_use
    self.log_level = log_level
    self._ce = cmd_exec or command_executer.GetCommandExecuter(
        self._logger,
        log_level=self.log_level)
    self._ct = cmd_term or command_executer.CommandTerminator()

  def Run(self, machine, label, benchmark, test_args, profiler_args):
    for i in range(0, benchmark.retries + 1):
      self.PinGovernorExecutionFrequencies(machine, label.chromeos_root)
      if benchmark.suite == 'telemetry':
        ret_tup = self.Telemetry_Run(machine, label, benchmark, profiler_args)
      elif benchmark.suite == 'telemetry_Crosperf':
        ret_tup = self.Telemetry_Crosperf_Run(machine, label, benchmark,
                                              test_args, profiler_args)
      else:
        ret_tup = self.Test_That_Run(machine, label, benchmark, test_args,
                                     profiler_args)
      if ret_tup[0] != 0:
        self._logger.LogOutput('benchmark %s failed. Retries left: %s' %
                               (benchmark.name, benchmark.retries - i))
      elif i > 0:
        self._logger.LogOutput('benchmark %s succeded after %s retries' %
                               (benchmark.name, i))
        break
      else:
        self._logger.LogOutput('benchmark %s succeded on first try' %
                               benchmark.name)
        break
    return ret_tup

  def GetHighestStaticFrequency(self, machine_name, chromeos_root):
    """Gets the highest static frequency for the specified machine."""
    get_avail_freqs = ('cd /sys/devices/system/cpu/cpu0/cpufreq/; '
                       'if [[ -e scaling_available_frequencies ]]; then '
                       '  cat scaling_available_frequencies; '
                       'else '
                       '  cat scaling_max_freq ; '
                       'fi')
    ret, freqs_str, _ = self._ce.CrosRunCommandWOutput(
        get_avail_freqs,
        machine=machine_name,
        chromeos_root=chromeos_root)
    self._logger.LogFatalIf(ret, 'Could not get available frequencies '
                            'from machine: %s' % machine_name)
    freqs = freqs_str.split()
    # We need to make sure that the frequencies are sorted in decreasing
    # order
    freqs.sort(key=int, reverse=True)

    ## When there is no scaling_available_frequencies file,
    ## we have only 1 choice.
    if len(freqs) == 1:
      return freqs[0]
    # The dynamic frequency ends with a "1000". So, ignore it if found.
    if freqs[0].endswith('1000'):
      return freqs[1]
    else:
      return freqs[0]

  def PinGovernorExecutionFrequencies(self, machine_name, chromeos_root):
    """Set min and max frequencies to max static frequency."""
    highest_freq = self.GetHighestStaticFrequency(machine_name, chromeos_root)
    BASH_FOR = 'for f in {list}; do {body}; done'
    CPUFREQ_DIRS = '/sys/devices/system/cpu/cpu*/cpufreq/'
    change_max_freq = BASH_FOR.format(list=CPUFREQ_DIRS + 'scaling_max_freq',
                                      body='echo %s > $f' % highest_freq)
    change_min_freq = BASH_FOR.format(list=CPUFREQ_DIRS + 'scaling_min_freq',
                                      body='echo %s > $f' % highest_freq)
    change_perf_gov = BASH_FOR.format(list=CPUFREQ_DIRS + 'scaling_governor',
                                      body='echo performance > $f')
    if self.log_level == 'average':
      self._logger.LogOutput('Pinning governor execution frequencies for %s' %
                             machine_name)
    ret = self._ce.CrosRunCommand(' && '.join((
        'set -e ', change_max_freq, change_min_freq, change_perf_gov)),
                                  machine=machine_name,
                                  chromeos_root=chromeos_root)
    self._logger.LogFatalIf(ret, 'Could not pin frequencies on machine: %s' %
                            machine_name)

  def RebootMachine(self, machine_name, chromeos_root):
    command = 'reboot && exit'
    self._ce.CrosRunCommand(command,
                            machine=machine_name,
                            chromeos_root=chromeos_root)
    time.sleep(60)
    # Whenever we reboot the machine, we need to restore the governor settings.
    self.PinGovernorExecutionFrequencies(machine_name, chromeos_root)

  def Test_That_Run(self, machine, label, benchmark, test_args, profiler_args):
    """Run the test_that test.."""
    options = ''
    if label.board:
      options += ' --board=%s' % label.board
    if test_args:
      options += ' %s' % test_args
    if profiler_args:
      self._logger.LogFatal('test_that does not support profiler.')
    command = 'rm -rf /usr/local/autotest/results/*'
    self._ce.CrosRunCommand(command,
                            machine=machine,
                            chromeos_root=label.chromeos_root)

    # We do this because some tests leave the machine in weird states.
    # Rebooting between iterations has proven to help with this.
    self.RebootMachine(machine, label.chromeos_root)

    command = (
        ('%s --autotest_dir ~/trunk/src/third_party/autotest/files --fast '
         '%s %s %s') % (TEST_THAT_PATH, options, machine, benchmark.test_name))
    if self.log_level != 'verbose':
      self._logger.LogOutput('Running test.')
      self._logger.LogOutput('CMD: %s' % command)
    # Use --no-ns-pid so that cros_sdk does not create a different
    # process namespace and we can kill process created easily by
    # their process group.
    return self._ce.ChrootRunCommandWOutput(label.chromeos_root,
                                            command,
                                            command_terminator=self._ct,
                                            cros_sdk_options='--no-ns-pid')

  def RemoveTelemetryTempFile(self, machine, chromeos_root):
    filename = 'telemetry@%s' % machine
    fullname = os.path.join(chromeos_root, 'chroot', 'tmp', filename)
    if os.path.exists(fullname):
      os.remove(fullname)

  def Telemetry_Crosperf_Run(self, machine, label, benchmark, test_args,
                             profiler_args):
    if not os.path.isdir(label.chrome_src):
      self._logger.LogFatal('Cannot find chrome src dir to'
                            ' run telemetry: %s' % label.chrome_src)

    # Check for and remove temporary file that may have been left by
    # previous telemetry runs (and which might prevent this run from
    # working).
    self.RemoveTelemetryTempFile(machine, label.chromeos_root)

    # For telemetry runs, we can use the autotest copy from the source
    # location. No need to have one under /build/<board>.
    autotest_dir_arg = '--autotest_dir ~/trunk/src/third_party/autotest/files'

    profiler_args = GetProfilerArgs(profiler_args)
    fast_arg = ''
    if not profiler_args:
      # --fast works unless we are doing profiling (autotest limitation).
      # --fast avoids unnecessary copies of syslogs.
      fast_arg = '--fast'
    args_string = ''
    if test_args:
      # Strip double quotes off args (so we can wrap them in single
      # quotes, to pass through to Telemetry).
      if test_args[0] == '"' and test_args[-1] == '"':
        test_args = test_args[1:-1]
      args_string = "test_args='%s'" % test_args

    cmd = ('{} {} {} --board={} --args="{} run_local={} test={} '
           '{}" {} telemetry_Crosperf'.format(
               TEST_THAT_PATH, autotest_dir_arg, fast_arg, label.board,
               args_string, benchmark.run_local, benchmark.test_name,
               profiler_args, machine))

    # Use --no-ns-pid so that cros_sdk does not create a different
    # process namespace and we can kill process created easily by their
    # process group.
    chrome_root_options = ('--no-ns-pid '
                           '--chrome_root={} --chrome_root_mount={} '
                           "FEATURES=\"-usersandbox\" "
                           'CHROME_ROOT={}'.format(label.chrome_src,
                                                   CHROME_MOUNT_DIR,
                                                   CHROME_MOUNT_DIR))
    if self.log_level != 'verbose':
      self._logger.LogOutput('Running test.')
      self._logger.LogOutput('CMD: %s' % cmd)
    return self._ce.ChrootRunCommandWOutput(
        label.chromeos_root,
        cmd,
        command_terminator=self._ct,
        cros_sdk_options=chrome_root_options)

  def Telemetry_Run(self, machine, label, benchmark, profiler_args):
    telemetry_run_path = ''
    if not os.path.isdir(label.chrome_src):
      self._logger.LogFatal('Cannot find chrome src dir to' ' run telemetry.')
    else:
      telemetry_run_path = os.path.join(label.chrome_src, 'src/tools/perf')
      if not os.path.exists(telemetry_run_path):
        self._logger.LogFatal('Cannot find %s directory.' % telemetry_run_path)

    if profiler_args:
      self._logger.LogFatal('Telemetry does not support the perf profiler.')

    # Check for and remove temporary file that may have been left by
    # previous telemetry runs (and which might prevent this run from
    # working).
    if not test_flag.GetTestMode():
      self.RemoveTelemetryTempFile(machine, label.chromeos_root)

    rsa_key = os.path.join(
        label.chromeos_root,
        'src/scripts/mod_for_test_scripts/ssh_keys/testing_rsa')

    cmd = ('cd {0} && '
           './run_measurement '
           '--browser=cros-chrome '
           '--output-format=csv '
           '--remote={1} '
           '--identity {2} '
           '{3} {4}'.format(telemetry_run_path, machine, rsa_key,
                            benchmark.test_name, benchmark.test_args))
    if self.log_level != 'verbose':
      self._logger.LogOutput('Running test.')
      self._logger.LogOutput('CMD: %s' % cmd)
    return self._ce.RunCommandWOutput(cmd, print_to_console=False)

  def CommandTerminator(self):
    return self._ct

  def Terminate(self):
    self._ct.Terminate()


class MockSuiteRunner(object):
  """Mock suite runner for test."""

  def __init__(self):
    self._true = True

  def Run(self, *_args):
    if self._true:
      return [0, '', '']
    else:
      return [0, '', '']