Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 1 | #!/usr/bin/python |
| 2 | |
| 3 | # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. |
| 4 | # Use of this source code is governed by a BSD-style license that can be |
| 5 | # found in the LICENSE file. |
| 6 | |
| 7 | import os |
| 8 | import time |
cmtice | e5a746f | 2013-11-25 14:57:10 -0500 | [diff] [blame] | 9 | import shlex |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 10 | |
| 11 | from utils import command_executer |
| 12 | |
Caroline Tice | 9277419 | 2013-09-10 16:29:18 -0700 | [diff] [blame] | 13 | TEST_THAT_PATH = '/usr/bin/test_that' |
| 14 | CHROME_MOUNT_DIR = '/tmp/chrome_root' |
| 15 | |
cmtice | 4f0309d | 2014-06-15 13:36:05 -0700 | [diff] [blame^] | 16 | def GetProfilerArgs (profiler_args): |
cmtice | e5a746f | 2013-11-25 14:57:10 -0500 | [diff] [blame] | 17 | # Remove "--" from in front of profiler args. |
| 18 | args_list = shlex.split(profiler_args) |
| 19 | new_list = [] |
| 20 | for arg in args_list: |
| 21 | if arg[0:2] == '--': |
| 22 | arg = arg[2:] |
| 23 | new_list.append(arg) |
| 24 | args_list = new_list |
| 25 | |
| 26 | # Remove "perf_options=" from middle of profiler args. |
| 27 | new_list = [] |
| 28 | for arg in args_list: |
| 29 | idx = arg.find("perf_options=") |
| 30 | if idx != -1: |
| 31 | prefix = arg[0:idx] |
| 32 | suffix = arg[idx + len("perf_options=") + 1 : -1] |
| 33 | new_arg = prefix + "'" + suffix + "'" |
| 34 | new_list.append(new_arg) |
| 35 | else: |
| 36 | new_list.append(arg) |
| 37 | args_list = new_list |
| 38 | |
| 39 | return " ".join(args_list) |
Caroline Tice | 9277419 | 2013-09-10 16:29:18 -0700 | [diff] [blame] | 40 | |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 41 | |
| 42 | class SuiteRunner(object): |
| 43 | """ This defines the interface from crosperf to test script. |
| 44 | """ |
Caroline Tice | 9277419 | 2013-09-10 16:29:18 -0700 | [diff] [blame] | 45 | |
cmtice | 4f0309d | 2014-06-15 13:36:05 -0700 | [diff] [blame^] | 46 | def __init__(self, logger_to_use=None, log_level="verbose", cmd_exec=None, |
| 47 | cmd_term=None): |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 48 | self._logger = logger_to_use |
cmtice | 1390924 | 2014-03-11 13:38:07 -0700 | [diff] [blame] | 49 | self.log_level = log_level |
cmtice | 4f0309d | 2014-06-15 13:36:05 -0700 | [diff] [blame^] | 50 | self._ce = cmd_exec or command_executer.GetCommandExecuter(self._logger, |
cmtice | 1390924 | 2014-03-11 13:38:07 -0700 | [diff] [blame] | 51 | log_level=self.log_level) |
cmtice | 4f0309d | 2014-06-15 13:36:05 -0700 | [diff] [blame^] | 52 | self._ct = cmd_term or command_executer.CommandTerminator() |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 53 | |
cmtice | e5a746f | 2013-11-25 14:57:10 -0500 | [diff] [blame] | 54 | def Run(self, machine, label, benchmark, test_args, profiler_args): |
Luis Lozano | 53c88e9 | 2013-10-08 15:15:48 -0700 | [diff] [blame] | 55 | self.PinGovernorExecutionFrequencies(machine, label.chromeos_root) |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 56 | if benchmark.suite == "telemetry": |
cmtice | 226e3e0 | 2014-04-27 22:28:42 -0700 | [diff] [blame] | 57 | return self.Telemetry_Run(machine, label, benchmark, profiler_args) |
Caroline Tice | 9277419 | 2013-09-10 16:29:18 -0700 | [diff] [blame] | 58 | elif benchmark.suite == "telemetry_Crosperf": |
cmtice | e5a746f | 2013-11-25 14:57:10 -0500 | [diff] [blame] | 59 | return self.Telemetry_Crosperf_Run(machine, label, benchmark, |
| 60 | test_args, profiler_args) |
cmtice | 98a5369 | 2014-04-16 14:48:47 -0700 | [diff] [blame] | 61 | else: |
cmtice | e5a746f | 2013-11-25 14:57:10 -0500 | [diff] [blame] | 62 | return self.Test_That_Run(machine, label, benchmark, test_args, |
| 63 | profiler_args) |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 64 | |
Luis Lozano | 53c88e9 | 2013-10-08 15:15:48 -0700 | [diff] [blame] | 65 | def GetHighestStaticFrequency(self, machine_name, chromeos_root): |
| 66 | """ Gets the highest static frequency for the specified machine |
| 67 | """ |
Han Shen | fd0b178 | 2014-02-12 15:13:01 -0800 | [diff] [blame] | 68 | get_avail_freqs = ("cd /sys/devices/system/cpu/cpu0/cpufreq/; " |
| 69 | "if [[ -e scaling_available_frequencies ]]; then " |
| 70 | " cat scaling_available_frequencies; " |
| 71 | "else " |
| 72 | " cat scaling_max_freq ; " |
| 73 | "fi") |
Luis Lozano | 53c88e9 | 2013-10-08 15:15:48 -0700 | [diff] [blame] | 74 | ret, freqs_str, _ = self._ce.CrosRunCommand( |
| 75 | get_avail_freqs, return_output=True, machine=machine_name, |
| 76 | chromeos_root=chromeos_root) |
| 77 | self._logger.LogFatalIf(ret, "Could not get available frequencies " |
| 78 | "from machine: %s" % machine_name) |
| 79 | freqs = freqs_str.split() |
Han Shen | fd0b178 | 2014-02-12 15:13:01 -0800 | [diff] [blame] | 80 | ## When there is no scaling_available_frequencies file, |
| 81 | ## we have only 1 choice. |
| 82 | if len(freqs) == 1: |
| 83 | return freqs[0] |
Luis Lozano | 53c88e9 | 2013-10-08 15:15:48 -0700 | [diff] [blame] | 84 | # The dynamic frequency ends with a "1000". So, ignore it if found. |
| 85 | if freqs[0].endswith("1000"): |
| 86 | return freqs[1] |
| 87 | else: |
| 88 | return freqs[0] |
| 89 | |
| 90 | def PinGovernorExecutionFrequencies(self, machine_name, chromeos_root): |
| 91 | """ Set min and max frequencies to max static frequency |
| 92 | """ |
| 93 | highest_freq = self.GetHighestStaticFrequency(machine_name, chromeos_root) |
| 94 | BASH_FOR = "for f in {list}; do {body}; done" |
| 95 | CPUFREQ_DIRS = "/sys/devices/system/cpu/cpu*/cpufreq/" |
| 96 | change_max_freq = BASH_FOR.format(list=CPUFREQ_DIRS + "scaling_max_freq", |
| 97 | body="echo %s > $f" % highest_freq) |
| 98 | change_min_freq = BASH_FOR.format(list=CPUFREQ_DIRS + "scaling_min_freq", |
| 99 | body="echo %s > $f" % highest_freq) |
| 100 | change_perf_gov = BASH_FOR.format(list=CPUFREQ_DIRS + "scaling_governor", |
| 101 | body="echo performance > $f") |
cmtice | 1390924 | 2014-03-11 13:38:07 -0700 | [diff] [blame] | 102 | if self.log_level == "average": |
| 103 | self._logger.LogOutput("Pinning governor execution frequencies for %s" |
| 104 | % machine_name) |
Luis Lozano | 53c88e9 | 2013-10-08 15:15:48 -0700 | [diff] [blame] | 105 | ret = self._ce.CrosRunCommand(" && ".join(("set -e ", |
| 106 | change_max_freq, |
| 107 | change_min_freq, |
| 108 | change_perf_gov)), |
| 109 | machine=machine_name, |
| 110 | chromeos_root=chromeos_root) |
| 111 | self._logger.LogFatalIf(ret, "Could not pin frequencies on machine: %s" |
| 112 | % machine_name) |
| 113 | |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 114 | def RebootMachine(self, machine_name, chromeos_root): |
Caroline Tice | 9277419 | 2013-09-10 16:29:18 -0700 | [diff] [blame] | 115 | command = "reboot && exit" |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 116 | self._ce.CrosRunCommand(command, machine=machine_name, |
| 117 | chromeos_root=chromeos_root) |
| 118 | time.sleep(60) |
Luis Lozano | 53c88e9 | 2013-10-08 15:15:48 -0700 | [diff] [blame] | 119 | # Whenever we reboot the machine, we need to restore the governor settings. |
| 120 | self.PinGovernorExecutionFrequencies(machine_name, chromeos_root) |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 121 | |
cmtice | e5a746f | 2013-11-25 14:57:10 -0500 | [diff] [blame] | 122 | def Test_That_Run(self, machine, label, benchmark, test_args, profiler_args): |
Caroline Tice | b47bff4 | 2013-08-19 15:59:02 -0700 | [diff] [blame] | 123 | """Run the test_that test..""" |
| 124 | options = "" |
| 125 | if label.board: |
| 126 | options += " --board=%s" % label.board |
| 127 | if test_args: |
| 128 | options += " %s" % test_args |
cmtice | e5a746f | 2013-11-25 14:57:10 -0500 | [diff] [blame] | 129 | if profiler_args: |
cmtice | 226e3e0 | 2014-04-27 22:28:42 -0700 | [diff] [blame] | 130 | self._logger.LogFatal("test_that does not support profiler.") |
Caroline Tice | b47bff4 | 2013-08-19 15:59:02 -0700 | [diff] [blame] | 131 | command = "rm -rf /usr/local/autotest/results/*" |
| 132 | self._ce.CrosRunCommand(command, machine=machine, username="root", |
| 133 | chromeos_root=label.chromeos_root) |
| 134 | |
cmtice | 98a5369 | 2014-04-16 14:48:47 -0700 | [diff] [blame] | 135 | # We do this because some tests leave the machine in weird states. |
Luis Lozano | 53c88e9 | 2013-10-08 15:15:48 -0700 | [diff] [blame] | 136 | # Rebooting between iterations has proven to help with this. |
Caroline Tice | b47bff4 | 2013-08-19 15:59:02 -0700 | [diff] [blame] | 137 | self.RebootMachine(machine, label.chromeos_root) |
| 138 | |
Caroline Tice | 9277419 | 2013-09-10 16:29:18 -0700 | [diff] [blame] | 139 | command = ("%s %s %s %s" % |
| 140 | (TEST_THAT_PATH, options, machine, benchmark.test_name)) |
cmtice | 1390924 | 2014-03-11 13:38:07 -0700 | [diff] [blame] | 141 | if self.log_level != "verbose": |
| 142 | self._logger.LogOutput("Running test.") |
| 143 | self._logger.LogOutput("CMD: %s" % command) |
Caroline Tice | b47bff4 | 2013-08-19 15:59:02 -0700 | [diff] [blame] | 144 | return self._ce.ChrootRunCommand(label.chromeos_root, |
| 145 | command, |
| 146 | True, |
| 147 | self._ct) |
| 148 | |
Caroline Tice | 9277419 | 2013-09-10 16:29:18 -0700 | [diff] [blame] | 149 | |
cmtice | e5a746f | 2013-11-25 14:57:10 -0500 | [diff] [blame] | 150 | def Telemetry_Crosperf_Run (self, machine, label, benchmark, test_args, |
| 151 | profiler_args): |
Caroline Tice | 9277419 | 2013-09-10 16:29:18 -0700 | [diff] [blame] | 152 | if not os.path.isdir(label.chrome_src): |
| 153 | self._logger.LogFatal("Cannot find chrome src dir to" |
cmtice | 5f3ccbd | 2013-11-20 16:36:04 -0800 | [diff] [blame] | 154 | " run telemetry: %s" % label.chrome_src) |
Caroline Tice | 9277419 | 2013-09-10 16:29:18 -0700 | [diff] [blame] | 155 | |
cmtice | 4f0309d | 2014-06-15 13:36:05 -0700 | [diff] [blame^] | 156 | profiler_args = GetProfilerArgs (profiler_args) |
Caroline Tice | 9277419 | 2013-09-10 16:29:18 -0700 | [diff] [blame] | 157 | chrome_root_options = "" |
| 158 | |
cmtice | 57f71b3 | 2014-04-29 13:43:13 -0700 | [diff] [blame] | 159 | chrome_root_options = (" --chrome_root={0} --chrome_root_mount={1} " |
| 160 | " FEATURES=\"-usersandbox\" " |
| 161 | "CHROME_ROOT={2}".format(label.chrome_src, |
| 162 | CHROME_MOUNT_DIR, |
| 163 | CHROME_MOUNT_DIR)) |
Caroline Tice | 9277419 | 2013-09-10 16:29:18 -0700 | [diff] [blame] | 164 | |
cmtice | e5a746f | 2013-11-25 14:57:10 -0500 | [diff] [blame] | 165 | args_string = "" |
| 166 | if test_args: |
| 167 | # Strip double quotes off args (so we can wrap them in single |
| 168 | # quotes, to pass through to Telemetry). |
| 169 | if test_args[0] == '"' and test_args[-1] == '"': |
| 170 | test_args = test_args[1:-1] |
| 171 | args_string = "test_args='%s'" % test_args |
| 172 | cmd = ('{0} --board={1} --args="{2} test={3} ' |
Caroline Tice | 9277419 | 2013-09-10 16:29:18 -0700 | [diff] [blame] | 173 | '{4}" {5} telemetry_Crosperf'.format(TEST_THAT_PATH, |
| 174 | label.board, |
cmtice | e5a746f | 2013-11-25 14:57:10 -0500 | [diff] [blame] | 175 | args_string, |
Caroline Tice | 9277419 | 2013-09-10 16:29:18 -0700 | [diff] [blame] | 176 | benchmark.test_name, |
| 177 | profiler_args, |
| 178 | machine)) |
cmtice | 1390924 | 2014-03-11 13:38:07 -0700 | [diff] [blame] | 179 | if self.log_level != "verbose": |
| 180 | self._logger.LogOutput("Running test.") |
| 181 | self._logger.LogOutput("CMD: %s" % cmd) |
Caroline Tice | 9277419 | 2013-09-10 16:29:18 -0700 | [diff] [blame] | 182 | return self._ce.ChrootRunCommand (label.chromeos_root, |
| 183 | cmd, |
| 184 | return_output=True, |
| 185 | command_terminator=self._ct, |
| 186 | cros_sdk_options=chrome_root_options) |
| 187 | |
| 188 | |
cmtice | 226e3e0 | 2014-04-27 22:28:42 -0700 | [diff] [blame] | 189 | def Telemetry_Run(self, machine, label, benchmark, profiler_args): |
cmtice | 98a5369 | 2014-04-16 14:48:47 -0700 | [diff] [blame] | 190 | telemetry_run_path = "" |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 191 | if not os.path.isdir(label.chrome_src): |
Caroline Tice | 6aa8528 | 2013-08-06 16:02:04 -0700 | [diff] [blame] | 192 | self._logger.LogFatal("Cannot find chrome src dir to" |
cmtice | 98a5369 | 2014-04-16 14:48:47 -0700 | [diff] [blame] | 193 | " run telemetry.") |
| 194 | else: |
| 195 | telemetry_run_path = os.path.join(label.chrome_src, "src/tools/perf") |
| 196 | if not os.path.exists(telemetry_run_path): |
| 197 | self._logger.LogFatal("Cannot find %s directory." % telemetry_run_path) |
| 198 | |
cmtice | 226e3e0 | 2014-04-27 22:28:42 -0700 | [diff] [blame] | 199 | if profiler_args: |
| 200 | self._logger.LogFatal("Telemetry does not support the perf profiler.") |
| 201 | |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 202 | rsa_key = os.path.join(label.chromeos_root, |
| 203 | "src/scripts/mod_for_test_scripts/ssh_keys/testing_rsa") |
| 204 | |
| 205 | cmd = ("cd {0} && " |
cmtice | 98a5369 | 2014-04-16 14:48:47 -0700 | [diff] [blame] | 206 | "./run_measurement " |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 207 | "--browser=cros-chrome " |
| 208 | "--output-format=csv " |
| 209 | "--remote={1} " |
| 210 | "--identity {2} " |
cmtice | 98a5369 | 2014-04-16 14:48:47 -0700 | [diff] [blame] | 211 | "{3} {4}".format(telemetry_run_path, machine, |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 212 | rsa_key, |
| 213 | benchmark.test_name, |
| 214 | benchmark.test_args)) |
cmtice | 1390924 | 2014-03-11 13:38:07 -0700 | [diff] [blame] | 215 | if self.log_level != "verbose": |
| 216 | self._logger.LogOutput("Running test.") |
| 217 | self._logger.LogOutput("CMD: %s" % cmd) |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 218 | return self._ce.RunCommand(cmd, return_output=True, |
| 219 | print_to_console=False) |
| 220 | |
| 221 | def Terminate(self): |
| 222 | self._ct.Terminate() |
| 223 | |
| 224 | |
| 225 | class MockSuiteRunner(object): |
| 226 | def __init__(self): |
Caroline Tice | 9277419 | 2013-09-10 16:29:18 -0700 | [diff] [blame] | 227 | self._true = True |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 228 | |
Caroline Tice | 9277419 | 2013-09-10 16:29:18 -0700 | [diff] [blame] | 229 | def Run(self, *_args): |
| 230 | if self._true: |
| 231 | return ["", "", 0] |
| 232 | else: |
| 233 | return ["", "", 0] |