Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 1 | #!/usr/bin/python |
| 2 | |
| 3 | # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. |
| 4 | # Use of this source code is governed by a BSD-style license that can be |
| 5 | # found in the LICENSE file. |
| 6 | |
| 7 | import os |
| 8 | import time |
cmtice | e5a746f | 2013-11-25 14:57:10 -0500 | [diff] [blame] | 9 | import shlex |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 10 | |
| 11 | from utils import command_executer |
| 12 | |
Caroline Tice | 9277419 | 2013-09-10 16:29:18 -0700 | [diff] [blame] | 13 | TEST_THAT_PATH = '/usr/bin/test_that' |
| 14 | CHROME_MOUNT_DIR = '/tmp/chrome_root' |
| 15 | |
cmtice | 4f0309d | 2014-06-15 13:36:05 -0700 | [diff] [blame] | 16 | def GetProfilerArgs (profiler_args): |
cmtice | e5a746f | 2013-11-25 14:57:10 -0500 | [diff] [blame] | 17 | # Remove "--" from in front of profiler args. |
| 18 | args_list = shlex.split(profiler_args) |
| 19 | new_list = [] |
| 20 | for arg in args_list: |
| 21 | if arg[0:2] == '--': |
| 22 | arg = arg[2:] |
| 23 | new_list.append(arg) |
| 24 | args_list = new_list |
| 25 | |
| 26 | # Remove "perf_options=" from middle of profiler args. |
| 27 | new_list = [] |
| 28 | for arg in args_list: |
| 29 | idx = arg.find("perf_options=") |
| 30 | if idx != -1: |
| 31 | prefix = arg[0:idx] |
| 32 | suffix = arg[idx + len("perf_options=") + 1 : -1] |
| 33 | new_arg = prefix + "'" + suffix + "'" |
| 34 | new_list.append(new_arg) |
| 35 | else: |
| 36 | new_list.append(arg) |
| 37 | args_list = new_list |
| 38 | |
| 39 | return " ".join(args_list) |
Caroline Tice | 9277419 | 2013-09-10 16:29:18 -0700 | [diff] [blame] | 40 | |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 41 | |
| 42 | class SuiteRunner(object): |
| 43 | """ This defines the interface from crosperf to test script. |
| 44 | """ |
Caroline Tice | 9277419 | 2013-09-10 16:29:18 -0700 | [diff] [blame] | 45 | |
cmtice | 4f0309d | 2014-06-15 13:36:05 -0700 | [diff] [blame] | 46 | def __init__(self, logger_to_use=None, log_level="verbose", cmd_exec=None, |
| 47 | cmd_term=None): |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 48 | self._logger = logger_to_use |
cmtice | 1390924 | 2014-03-11 13:38:07 -0700 | [diff] [blame] | 49 | self.log_level = log_level |
cmtice | 4f0309d | 2014-06-15 13:36:05 -0700 | [diff] [blame] | 50 | self._ce = cmd_exec or command_executer.GetCommandExecuter(self._logger, |
cmtice | 1390924 | 2014-03-11 13:38:07 -0700 | [diff] [blame] | 51 | log_level=self.log_level) |
cmtice | 4f0309d | 2014-06-15 13:36:05 -0700 | [diff] [blame] | 52 | self._ct = cmd_term or command_executer.CommandTerminator() |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 53 | |
cmtice | e5a746f | 2013-11-25 14:57:10 -0500 | [diff] [blame] | 54 | def Run(self, machine, label, benchmark, test_args, profiler_args): |
Luis Lozano | df76222 | 2015-05-19 12:22:37 -0700 | [diff] [blame] | 55 | for i in range(0, benchmark.retries + 1): |
| 56 | self.PinGovernorExecutionFrequencies(machine, label.chromeos_root) |
| 57 | if benchmark.suite == "telemetry": |
| 58 | ret_tup = self.Telemetry_Run(machine, label, benchmark, profiler_args) |
| 59 | elif benchmark.suite == "telemetry_Crosperf": |
| 60 | ret_tup = self.Telemetry_Crosperf_Run(machine, label, benchmark, |
| 61 | test_args, profiler_args) |
| 62 | else: |
| 63 | ret_tup = self.Test_That_Run(machine, label, benchmark, test_args, |
| 64 | profiler_args) |
| 65 | if ret_tup[0] != 0: |
| 66 | self._logger.LogOutput("benchmark %s failed. Retries left: %s" |
| 67 | % (benchmark.name, benchmark.retries - i)) |
| 68 | elif i > 0: |
| 69 | self._logger.LogOutput("benchmark %s succeded after %s retries" |
| 70 | % (benchmark.name, i)) |
| 71 | break |
| 72 | else: |
| 73 | self._logger.LogOutput("benchmark %s succeded on first try" |
| 74 | % benchmark.name) |
| 75 | break |
| 76 | return ret_tup |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 77 | |
Luis Lozano | 53c88e9 | 2013-10-08 15:15:48 -0700 | [diff] [blame] | 78 | def GetHighestStaticFrequency(self, machine_name, chromeos_root): |
| 79 | """ Gets the highest static frequency for the specified machine |
| 80 | """ |
Han Shen | fd0b178 | 2014-02-12 15:13:01 -0800 | [diff] [blame] | 81 | get_avail_freqs = ("cd /sys/devices/system/cpu/cpu0/cpufreq/; " |
| 82 | "if [[ -e scaling_available_frequencies ]]; then " |
| 83 | " cat scaling_available_frequencies; " |
| 84 | "else " |
| 85 | " cat scaling_max_freq ; " |
| 86 | "fi") |
Luis Lozano | 53c88e9 | 2013-10-08 15:15:48 -0700 | [diff] [blame] | 87 | ret, freqs_str, _ = self._ce.CrosRunCommand( |
| 88 | get_avail_freqs, return_output=True, machine=machine_name, |
| 89 | chromeos_root=chromeos_root) |
| 90 | self._logger.LogFatalIf(ret, "Could not get available frequencies " |
| 91 | "from machine: %s" % machine_name) |
| 92 | freqs = freqs_str.split() |
Han Shen | fd0b178 | 2014-02-12 15:13:01 -0800 | [diff] [blame] | 93 | ## When there is no scaling_available_frequencies file, |
| 94 | ## we have only 1 choice. |
| 95 | if len(freqs) == 1: |
| 96 | return freqs[0] |
Luis Lozano | 53c88e9 | 2013-10-08 15:15:48 -0700 | [diff] [blame] | 97 | # The dynamic frequency ends with a "1000". So, ignore it if found. |
| 98 | if freqs[0].endswith("1000"): |
| 99 | return freqs[1] |
| 100 | else: |
| 101 | return freqs[0] |
| 102 | |
| 103 | def PinGovernorExecutionFrequencies(self, machine_name, chromeos_root): |
| 104 | """ Set min and max frequencies to max static frequency |
| 105 | """ |
| 106 | highest_freq = self.GetHighestStaticFrequency(machine_name, chromeos_root) |
| 107 | BASH_FOR = "for f in {list}; do {body}; done" |
| 108 | CPUFREQ_DIRS = "/sys/devices/system/cpu/cpu*/cpufreq/" |
| 109 | change_max_freq = BASH_FOR.format(list=CPUFREQ_DIRS + "scaling_max_freq", |
| 110 | body="echo %s > $f" % highest_freq) |
| 111 | change_min_freq = BASH_FOR.format(list=CPUFREQ_DIRS + "scaling_min_freq", |
| 112 | body="echo %s > $f" % highest_freq) |
| 113 | change_perf_gov = BASH_FOR.format(list=CPUFREQ_DIRS + "scaling_governor", |
| 114 | body="echo performance > $f") |
cmtice | 1390924 | 2014-03-11 13:38:07 -0700 | [diff] [blame] | 115 | if self.log_level == "average": |
| 116 | self._logger.LogOutput("Pinning governor execution frequencies for %s" |
| 117 | % machine_name) |
Luis Lozano | 53c88e9 | 2013-10-08 15:15:48 -0700 | [diff] [blame] | 118 | ret = self._ce.CrosRunCommand(" && ".join(("set -e ", |
| 119 | change_max_freq, |
| 120 | change_min_freq, |
| 121 | change_perf_gov)), |
| 122 | machine=machine_name, |
| 123 | chromeos_root=chromeos_root) |
| 124 | self._logger.LogFatalIf(ret, "Could not pin frequencies on machine: %s" |
| 125 | % machine_name) |
| 126 | |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 127 | def RebootMachine(self, machine_name, chromeos_root): |
Caroline Tice | 9277419 | 2013-09-10 16:29:18 -0700 | [diff] [blame] | 128 | command = "reboot && exit" |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 129 | self._ce.CrosRunCommand(command, machine=machine_name, |
| 130 | chromeos_root=chromeos_root) |
| 131 | time.sleep(60) |
Luis Lozano | 53c88e9 | 2013-10-08 15:15:48 -0700 | [diff] [blame] | 132 | # Whenever we reboot the machine, we need to restore the governor settings. |
| 133 | self.PinGovernorExecutionFrequencies(machine_name, chromeos_root) |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 134 | |
cmtice | e5a746f | 2013-11-25 14:57:10 -0500 | [diff] [blame] | 135 | def Test_That_Run(self, machine, label, benchmark, test_args, profiler_args): |
Caroline Tice | b47bff4 | 2013-08-19 15:59:02 -0700 | [diff] [blame] | 136 | """Run the test_that test..""" |
| 137 | options = "" |
| 138 | if label.board: |
| 139 | options += " --board=%s" % label.board |
| 140 | if test_args: |
| 141 | options += " %s" % test_args |
cmtice | e5a746f | 2013-11-25 14:57:10 -0500 | [diff] [blame] | 142 | if profiler_args: |
cmtice | 226e3e0 | 2014-04-27 22:28:42 -0700 | [diff] [blame] | 143 | self._logger.LogFatal("test_that does not support profiler.") |
Caroline Tice | b47bff4 | 2013-08-19 15:59:02 -0700 | [diff] [blame] | 144 | command = "rm -rf /usr/local/autotest/results/*" |
Luis Lozano | bd44704 | 2015-10-01 14:24:27 -0700 | [diff] [blame^] | 145 | self._ce.CrosRunCommand(command, machine=machine, |
Caroline Tice | b47bff4 | 2013-08-19 15:59:02 -0700 | [diff] [blame] | 146 | chromeos_root=label.chromeos_root) |
| 147 | |
cmtice | 98a5369 | 2014-04-16 14:48:47 -0700 | [diff] [blame] | 148 | # We do this because some tests leave the machine in weird states. |
Luis Lozano | 53c88e9 | 2013-10-08 15:15:48 -0700 | [diff] [blame] | 149 | # Rebooting between iterations has proven to help with this. |
Caroline Tice | b47bff4 | 2013-08-19 15:59:02 -0700 | [diff] [blame] | 150 | self.RebootMachine(machine, label.chromeos_root) |
| 151 | |
Caroline Tice | 9277419 | 2013-09-10 16:29:18 -0700 | [diff] [blame] | 152 | command = ("%s %s %s %s" % |
| 153 | (TEST_THAT_PATH, options, machine, benchmark.test_name)) |
cmtice | 1390924 | 2014-03-11 13:38:07 -0700 | [diff] [blame] | 154 | if self.log_level != "verbose": |
| 155 | self._logger.LogOutput("Running test.") |
| 156 | self._logger.LogOutput("CMD: %s" % command) |
Luis Lozano | 45b53c5 | 2015-09-30 11:36:27 -0700 | [diff] [blame] | 157 | # Use --no-ns-pid so that cros_sdk does not create a different |
| 158 | # process namespace and we can kill process created easily by |
| 159 | # their process group. |
Caroline Tice | b47bff4 | 2013-08-19 15:59:02 -0700 | [diff] [blame] | 160 | return self._ce.ChrootRunCommand(label.chromeos_root, |
| 161 | command, |
| 162 | True, |
Luis Lozano | 45b53c5 | 2015-09-30 11:36:27 -0700 | [diff] [blame] | 163 | self._ct, |
| 164 | cros_sdk_options="--no-ns-pid") |
Caroline Tice | b47bff4 | 2013-08-19 15:59:02 -0700 | [diff] [blame] | 165 | |
cmtice | 146058f | 2015-07-24 14:04:12 -0700 | [diff] [blame] | 166 | def RemoveTelemetryTempFile (self, machine, chromeos_root): |
| 167 | filename = "telemetry@%s" % machine |
| 168 | fullname = os.path.join (chromeos_root, |
| 169 | "chroot", |
| 170 | "tmp", |
| 171 | filename) |
| 172 | if os.path.exists(fullname): |
| 173 | os.remove(fullname) |
Caroline Tice | 9277419 | 2013-09-10 16:29:18 -0700 | [diff] [blame] | 174 | |
cmtice | e5a746f | 2013-11-25 14:57:10 -0500 | [diff] [blame] | 175 | def Telemetry_Crosperf_Run (self, machine, label, benchmark, test_args, |
| 176 | profiler_args): |
Caroline Tice | 9277419 | 2013-09-10 16:29:18 -0700 | [diff] [blame] | 177 | if not os.path.isdir(label.chrome_src): |
| 178 | self._logger.LogFatal("Cannot find chrome src dir to" |
cmtice | 5f3ccbd | 2013-11-20 16:36:04 -0800 | [diff] [blame] | 179 | " run telemetry: %s" % label.chrome_src) |
Caroline Tice | 9277419 | 2013-09-10 16:29:18 -0700 | [diff] [blame] | 180 | |
cmtice | 146058f | 2015-07-24 14:04:12 -0700 | [diff] [blame] | 181 | # Check for and remove temporary file that may have been left by |
| 182 | # previous telemetry runs (and which might prevent this run from |
| 183 | # working). |
| 184 | self.RemoveTelemetryTempFile (machine, label.chromeos_root) |
| 185 | |
Luis Lozano | 004bccb | 2015-03-20 18:19:58 -0700 | [diff] [blame] | 186 | # For telemetry runs, we can use the autotest copy from the source |
| 187 | # location. No need to have one under /build/<board>. |
| 188 | autotest_dir_arg = '--autotest_dir ~/trunk/src/third_party/autotest/files' |
| 189 | |
cmtice | 4f0309d | 2014-06-15 13:36:05 -0700 | [diff] [blame] | 190 | profiler_args = GetProfilerArgs (profiler_args) |
Luis Lozano | 5d7e21d | 2015-03-20 17:57:36 -0700 | [diff] [blame] | 191 | fast_arg = "" |
| 192 | if not profiler_args: |
| 193 | # --fast works unless we are doing profiling (autotest limitation). |
| 194 | # --fast avoids unnecessary copies of syslogs. |
| 195 | fast_arg = "--fast" |
cmtice | e5a746f | 2013-11-25 14:57:10 -0500 | [diff] [blame] | 196 | args_string = "" |
| 197 | if test_args: |
| 198 | # Strip double quotes off args (so we can wrap them in single |
| 199 | # quotes, to pass through to Telemetry). |
| 200 | if test_args[0] == '"' and test_args[-1] == '"': |
| 201 | test_args = test_args[1:-1] |
| 202 | args_string = "test_args='%s'" % test_args |
Luis Lozano | 004bccb | 2015-03-20 18:19:58 -0700 | [diff] [blame] | 203 | |
Ting-Yuan Huang | bc2d3d1 | 2015-07-02 13:09:03 +0800 | [diff] [blame] | 204 | cmd = ('{} {} {} --board={} --args="{} run_local={} test={} ' |
Luis Lozano | 5d7e21d | 2015-03-20 17:57:36 -0700 | [diff] [blame] | 205 | '{}" {} telemetry_Crosperf'.format(TEST_THAT_PATH, |
Luis Lozano | 004bccb | 2015-03-20 18:19:58 -0700 | [diff] [blame] | 206 | autotest_dir_arg, |
Luis Lozano | 5d7e21d | 2015-03-20 17:57:36 -0700 | [diff] [blame] | 207 | fast_arg, |
| 208 | label.board, |
| 209 | args_string, |
Ting-Yuan Huang | bc2d3d1 | 2015-07-02 13:09:03 +0800 | [diff] [blame] | 210 | benchmark.run_local, |
Luis Lozano | 5d7e21d | 2015-03-20 17:57:36 -0700 | [diff] [blame] | 211 | benchmark.test_name, |
| 212 | profiler_args, |
| 213 | machine)) |
| 214 | |
Luis Lozano | 45b53c5 | 2015-09-30 11:36:27 -0700 | [diff] [blame] | 215 | # Use --no-ns-pid so that cros_sdk does not create a different |
| 216 | # process namespace and we can kill process created easily by their |
| 217 | # process group. |
| 218 | chrome_root_options = ("--no-ns-pid " |
| 219 | "--chrome_root={} --chrome_root_mount={} " |
| 220 | "FEATURES=\"-usersandbox\" " |
Luis Lozano | 5d7e21d | 2015-03-20 17:57:36 -0700 | [diff] [blame] | 221 | "CHROME_ROOT={}".format(label.chrome_src, |
| 222 | CHROME_MOUNT_DIR, |
| 223 | CHROME_MOUNT_DIR)) |
cmtice | 1390924 | 2014-03-11 13:38:07 -0700 | [diff] [blame] | 224 | if self.log_level != "verbose": |
| 225 | self._logger.LogOutput("Running test.") |
| 226 | self._logger.LogOutput("CMD: %s" % cmd) |
Luis Lozano | 45b53c5 | 2015-09-30 11:36:27 -0700 | [diff] [blame] | 227 | return self._ce.ChrootRunCommand(label.chromeos_root, |
| 228 | cmd, |
| 229 | return_output=True, |
| 230 | command_terminator=self._ct, |
| 231 | cros_sdk_options=chrome_root_options) |
Caroline Tice | 9277419 | 2013-09-10 16:29:18 -0700 | [diff] [blame] | 232 | |
| 233 | |
cmtice | 226e3e0 | 2014-04-27 22:28:42 -0700 | [diff] [blame] | 234 | def Telemetry_Run(self, machine, label, benchmark, profiler_args): |
cmtice | 98a5369 | 2014-04-16 14:48:47 -0700 | [diff] [blame] | 235 | telemetry_run_path = "" |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 236 | if not os.path.isdir(label.chrome_src): |
Caroline Tice | 6aa8528 | 2013-08-06 16:02:04 -0700 | [diff] [blame] | 237 | self._logger.LogFatal("Cannot find chrome src dir to" |
cmtice | 98a5369 | 2014-04-16 14:48:47 -0700 | [diff] [blame] | 238 | " run telemetry.") |
| 239 | else: |
| 240 | telemetry_run_path = os.path.join(label.chrome_src, "src/tools/perf") |
| 241 | if not os.path.exists(telemetry_run_path): |
| 242 | self._logger.LogFatal("Cannot find %s directory." % telemetry_run_path) |
| 243 | |
cmtice | 226e3e0 | 2014-04-27 22:28:42 -0700 | [diff] [blame] | 244 | if profiler_args: |
| 245 | self._logger.LogFatal("Telemetry does not support the perf profiler.") |
| 246 | |
cmtice | 146058f | 2015-07-24 14:04:12 -0700 | [diff] [blame] | 247 | # Check for and remove temporary file that may have been left by |
| 248 | # previous telemetry runs (and which might prevent this run from |
| 249 | # working). |
| 250 | self.RemoveTelemetryTempFile (machine, label.chromeos_root) |
| 251 | |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 252 | rsa_key = os.path.join(label.chromeos_root, |
| 253 | "src/scripts/mod_for_test_scripts/ssh_keys/testing_rsa") |
| 254 | |
| 255 | cmd = ("cd {0} && " |
cmtice | 98a5369 | 2014-04-16 14:48:47 -0700 | [diff] [blame] | 256 | "./run_measurement " |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 257 | "--browser=cros-chrome " |
| 258 | "--output-format=csv " |
| 259 | "--remote={1} " |
| 260 | "--identity {2} " |
cmtice | 98a5369 | 2014-04-16 14:48:47 -0700 | [diff] [blame] | 261 | "{3} {4}".format(telemetry_run_path, machine, |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 262 | rsa_key, |
| 263 | benchmark.test_name, |
| 264 | benchmark.test_args)) |
cmtice | 1390924 | 2014-03-11 13:38:07 -0700 | [diff] [blame] | 265 | if self.log_level != "verbose": |
| 266 | self._logger.LogOutput("Running test.") |
| 267 | self._logger.LogOutput("CMD: %s" % cmd) |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 268 | return self._ce.RunCommand(cmd, return_output=True, |
| 269 | print_to_console=False) |
| 270 | |
| 271 | def Terminate(self): |
| 272 | self._ct.Terminate() |
| 273 | |
| 274 | |
| 275 | class MockSuiteRunner(object): |
| 276 | def __init__(self): |
Caroline Tice | 9277419 | 2013-09-10 16:29:18 -0700 | [diff] [blame] | 277 | self._true = True |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 278 | |
Caroline Tice | 9277419 | 2013-09-10 16:29:18 -0700 | [diff] [blame] | 279 | def Run(self, *_args): |
| 280 | if self._true: |
Luis Lozano | df76222 | 2015-05-19 12:22:37 -0700 | [diff] [blame] | 281 | return [0, "", ""] |
Caroline Tice | 9277419 | 2013-09-10 16:29:18 -0700 | [diff] [blame] | 282 | else: |
Luis Lozano | df76222 | 2015-05-19 12:22:37 -0700 | [diff] [blame] | 283 | return [0, "", ""] |