cmtice | 4f0309d | 2014-06-15 13:36:05 -0700 | [diff] [blame] | 1 | #!/usr/bin/python |
| 2 | # |
| 3 | # Copyright 2014 Google Inc. All Rights Reserved. |
| 4 | |
| 5 | """Unittest for machine_manager.""" |
| 6 | import os.path |
| 7 | import time |
| 8 | import hashlib |
| 9 | |
| 10 | import mock |
| 11 | import unittest |
| 12 | |
| 13 | import suite_runner |
| 14 | import machine_manager |
| 15 | import image_checksummer |
| 16 | import label |
Caroline Tice | 7057cf6 | 2015-12-10 12:09:40 -0800 | [diff] [blame^] | 17 | import test_flag |
cmtice | 4f0309d | 2014-06-15 13:36:05 -0700 | [diff] [blame] | 18 | |
| 19 | from benchmark import Benchmark |
| 20 | from benchmark_run import MockBenchmarkRun |
| 21 | |
Yunlian Jiang | 0d1a9f3 | 2015-12-09 10:47:11 -0800 | [diff] [blame] | 22 | from cros_utils import command_executer |
| 23 | from cros_utils import logger |
cmtice | 4f0309d | 2014-06-15 13:36:05 -0700 | [diff] [blame] | 24 | |
| 25 | |
| 26 | class SuiteRunnerTest(unittest.TestCase): |
| 27 | |
| 28 | real_logger = logger.GetLogger() |
| 29 | |
| 30 | mock_cmd_exec = mock.Mock(spec=command_executer.CommandExecuter) |
| 31 | mock_cmd_term = mock.Mock(spec=command_executer.CommandTerminator) |
| 32 | mock_logger = mock.Mock(spec=logger.Logger) |
| 33 | mock_label = label.MockLabel("lumpy", "lumpy_chromeos_image", "/tmp/chromeos", |
| 34 | "lumpy", [ "lumpy1.cros", "lumpy.cros2" ], |
Caroline Tice | 7057cf6 | 2015-12-10 12:09:40 -0800 | [diff] [blame^] | 35 | "", "", False, "average", "gcc", "") |
cmtice | 4f0309d | 2014-06-15 13:36:05 -0700 | [diff] [blame] | 36 | telemetry_crosperf_bench = Benchmark("b1_test", # name |
| 37 | "octane", # test_name |
| 38 | "", # test_args |
| 39 | 3, # iterations |
| 40 | False, # rm_chroot_tmp |
| 41 | "record -e cycles", # perf_args |
| 42 | "telemetry_Crosperf", # suite |
| 43 | True) # show_all_results |
| 44 | |
| 45 | test_that_bench = Benchmark("b2_test", # name |
| 46 | "octane", # test_name |
| 47 | "", # test_args |
| 48 | 3, # iterations |
| 49 | False, # rm_chroot_tmp |
| 50 | "record -e cycles") # perf_args |
| 51 | |
| 52 | telemetry_bench = Benchmark("b3_test", # name |
| 53 | "octane", # test_name |
| 54 | "", # test_args |
| 55 | 3, # iterations |
| 56 | False, # rm_chroot_tmp |
| 57 | "record -e cycles", # perf_args |
| 58 | "telemetry", # suite |
| 59 | False) # show_all_results |
| 60 | |
| 61 | def setUp(self): |
| 62 | self.runner = suite_runner.SuiteRunner(self.mock_logger, "verbose", |
| 63 | self.mock_cmd_exec, self.mock_cmd_term) |
| 64 | |
| 65 | |
| 66 | def test_get_profiler_args(self): |
| 67 | input_str = ('--profiler=custom_perf --profiler_args=\'perf_options' |
| 68 | '="record -a -e cycles,instructions"\'') |
| 69 | output_str = ("profiler=custom_perf profiler_args='record -a -e " |
| 70 | "cycles,instructions'") |
| 71 | res = suite_runner.GetProfilerArgs(input_str) |
| 72 | self.assertEqual(res, output_str) |
| 73 | |
| 74 | def test_run(self): |
| 75 | |
| 76 | def reset(): |
| 77 | self.call_pin_governor = False |
| 78 | self.call_test_that_run = False |
| 79 | self.call_telemetry_run = False |
| 80 | self.call_telemetry_crosperf_run = False |
| 81 | self.pin_governor_args = [] |
| 82 | self.test_that_args = [] |
| 83 | self.telemetry_run_args = [] |
| 84 | self.telemetry_crosperf_args = [] |
| 85 | |
| 86 | |
| 87 | def FakePinGovernor(machine, chroot): |
| 88 | self.call_pin_governor = True |
| 89 | self.pin_governor_args = [machine, chroot] |
| 90 | |
| 91 | |
| 92 | def FakeTelemetryRun(machine, label, benchmark, profiler_args): |
| 93 | self.telemetry_run_args = [machine, label, benchmark, profiler_args] |
| 94 | self.call_telemetry_run = True |
| 95 | return "Ran FakeTelemetryRun" |
| 96 | |
| 97 | |
| 98 | def FakeTelemetryCrosperfRun(machine, label, benchmark, test_args, |
| 99 | profiler_args): |
| 100 | self.telemetry_crosperf_args = [machine, label, benchmark, test_args, |
| 101 | profiler_args] |
| 102 | self.call_telemetry_crosperf_run = True |
| 103 | return "Ran FakeTelemetryCrosperfRun" |
| 104 | |
| 105 | |
| 106 | def FakeTestThatRun(machine, label, benchmark, test_args, profiler_args): |
| 107 | self.test_that_args = [machine, label, benchmark, test_args, profiler_args] |
| 108 | self.call_test_that_run = True |
| 109 | return "Ran FakeTestThatRun" |
| 110 | |
| 111 | self.runner.PinGovernorExecutionFrequencies = FakePinGovernor |
| 112 | self.runner.Telemetry_Run = FakeTelemetryRun |
| 113 | self.runner.Telemetry_Crosperf_Run = FakeTelemetryCrosperfRun |
| 114 | self.runner.Test_That_Run = FakeTestThatRun |
| 115 | |
| 116 | machine = 'fake_machine' |
| 117 | test_args = '' |
| 118 | profiler_args = '' |
| 119 | reset() |
| 120 | res = self.runner.Run(machine, self.mock_label, self.telemetry_bench, |
| 121 | test_args, profiler_args) |
| 122 | self.assertTrue(self.call_pin_governor) |
| 123 | self.assertTrue(self.call_telemetry_run) |
| 124 | self.assertFalse(self.call_test_that_run) |
| 125 | self.assertFalse(self.call_telemetry_crosperf_run) |
| 126 | self.assertEqual(self.telemetry_run_args, |
| 127 | ['fake_machine', self.mock_label, self.telemetry_bench, '']) |
| 128 | |
| 129 | reset() |
| 130 | res = self.runner.Run(machine, self.mock_label, self.test_that_bench, |
| 131 | test_args, profiler_args) |
| 132 | self.assertTrue(self.call_pin_governor) |
| 133 | self.assertFalse(self.call_telemetry_run) |
| 134 | self.assertTrue(self.call_test_that_run) |
| 135 | self.assertFalse(self.call_telemetry_crosperf_run) |
| 136 | self.assertEqual(self.test_that_args, |
| 137 | ['fake_machine', self.mock_label, self.test_that_bench, '', |
| 138 | '']) |
| 139 | |
| 140 | reset() |
| 141 | res = self.runner.Run(machine, self.mock_label, self.telemetry_crosperf_bench, |
| 142 | test_args, profiler_args) |
| 143 | self.assertTrue(self.call_pin_governor) |
| 144 | self.assertFalse(self.call_telemetry_run) |
| 145 | self.assertFalse(self.call_test_that_run) |
| 146 | self.assertTrue(self.call_telemetry_crosperf_run) |
| 147 | self.assertEqual(self.telemetry_crosperf_args, |
| 148 | ['fake_machine', self.mock_label, |
| 149 | self.telemetry_crosperf_bench, '', '']) |
| 150 | |
| 151 | |
| 152 | |
| 153 | @mock.patch.object (command_executer.CommandExecuter, 'CrosRunCommand') |
| 154 | def test_get_highest_static_frequency(self, mock_cros_runcmd): |
| 155 | |
| 156 | self.mock_cmd_exec.CrosRunCommand = mock_cros_runcmd |
| 157 | mock_cros_runcmd.return_value = [ 0, '1666000 1333000 1000000', ''] |
| 158 | freq = self.runner.GetHighestStaticFrequency ('lumpy1.cros', '/tmp/chromeos') |
| 159 | self.assertEqual(freq, '1666000') |
| 160 | |
| 161 | mock_cros_runcmd.return_value = [ 0, '1333000', ''] |
| 162 | freq = self.runner.GetHighestStaticFrequency ('lumpy1.cros', '/tmp/chromeos') |
| 163 | self.assertEqual(freq, '1333000') |
| 164 | |
| 165 | mock_cros_runcmd.return_value = [ 0, '1661000 1333000 1000000', ''] |
| 166 | freq = self.runner.GetHighestStaticFrequency ('lumpy1.cros', '/tmp/chromeos') |
| 167 | self.assertEqual(freq, '1333000') |
| 168 | |
| 169 | |
| 170 | |
| 171 | @mock.patch.object (command_executer.CommandExecuter, 'CrosRunCommand') |
| 172 | def test_pin_governor_execution_frequencies(self, mock_cros_runcmd): |
| 173 | |
| 174 | def FakeGetHighestFreq(machine_name, chromeos_root): |
| 175 | return '1666000' |
| 176 | |
| 177 | self.mock_cmd_exec.CrosRunCommand = mock_cros_runcmd |
| 178 | self.runner.GetHighestStaticFrequency = FakeGetHighestFreq |
| 179 | self.runner.PinGovernorExecutionFrequencies('lumpy1.cros', '/tmp/chromeos') |
| 180 | self.assertEqual(mock_cros_runcmd.call_count, 1) |
| 181 | cmd = mock_cros_runcmd.call_args_list[0][0] |
| 182 | self.assertEqual (cmd, ('set -e && for f in /sys/devices/system/cpu/cpu*/cpufreq/scaling_max_freq; do echo 1666000 > $f; done && for f in /sys/devices/system/cpu/cpu*/cpufreq/scaling_min_freq; do echo 1666000 > $f; done && for f in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do echo performance > $f; done',)) |
| 183 | |
| 184 | |
| 185 | @mock.patch.object (time, 'sleep') |
| 186 | @mock.patch.object (command_executer.CommandExecuter, 'CrosRunCommand') |
| 187 | def test_reboot_machine(self, mock_cros_runcmd, mock_sleep): |
| 188 | |
| 189 | def FakePinGovernor(machine_name, chromeos_root): |
| 190 | pass |
| 191 | |
| 192 | self.mock_cmd_exec.CrosRunCommand = mock_cros_runcmd |
| 193 | self.runner.PinGovernorExecutionFrequencies = FakePinGovernor |
| 194 | self.runner.RebootMachine('lumpy1.cros', '/tmp/chromeos') |
| 195 | self.assertEqual(mock_cros_runcmd.call_count, 1) |
| 196 | self.assertEqual(mock_cros_runcmd.call_args_list[0][0], ('reboot && exit',)) |
| 197 | self.assertEqual(mock_sleep.call_count, 1) |
| 198 | self.assertEqual(mock_sleep.call_args_list[0][0], (60,)) |
| 199 | |
| 200 | |
| 201 | @mock.patch.object (command_executer.CommandExecuter, 'CrosRunCommand') |
| 202 | @mock.patch.object (command_executer.CommandExecuter, 'ChrootRunCommand') |
| 203 | def test_test_that_run(self, mock_chroot_runcmd, mock_cros_runcmd): |
| 204 | |
| 205 | def FakeRebootMachine (machine, chroot): |
| 206 | pass |
| 207 | |
| 208 | def FakeLogMsg (fd, termfd, msg, flush): |
| 209 | pass |
| 210 | |
| 211 | save_log_msg = self.real_logger._LogMsg |
| 212 | self.real_logger._LogMsg = FakeLogMsg |
| 213 | self.runner._logger = self.real_logger |
| 214 | self.runner.RebootMachine = FakeRebootMachine |
| 215 | |
| 216 | raised_exception = False |
| 217 | try: |
| 218 | self.runner.Test_That_Run('lumpy1.cros', self.mock_label, |
Caroline Tice | 7057cf6 | 2015-12-10 12:09:40 -0800 | [diff] [blame^] | 219 | self.test_that_bench, '', |
| 220 | 'record -a -e cycles') |
cmtice | 4f0309d | 2014-06-15 13:36:05 -0700 | [diff] [blame] | 221 | except: |
| 222 | raised_exception = True |
| 223 | self.assertTrue(raised_exception) |
| 224 | |
| 225 | mock_chroot_runcmd.return_value = 0 |
| 226 | self.mock_cmd_exec.ChrootRunCommand = mock_chroot_runcmd |
| 227 | self.mock_cmd_exec.CrosRunCommand = mock_cros_runcmd |
| 228 | res = self.runner.Test_That_Run ('lumpy1.cros', self.mock_label, |
Caroline Tice | 7057cf6 | 2015-12-10 12:09:40 -0800 | [diff] [blame^] | 229 | self.test_that_bench, '--iterations=2', |
| 230 | '') |
cmtice | 4f0309d | 2014-06-15 13:36:05 -0700 | [diff] [blame] | 231 | self.assertEqual(mock_cros_runcmd.call_count, 1) |
| 232 | self.assertEqual(mock_chroot_runcmd.call_count, 1) |
| 233 | self.assertEqual(res, 0) |
| 234 | self.assertEqual(mock_cros_runcmd.call_args_list[0][0], |
| 235 | ('rm -rf /usr/local/autotest/results/*',)) |
| 236 | args_list = mock_chroot_runcmd.call_args_list[0][0] |
| 237 | self.assertEqual(len(args_list), 4) |
| 238 | self.assertEqual(args_list[0], '/tmp/chromeos') |
Caroline Tice | 7057cf6 | 2015-12-10 12:09:40 -0800 | [diff] [blame^] | 239 | self.assertEqual(args_list[1], ('/usr/bin/test_that --autotest_dir ' |
| 240 | '~/trunk/src/third_party/autotest/files ' |
| 241 | '--fast --board=lumpy ' |
cmtice | 4f0309d | 2014-06-15 13:36:05 -0700 | [diff] [blame] | 242 | '--iterations=2 lumpy1.cros octane')) |
| 243 | self.assertTrue(args_list[2]) |
| 244 | self.assertEqual(args_list[3], self.mock_cmd_term) |
| 245 | |
| 246 | self.real_logger._LogMsg = save_log_msg |
| 247 | |
| 248 | |
| 249 | @mock.patch.object (os.path, 'isdir') |
| 250 | @mock.patch.object (command_executer.CommandExecuter, 'ChrootRunCommand') |
| 251 | def test_telemetry_crosperf_run(self, mock_chroot_runcmd, mock_isdir): |
| 252 | |
| 253 | mock_isdir.return_value = True |
| 254 | mock_chroot_runcmd.return_value = 0 |
| 255 | self.mock_cmd_exec.ChrootRunCommand = mock_chroot_runcmd |
| 256 | profiler_args = ('--profiler=custom_perf --profiler_args=\'perf_options' |
| 257 | '="record -a -e cycles,instructions"\'') |
| 258 | res = self.runner.Telemetry_Crosperf_Run ('lumpy1.cros', self.mock_label, |
| 259 | self.telemetry_crosperf_bench, |
| 260 | '', profiler_args) |
| 261 | self.assertEqual(res, 0) |
| 262 | self.assertEqual(mock_chroot_runcmd.call_count, 1) |
| 263 | args_list = mock_chroot_runcmd.call_args_list[0][0] |
| 264 | args_dict = mock_chroot_runcmd.call_args_list[0][1] |
| 265 | self.assertEqual(args_list[0], '/tmp/chromeos') |
| 266 | self.assertEqual(args_list[1], |
cmtice | 1d65210 | 2015-04-22 13:59:55 -0700 | [diff] [blame] | 267 | ('/usr/bin/test_that --autotest_dir ' |
| 268 | '~/trunk/src/third_party/autotest/files ' |
Caroline Tice | 7057cf6 | 2015-12-10 12:09:40 -0800 | [diff] [blame^] | 269 | ' --board=lumpy --args=" run_local=False test=octane ' |
cmtice | 4f0309d | 2014-06-15 13:36:05 -0700 | [diff] [blame] | 270 | 'profiler=custom_perf profiler_args=\'record -a -e ' |
| 271 | 'cycles,instructions\'" lumpy1.cros telemetry_Crosperf')) |
| 272 | self.assertEqual(args_dict['cros_sdk_options'], |
Caroline Tice | 7057cf6 | 2015-12-10 12:09:40 -0800 | [diff] [blame^] | 273 | ('--no-ns-pid --chrome_root= ' |
| 274 | '--chrome_root_mount=/tmp/chrome_root ' |
cmtice | 4f0309d | 2014-06-15 13:36:05 -0700 | [diff] [blame] | 275 | 'FEATURES="-usersandbox" CHROME_ROOT=/tmp/chrome_root')) |
| 276 | self.assertEqual(args_dict['command_terminator'], self.mock_cmd_term) |
| 277 | self.assertTrue(args_dict['return_output']) |
| 278 | self.assertEqual(len(args_dict), 3) |
| 279 | |
| 280 | |
| 281 | @mock.patch.object (os.path, 'isdir') |
| 282 | @mock.patch.object (os.path, 'exists') |
| 283 | @mock.patch.object (command_executer.CommandExecuter, 'RunCommand') |
| 284 | def test_telemetry_run(self, mock_runcmd, mock_exists, mock_isdir): |
| 285 | |
| 286 | def FakeLogMsg (fd, termfd, msg, flush): |
| 287 | pass |
| 288 | |
| 289 | save_log_msg = self.real_logger._LogMsg |
| 290 | self.real_logger._LogMsg = FakeLogMsg |
| 291 | mock_runcmd.return_value = 0 |
| 292 | |
| 293 | self.mock_cmd_exec.RunCommand = mock_runcmd |
| 294 | self.runner._logger = self.real_logger |
| 295 | |
| 296 | profiler_args = ('--profiler=custom_perf --profiler_args=\'perf_options' |
| 297 | '="record -a -e cycles,instructions"\'') |
| 298 | |
| 299 | raises_exception = False |
| 300 | mock_isdir.return_value = False |
| 301 | try: |
| 302 | self.runner.Telemetry_Run('lumpy1.cros', self.mock_label, |
| 303 | self.telemetry_bench, '') |
| 304 | except: |
| 305 | raises_exception = True |
| 306 | self.assertTrue(raises_exception) |
| 307 | |
| 308 | raises_exception = False |
| 309 | mock_isdir.return_value = True |
| 310 | mock_exists.return_value = False |
| 311 | try: |
| 312 | self.runner.Telemetry_Run('lumpy1.cros', self.mock_label, |
| 313 | self.telemetry_bench, '') |
| 314 | except: |
| 315 | raises_exception = True |
| 316 | self.assertTrue(raises_exception) |
| 317 | |
| 318 | raises_exception = False |
| 319 | mock_isdir.return_value = True |
| 320 | mock_exists.return_value = True |
| 321 | try: |
| 322 | self.runner.Telemetry_Run('lumpy1.cros', self.mock_label, |
| 323 | self.telemetry_bench, profiler_args) |
| 324 | except: |
| 325 | raises_exception = True |
| 326 | self.assertTrue(raises_exception) |
| 327 | |
Caroline Tice | 7057cf6 | 2015-12-10 12:09:40 -0800 | [diff] [blame^] | 328 | test_flag.SetTestMode(True) |
cmtice | 4f0309d | 2014-06-15 13:36:05 -0700 | [diff] [blame] | 329 | res = self.runner.Telemetry_Run('lumpy1.cros', self.mock_label, |
| 330 | self.telemetry_bench, '') |
| 331 | self.assertEqual(res, 0) |
| 332 | self.assertEqual(mock_runcmd.call_count, 1) |
| 333 | self.assertEqual(mock_runcmd.call_args_list[0][0], |
| 334 | (('cd src/tools/perf && ./run_measurement ' |
| 335 | '--browser=cros-chrome --output-format=csv ' |
| 336 | '--remote=lumpy1.cros --identity /tmp/chromeos/src/scripts' |
| 337 | '/mod_for_test_scripts/ssh_keys/testing_rsa octane '),)) |
| 338 | |
| 339 | self.real_logger._LogMsg = save_log_msg |
| 340 | |
| 341 | if __name__ == "__main__": |
| 342 | unittest.main() |