Simran Basi | 14622bb | 2015-11-25 13:23:40 -0800 | [diff] [blame] | 1 | #!/usr/bin/python |
| 2 | # Copyright 2015 The Chromium OS Authors. All rights reserved. |
| 3 | # Use of this source code is governed by a BSD-style license that can be |
| 4 | # found in the LICENSE file. |
| 5 | # pylint: disable-msg=C0111 |
| 6 | |
| 7 | import os, unittest |
| 8 | import mox |
| 9 | import common |
| 10 | import subprocess |
Jacob Kopczynski | 2cefa1f | 2018-01-10 17:25:38 -0800 | [diff] [blame] | 11 | import shutil |
| 12 | import tempfile |
Simran Basi | 14622bb | 2015-11-25 13:23:40 -0800 | [diff] [blame] | 13 | import types |
Jacob Kopczynski | 2cefa1f | 2018-01-10 17:25:38 -0800 | [diff] [blame] | 14 | from autotest_lib.client.common_lib import control_data |
Simran Basi | 14622bb | 2015-11-25 13:23:40 -0800 | [diff] [blame] | 15 | from autotest_lib.server import utils |
| 16 | from autotest_lib.server.cros.dynamic_suite import constants |
Jacob Kopczynski | 2cefa1f | 2018-01-10 17:25:38 -0800 | [diff] [blame] | 17 | from autotest_lib.server.cros.dynamic_suite import control_file_getter |
| 18 | from autotest_lib.server.cros.dynamic_suite import suite as suite_module |
Simran Basi | 14622bb | 2015-11-25 13:23:40 -0800 | [diff] [blame] | 19 | from autotest_lib.site_utils import test_runner_utils |
| 20 | |
| 21 | |
| 22 | class StartsWithList(mox.Comparator): |
| 23 | def __init__(self, start_of_list): |
| 24 | """Mox comparator which returns True if the argument |
| 25 | to the mocked function is a list that begins with the elements |
| 26 | in start_of_list. |
| 27 | """ |
| 28 | self._lhs = start_of_list |
| 29 | |
| 30 | def equals(self, rhs): |
| 31 | if len(rhs)<len(self._lhs): |
| 32 | return False |
| 33 | for (x, y) in zip(self._lhs, rhs): |
| 34 | if x != y: |
| 35 | return False |
| 36 | return True |
| 37 | |
| 38 | |
| 39 | class ContainsSublist(mox.Comparator): |
| 40 | def __init__(self, sublist): |
| 41 | """Mox comparator which returns True if the argument |
| 42 | to the mocked function is a list that contains sublist |
| 43 | as a sub-list. |
| 44 | """ |
| 45 | self._sublist = sublist |
| 46 | |
| 47 | def equals(self, rhs): |
| 48 | n = len(self._sublist) |
| 49 | if len(rhs)<n: |
| 50 | return False |
| 51 | return any((self._sublist == rhs[i:i+n]) |
| 52 | for i in xrange(len(rhs) - n + 1)) |
| 53 | |
Jacob Kopczynski | 2cefa1f | 2018-01-10 17:25:38 -0800 | [diff] [blame] | 54 | class DummyJob(object): |
| 55 | def __init__(self, id=1): |
| 56 | self.id = id |
Simran Basi | 14622bb | 2015-11-25 13:23:40 -0800 | [diff] [blame] | 57 | |
Jacob Kopczynski | 2cefa1f | 2018-01-10 17:25:38 -0800 | [diff] [blame] | 58 | class TestRunnerUnittests(mox.MoxTestBase): |
| 59 | |
| 60 | def setUp(self): |
| 61 | mox.MoxTestBase.setUp(self) |
| 62 | |
Simran Basi | 14622bb | 2015-11-25 13:23:40 -0800 | [diff] [blame] | 63 | |
| 64 | def test_fetch_local_suite(self): |
| 65 | # Deferred until fetch_local_suite knows about non-local builds. |
| 66 | pass |
| 67 | |
Jacob Kopczynski | 2cefa1f | 2018-01-10 17:25:38 -0800 | [diff] [blame] | 68 | |
| 69 | def test_handle_local_result_for_good_test(self): |
| 70 | getter = self.mox.CreateMock(control_file_getter.DevServerGetter) |
| 71 | getter.get_control_file_list(suite_name=mox.IgnoreArg()).AndReturn([]) |
| 72 | job = DummyJob() |
| 73 | test = self.mox.CreateMock(control_data.ControlData) |
| 74 | test.job_retries = 5 |
| 75 | self.mox.StubOutWithMock(test_runner_utils.LocalSuite, |
| 76 | '_retry_local_result') |
| 77 | self.mox.ReplayAll() |
| 78 | suite = test_runner_utils.LocalSuite([], "tag", [], None, getter, |
| 79 | job_retry=True) |
| 80 | suite._retry_handler = suite_module.RetryHandler({job.id: test}) |
| 81 | #No calls, should not be retried |
| 82 | directory = tempfile.mkdtemp() |
| 83 | try: |
| 84 | os.mkdir('%s/debug' % directory) |
| 85 | with open("%s/status.log" % directory, mode='w+') as status: |
| 86 | status.write("GOOD: nonexistent test completed successfully") |
| 87 | status.flush() |
| 88 | new_id = suite.handle_local_result( |
| 89 | job.id, directory, |
| 90 | lambda log_entry, log_in_subdir=False: None) |
| 91 | finally: |
| 92 | shutil.rmtree(directory) |
| 93 | |
| 94 | |
| 95 | def test_handle_local_result_for_bad_test(self): |
| 96 | getter = self.mox.CreateMock(control_file_getter.DevServerGetter) |
| 97 | getter.get_control_file_list(suite_name=mox.IgnoreArg()).AndReturn([]) |
| 98 | job = DummyJob() |
| 99 | test = self.mox.CreateMock(control_data.ControlData) |
| 100 | test.job_retries = 5 |
| 101 | self.mox.StubOutWithMock(test_runner_utils.LocalSuite, |
| 102 | '_retry_local_result') |
| 103 | test_runner_utils.LocalSuite._retry_local_result( |
| 104 | job.id, mox.IgnoreArg()).AndReturn(42) |
| 105 | self.mox.ReplayAll() |
| 106 | suite = test_runner_utils.LocalSuite([], "tag", [], None, getter, |
| 107 | job_retry=True) |
| 108 | suite._retry_handler = suite_module.RetryHandler({job.id: test}) |
| 109 | directory = tempfile.mkdtemp() |
| 110 | try: |
| 111 | os.mkdir('%s/debug' % directory) |
| 112 | with open("%s/status.log" % directory, mode='w+') as status: |
| 113 | status.write("FAIL") |
| 114 | status.flush() |
| 115 | new_id = suite.handle_local_result( |
| 116 | job.id, directory, |
| 117 | lambda log_entry, log_in_subdir=False: None) |
| 118 | self.assertIsNotNone(new_id) |
| 119 | finally: |
| 120 | shutil.rmtree(directory) |
| 121 | |
| 122 | |
Simran Basi | 14622bb | 2015-11-25 13:23:40 -0800 | [diff] [blame] | 123 | def test_get_predicate_for_test_arg(self): |
| 124 | # Assert the type signature of get_predicate_for_test(...) |
| 125 | # Because control.test_utils_wrapper calls this function, |
| 126 | # it is imperative for backwards compatilbility that |
| 127 | # the return type of the tested function does not change. |
| 128 | tests = ['dummy_test', 'e:name_expression', 'f:expression', |
| 129 | 'suite:suitename'] |
| 130 | for test in tests: |
| 131 | pred, desc = test_runner_utils.get_predicate_for_test_arg(test) |
| 132 | self.assertTrue(isinstance(pred, types.FunctionType)) |
| 133 | self.assertTrue(isinstance(desc, str)) |
| 134 | |
| 135 | def test_run_job(self): |
| 136 | class Object(): |
| 137 | pass |
| 138 | |
| 139 | autotest_path = 'htap_tsetotua' |
| 140 | autoserv_command = os.path.join(autotest_path, 'server', 'autoserv') |
| 141 | remote = 'etomer' |
| 142 | results_dir = '/tmp/fakeresults' |
| 143 | fast_mode = False |
| 144 | job1_results_dir = '/tmp/fakeresults/results-1-gilbert' |
| 145 | job2_results_dir = '/tmp/fakeresults/results-2-sullivan' |
| 146 | args = 'matey' |
| 147 | expected_args_sublist = ['--args', args] |
| 148 | experimental_keyval = {constants.JOB_EXPERIMENTAL_KEY: False} |
Simran Basi | 14622bb | 2015-11-25 13:23:40 -0800 | [diff] [blame] | 149 | |
| 150 | # Create some dummy job objects. |
| 151 | job1 = Object() |
| 152 | job2 = Object() |
| 153 | setattr(job1, 'control_type', 'cLiEnT') |
| 154 | setattr(job1, 'control_file', 'c1') |
| 155 | setattr(job1, 'id', 1) |
| 156 | setattr(job1, 'name', 'gilbert') |
| 157 | setattr(job1, 'keyvals', experimental_keyval) |
| 158 | |
| 159 | setattr(job2, 'control_type', 'Server') |
| 160 | setattr(job2, 'control_file', 'c2') |
| 161 | setattr(job2, 'id', 2) |
| 162 | setattr(job2, 'name', 'sullivan') |
| 163 | setattr(job2, 'keyvals', experimental_keyval) |
| 164 | |
| 165 | id_digits = 1 |
| 166 | |
| 167 | # Stub out subprocess.Popen and wait calls. |
| 168 | # Make them expect correct arguments. |
| 169 | def fake_readline(): |
| 170 | return b'' |
| 171 | mock_process_1 = self.mox.CreateMock(subprocess.Popen) |
| 172 | mock_process_2 = self.mox.CreateMock(subprocess.Popen) |
| 173 | fake_stdout = self.mox.CreateMock(file) |
| 174 | fake_returncode = 0 |
| 175 | mock_process_1.stdout = fake_stdout |
| 176 | mock_process_1.returncode = fake_returncode |
| 177 | mock_process_2.stdout = fake_stdout |
| 178 | mock_process_2.returncode = fake_returncode |
| 179 | |
| 180 | self.mox.StubOutWithMock(os, 'makedirs') |
| 181 | self.mox.StubOutWithMock(utils, 'write_keyval') |
| 182 | self.mox.StubOutWithMock(subprocess, 'Popen') |
| 183 | |
| 184 | os.makedirs(job1_results_dir) |
| 185 | utils.write_keyval(job1_results_dir, experimental_keyval) |
| 186 | arglist_1 = [autoserv_command, '-p', '-r', job1_results_dir, |
| 187 | '-m', remote, '--no_console_prefix', '-l', 'gilbert', |
| 188 | '-c'] |
| 189 | subprocess.Popen(mox.And(StartsWithList(arglist_1), |
| 190 | ContainsSublist(expected_args_sublist)), |
| 191 | stdout=subprocess.PIPE, |
| 192 | stderr=subprocess.STDOUT |
| 193 | ).AndReturn(mock_process_1) |
| 194 | mock_process_1.stdout.readline().AndReturn(b'') |
| 195 | mock_process_1.wait().AndReturn(0) |
| 196 | |
| 197 | os.makedirs(job2_results_dir) |
| 198 | utils.write_keyval(job2_results_dir, experimental_keyval) |
| 199 | arglist_2 = [autoserv_command, '-p', '-r', job2_results_dir, |
| 200 | '-m', remote, '--no_console_prefix', '-l', 'sullivan', |
| 201 | '-s'] |
| 202 | subprocess.Popen(mox.And(StartsWithList(arglist_2), |
| 203 | ContainsSublist(expected_args_sublist)), |
| 204 | stdout=subprocess.PIPE, |
| 205 | stderr=subprocess.STDOUT |
| 206 | ).AndReturn(mock_process_2) |
| 207 | mock_process_2.stdout.readline().AndReturn(b'') |
| 208 | mock_process_2.wait().AndReturn(0) |
| 209 | |
| 210 | # Test run_job. |
| 211 | self.mox.ReplayAll() |
| 212 | code, job_res = test_runner_utils.run_job( |
| 213 | job1, remote, autotest_path,results_dir, fast_mode, id_digits, |
| 214 | 0, None, args) |
| 215 | self.assertEqual(job_res, job1_results_dir) |
| 216 | self.assertEqual(code, 0) |
| 217 | code, job_res = test_runner_utils.run_job( |
| 218 | job2, remote, autotest_path, results_dir, fast_mode, id_digits, |
| 219 | 0, None, args) |
| 220 | |
| 221 | self.assertEqual(job_res, job2_results_dir) |
| 222 | self.assertEqual(code, 0) |
Simran Basi | 14622bb | 2015-11-25 13:23:40 -0800 | [diff] [blame] | 223 | self.mox.ResetAll() |
| 224 | |
| 225 | def test_perform_local_run(self): |
| 226 | afe = test_runner_utils.setup_local_afe() |
| 227 | autotest_path = 'ottotest_path' |
| 228 | suite_name = 'sweet_name' |
| 229 | test_arg = 'suite:' + suite_name |
| 230 | remote = 'remoat' |
| 231 | build = 'bild' |
| 232 | board = 'bored' |
| 233 | fast_mode = False |
| 234 | suite_control_files = ['c1', 'c2', 'c3', 'c4'] |
| 235 | results_dir = '/tmp/test_that_results_fake' |
| 236 | id_digits = 1 |
| 237 | ssh_verbosity = 2 |
| 238 | ssh_options = '-F /dev/null -i /dev/null' |
| 239 | args = 'matey' |
| 240 | ignore_deps = False |
| 241 | |
| 242 | # Fake suite objects that will be returned by fetch_local_suite |
| 243 | class fake_suite(object): |
| 244 | def __init__(self, suite_control_files, hosts): |
| 245 | self._suite_control_files = suite_control_files |
| 246 | self._hosts = hosts |
Jacob Kopczynski | 2cefa1f | 2018-01-10 17:25:38 -0800 | [diff] [blame] | 247 | self._jobs = [] |
| 248 | self._jobs_to_tests = {} |
| 249 | self.retry_hack = True |
Simran Basi | 14622bb | 2015-11-25 13:23:40 -0800 | [diff] [blame] | 250 | |
| 251 | def schedule(self, *args, **kwargs): |
| 252 | for control_file in self._suite_control_files: |
Jacob Kopczynski | 2cefa1f | 2018-01-10 17:25:38 -0800 | [diff] [blame] | 253 | job_id = afe.create_job(control_file, hosts=self._hosts) |
| 254 | self._jobs.append(job_id) |
| 255 | self._jobs_to_tests[job_id] = control_file |
| 256 | |
| 257 | def handle_local_result(self, job_id, results_dir, logger, |
| 258 | **kwargs): |
| 259 | if results_dir == "success_directory": |
| 260 | return None |
| 261 | retries = True |
| 262 | if 'retries' in kwargs: |
| 263 | retries = kwargs['retries'] |
| 264 | if retries and self.retry_hack: |
| 265 | self.retry_hack = False |
| 266 | else: |
| 267 | return None |
| 268 | control_file = self._jobs_to_tests.get(job_id) |
| 269 | job_id = afe.create_job(control_file, hosts=self._hosts) |
| 270 | self._jobs.append(job_id) |
| 271 | self._jobs_to_tests[job_id] = control_file |
| 272 | return job_id |
| 273 | |
| 274 | @property |
| 275 | def jobs(self): |
| 276 | return self._jobs |
| 277 | |
| 278 | def test_name_from_job(self, id): |
| 279 | return "" |
Simran Basi | 14622bb | 2015-11-25 13:23:40 -0800 | [diff] [blame] | 280 | |
| 281 | # Mock out scheduling of suite and running of jobs. |
Simran Basi | 14622bb | 2015-11-25 13:23:40 -0800 | [diff] [blame] | 282 | self.mox.StubOutWithMock(test_runner_utils, 'fetch_local_suite') |
| 283 | test_runner_utils.fetch_local_suite(autotest_path, mox.IgnoreArg(), |
| 284 | afe, test_arg=test_arg, remote=remote, build=build, |
| 285 | board=board, results_directory=results_dir, |
| 286 | no_experimental=False, |
| 287 | ignore_deps=ignore_deps |
| 288 | ).AndReturn(fake_suite(suite_control_files, [remote])) |
| 289 | self.mox.StubOutWithMock(test_runner_utils, 'run_job') |
| 290 | self.mox.StubOutWithMock(test_runner_utils, 'run_provisioning_job') |
| 291 | self.mox.StubOutWithMock(test_runner_utils, '_auto_detect_labels') |
| 292 | |
| 293 | test_runner_utils._auto_detect_labels(afe, remote) |
| 294 | # Test perform_local_run. Enforce that run_provisioning_job, |
| 295 | # run_job and _auto_detect_labels are called correctly. |
| 296 | test_runner_utils.run_provisioning_job( |
| 297 | 'cros-version:' + build, remote, autotest_path, |
| 298 | results_dir, fast_mode, |
| 299 | ssh_verbosity, ssh_options, |
| 300 | False, False) |
| 301 | |
| 302 | for control_file in suite_control_files: |
| 303 | test_runner_utils.run_job( |
| 304 | mox.ContainsAttributeValue('control_file', control_file), |
| 305 | remote, autotest_path, results_dir, fast_mode,id_digits, |
| 306 | ssh_verbosity, ssh_options,args, False, |
| 307 | False, {}).AndReturn((0, '/fake/dir')) |
| 308 | self.mox.ReplayAll() |
| 309 | test_runner_utils.perform_local_run( |
| 310 | afe, autotest_path, ['suite:'+suite_name], remote, fast_mode, |
| 311 | build=build, board=board, ignore_deps=False, |
| 312 | ssh_verbosity=ssh_verbosity, ssh_options=ssh_options, |
| 313 | args=args, results_directory=results_dir) |
Simran Basi | 14622bb | 2015-11-25 13:23:40 -0800 | [diff] [blame] | 314 | |
| 315 | |
| 316 | if __name__ == '__main__': |
| 317 | unittest.main() |