Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 1 | #!/usr/bin/python |
| 2 | # Copyright (c) 2014 The Chromium OS Authors. All rights reserved. |
| 3 | # Use of this source code is governed by a BSD-style license that can be |
| 4 | # found in the LICENSE file. |
| 5 | |
Allen Li | cc20549 | 2017-07-10 17:26:04 -0700 | [diff] [blame] | 6 | import collections |
Prashanth B | a7be207 | 2014-07-15 15:03:21 -0700 | [diff] [blame] | 7 | import datetime as datetime_base |
| 8 | from datetime import datetime |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 9 | import mock |
Prashanth B | a7be207 | 2014-07-15 15:03:21 -0700 | [diff] [blame] | 10 | import time |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 11 | import unittest |
| 12 | |
| 13 | import common |
| 14 | |
| 15 | from autotest_lib.server.cros.dynamic_suite import constants |
| 16 | from autotest_lib.site_utils import run_suite |
Xixuan Wu | 888ee7a | 2018-04-24 10:27:27 -0700 | [diff] [blame] | 17 | from autotest_lib.site_utils import run_suite_common |
Prashanth B | a7be207 | 2014-07-15 15:03:21 -0700 | [diff] [blame] | 18 | from autotest_lib.site_utils import diagnosis_utils |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 19 | |
| 20 | |
Allen Li | cc20549 | 2017-07-10 17:26:04 -0700 | [diff] [blame] | 21 | class ReturnResultUnittest(unittest.TestCase): |
| 22 | """_ReturnResult tests.""" |
| 23 | |
| 24 | def setUp(self): |
| 25 | super(ReturnResultUnittest, self).setUp() |
| 26 | patcher = mock.patch.object(run_suite, '_RETURN_RESULTS', |
| 27 | collections.OrderedDict()) |
| 28 | self.results = results = patcher.start() |
| 29 | self.addCleanup(patcher.stop) |
| 30 | results['small'] = run_suite._ReturnResult(0, 'small') |
| 31 | results['big'] = run_suite._ReturnResult(1, 'big') |
| 32 | |
| 33 | patcher = mock.patch.object(run_suite, '_RETURN_RESULTS_LIST', |
| 34 | list(results.values())) |
| 35 | patcher.start() |
| 36 | self.addCleanup(patcher.stop) |
| 37 | |
| 38 | def test_equal(self): |
| 39 | """Test _ReturnResult equal.""" |
| 40 | self.assertEqual(self.results['small'], self.results['small']) |
| 41 | |
| 42 | def test_unequal(self): |
| 43 | """Test _ReturnResult unequal.""" |
| 44 | self.assertNotEqual(self.results['big'], self.results['small']) |
| 45 | |
| 46 | def test_greater_than(self): |
| 47 | """Test _ReturnResult greater than.""" |
| 48 | self.assertGreater(self.results['big'], self.results['small']) |
| 49 | |
| 50 | def test_bitwise_or(self): |
| 51 | """Test _ReturnResult bitwise or.""" |
| 52 | self.assertEqual(self.results['big'], |
| 53 | self.results['big'] | self.results['small']) |
| 54 | |
| 55 | |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 56 | class ResultCollectorUnittest(unittest.TestCase): |
| 57 | """Runsuite unittest""" |
| 58 | |
Fang Deng | f850353 | 2014-06-12 18:21:55 -0700 | [diff] [blame] | 59 | JOB_MAX_RUNTIME_MINS = 10 |
| 60 | |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 61 | def setUp(self): |
| 62 | """Set up test.""" |
| 63 | self.afe = mock.MagicMock() |
| 64 | self.tko = mock.MagicMock() |
| 65 | |
| 66 | |
| 67 | def _build_view(self, test_idx, test_name, subdir, status, afe_job_id, |
| 68 | job_name='fake_job_name', reason='fake reason', |
| 69 | job_keyvals=None, test_started_time=None, |
Fang Deng | f850353 | 2014-06-12 18:21:55 -0700 | [diff] [blame] | 70 | test_finished_time=None, invalidates_test_idx=None, |
| 71 | job_started_time=None, job_finished_time=None): |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 72 | """Build a test view using the given fields. |
| 73 | |
| 74 | @param test_idx: An integer representing test_idx. |
| 75 | @param test_name: A string, e.g. 'dummy_Pass' |
| 76 | @param subdir: A string representing the subdir field of the test view. |
| 77 | e.g. 'dummy_Pass'. |
| 78 | @param status: A string representing the test status. |
| 79 | e.g. 'FAIL', 'PASS' |
| 80 | @param afe_job_id: An integer representing the afe job id. |
| 81 | @param job_name: A string representing the job name. |
| 82 | @param reason: A string representing the reason field of the test view. |
| 83 | @param job_keyvals: A dictionary stroing the job keyvals. |
| 84 | @param test_started_time: A string, e.g. '2014-04-12 12:35:33' |
| 85 | @param test_finished_time: A string, e.g. '2014-04-12 12:35:33' |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 86 | @param invalidates_test_idx: An integer, representing the idx of the |
| 87 | test that has been retried. |
Fang Deng | f850353 | 2014-06-12 18:21:55 -0700 | [diff] [blame] | 88 | @param job_started_time: A string, e.g. '2014-04-12 12:35:33' |
| 89 | @param job_finished_time: A string, e.g. '2014-04-12 12:35:33' |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 90 | |
| 91 | @reutrn: A dictionary representing a test view. |
| 92 | |
| 93 | """ |
| 94 | if job_keyvals is None: |
| 95 | job_keyvals = {} |
| 96 | return {'test_idx': test_idx, 'test_name': test_name, 'subdir':subdir, |
| 97 | 'status': status, 'afe_job_id': afe_job_id, |
| 98 | 'job_name': job_name, 'reason': reason, |
| 99 | 'job_keyvals': job_keyvals, |
| 100 | 'test_started_time': test_started_time, |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 101 | 'test_finished_time': test_finished_time, |
Fang Deng | f850353 | 2014-06-12 18:21:55 -0700 | [diff] [blame] | 102 | 'invalidates_test_idx': invalidates_test_idx, |
| 103 | 'job_started_time': job_started_time, |
| 104 | 'job_finished_time': job_finished_time} |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 105 | |
| 106 | |
David Riley | dcd1a64 | 2017-03-01 23:15:08 -0800 | [diff] [blame] | 107 | def _mock_tko_get_detailed_test_views(self, test_views, |
| 108 | missing_results=[]): |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 109 | """Mock tko method get_detailed_test_views call. |
| 110 | |
| 111 | @param test_views: A list of test views that will be returned |
| 112 | by get_detailed_test_views. |
| 113 | """ |
| 114 | return_values = {} |
| 115 | for v in test_views: |
| 116 | views_of_job = return_values.setdefault( |
| 117 | ('get_detailed_test_views', v['afe_job_id']), []) |
| 118 | views_of_job.append(v) |
David Riley | dcd1a64 | 2017-03-01 23:15:08 -0800 | [diff] [blame] | 119 | for job_id in missing_results: |
| 120 | views_of_job = return_values.setdefault( |
| 121 | ('get_detailed_test_views', job_id), []) |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 122 | |
| 123 | def side_effect(*args, **kwargs): |
| 124 | """Maps args and kwargs to the mocked return values.""" |
| 125 | key = (kwargs['call'], kwargs['afe_job_id']) |
| 126 | return return_values[key] |
| 127 | |
| 128 | self.tko.run = mock.MagicMock(side_effect=side_effect) |
| 129 | |
| 130 | |
| 131 | def _mock_afe_get_jobs(self, suite_job_id, child_job_ids): |
| 132 | """Mock afe get_jobs call. |
| 133 | |
| 134 | @param suite_job_id: The afe job id of the suite job. |
| 135 | @param child_job_ids: A list of job ids of the child jobs. |
| 136 | |
| 137 | """ |
Fang Deng | f850353 | 2014-06-12 18:21:55 -0700 | [diff] [blame] | 138 | suite_job = mock.MagicMock() |
| 139 | suite_job.id = suite_job_id |
| 140 | suite_job.max_runtime_mins = 10 |
| 141 | suite_job.parent_job = None |
| 142 | |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 143 | return_values = {suite_job_id: []} |
| 144 | for job_id in child_job_ids: |
| 145 | new_job = mock.MagicMock() |
| 146 | new_job.id = job_id |
David Riley | dcd1a64 | 2017-03-01 23:15:08 -0800 | [diff] [blame] | 147 | new_job.name = 'test.%d' % job_id |
Fang Deng | f850353 | 2014-06-12 18:21:55 -0700 | [diff] [blame] | 148 | new_job.max_runtime_mins = self.JOB_MAX_RUNTIME_MINS |
| 149 | new_job.parent_job = suite_job |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 150 | return_values[suite_job_id].append(new_job) |
| 151 | |
| 152 | def side_effect(*args, **kwargs): |
| 153 | """Maps args and kwargs to the mocked return values.""" |
Fang Deng | f850353 | 2014-06-12 18:21:55 -0700 | [diff] [blame] | 154 | if kwargs.get('id') == suite_job_id: |
| 155 | return [suite_job] |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 156 | return return_values[kwargs['parent_job_id']] |
| 157 | |
| 158 | self.afe.get_jobs = mock.MagicMock(side_effect=side_effect) |
| 159 | |
| 160 | |
| 161 | def testFetchSuiteTestView(self): |
| 162 | """Test that it fetches the correct suite test views.""" |
| 163 | suite_job_id = 100 |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 164 | suite_name = 'dummy' |
| 165 | build = 'R23-1.1.1.1' |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 166 | server_job_view = self._build_view( |
| 167 | 10, 'SERVER_JOB', '----', 'GOOD', suite_job_id) |
| 168 | test_to_ignore = self._build_view( |
| 169 | 11, 'dummy_Pass', '101-user/host/dummy_Pass', |
| 170 | 'GOOD', suite_job_id) |
| 171 | test_to_include = self._build_view( |
| 172 | 12, 'dummy_Pass.bluetooth', None, 'TEST_NA', suite_job_id) |
David Riley | dcd1a64 | 2017-03-01 23:15:08 -0800 | [diff] [blame] | 173 | test_missing = self._build_view( |
| 174 | 13, 'dummy_Missing', None, 'ABORT', suite_job_id) |
Fang Deng | f850353 | 2014-06-12 18:21:55 -0700 | [diff] [blame] | 175 | self._mock_afe_get_jobs(suite_job_id, []) |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 176 | self._mock_tko_get_detailed_test_views( |
David Riley | dcd1a64 | 2017-03-01 23:15:08 -0800 | [diff] [blame] | 177 | [server_job_view, test_to_ignore, test_to_include, |
| 178 | test_missing]) |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 179 | collector = run_suite.ResultCollector( |
| 180 | 'fake_server', self.afe, self.tko, |
Richard Barnette | ed5115a | 2018-09-12 17:12:54 -0700 | [diff] [blame] | 181 | build=build, suite_name=suite_name, |
Allen Li | 0b675b6 | 2017-07-05 13:38:04 -0700 | [diff] [blame] | 182 | suite_job_id=suite_job_id, |
| 183 | return_code_function=run_suite._ReturnCodeComputer()) |
David Riley | dcd1a64 | 2017-03-01 23:15:08 -0800 | [diff] [blame] | 184 | collector._missing_results = { |
| 185 | test_missing['test_name']: [14, 15], |
| 186 | } |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 187 | suite_views = collector._fetch_relevant_test_views_of_suite() |
| 188 | suite_views = sorted(suite_views, key=lambda view: view['test_idx']) |
Shuqian Zhao | c085abb | 2016-02-24 11:27:26 -0800 | [diff] [blame] | 189 | # Verify that SERVER_JOB is renamed to 'Suite job' |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 190 | self.assertEqual(suite_views[0].get_testname(), |
Shuqian Zhao | c085abb | 2016-02-24 11:27:26 -0800 | [diff] [blame] | 191 | run_suite.TestView.SUITE_JOB) |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 192 | # Verify that the test with a subidr is not included. |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 193 | self.assertEqual(suite_views[0]['test_idx'], 10) |
| 194 | self.assertEqual(suite_views[1]['test_idx'], 12) |
David Riley | dcd1a64 | 2017-03-01 23:15:08 -0800 | [diff] [blame] | 195 | self.assertEqual(suite_views[1]['afe_job_id'], suite_job_id) |
| 196 | # Verify that the test with missing results had it's AFE job id |
| 197 | # replaced. |
| 198 | self.assertEqual(suite_views[2]['test_idx'], 13) |
| 199 | self.assertEqual(suite_views[2]['afe_job_id'], 14) |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 200 | |
| 201 | |
| 202 | def testFetchTestViewOfChildJobs(self): |
| 203 | """Test that it fetches the correct child test views.""" |
| 204 | build = 'lumpy-release/R36-5788.0.0' |
MK Ryu | 977a975 | 2014-10-21 11:58:09 -0700 | [diff] [blame] | 205 | board = 'lumpy' |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 206 | suite_name = 'my_suite' |
| 207 | suite_job_id = 100 |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 208 | invalid_job_id = 101 |
| 209 | invalid_job_name = '%s/%s/test_Pass' % (build, suite_name) |
| 210 | good_job_id = 102 |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 211 | good_job_name = '%s/%s/test_Pass' % (build, suite_name) |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 212 | bad_job_id = 103 |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 213 | bad_job_name = '%s/%s/test_ServerJobFail' % (build, suite_name) |
David Riley | dcd1a64 | 2017-03-01 23:15:08 -0800 | [diff] [blame] | 214 | missing_job_id = 104 |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 215 | |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 216 | invalid_test = self._build_view( |
| 217 | 19, 'test_Pass_Old', 'fake/subdir', |
| 218 | 'FAIL', invalid_job_id, invalid_job_name) |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 219 | good_job_server_job = self._build_view( |
| 220 | 20, 'SERVER_JOB', '----', 'GOOD', good_job_id, good_job_name) |
| 221 | good_job_test = self._build_view( |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 222 | 21, 'test_Pass', 'fake/subdir', 'GOOD', |
| 223 | good_job_id, good_job_name, |
| 224 | job_keyvals={'retry_original_job_id': invalid_job_id}) |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 225 | bad_job_server_job = self._build_view( |
| 226 | 22, 'SERVER_JOB', '----', 'FAIL', bad_job_id, bad_job_name) |
| 227 | bad_job_test = self._build_view( |
| 228 | 23, 'test_ServerJobFail', 'fake/subdir', 'GOOD', |
| 229 | bad_job_id, bad_job_name) |
| 230 | self._mock_tko_get_detailed_test_views( |
| 231 | [good_job_server_job, good_job_test, |
David Riley | dcd1a64 | 2017-03-01 23:15:08 -0800 | [diff] [blame] | 232 | bad_job_server_job, bad_job_test, invalid_test], |
| 233 | [missing_job_id]) |
| 234 | self._mock_afe_get_jobs(suite_job_id, |
| 235 | [good_job_id, bad_job_id, missing_job_id]) |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 236 | collector = run_suite.ResultCollector( |
| 237 | 'fake_server', self.afe, self.tko, |
Richard Barnette | ed5115a | 2018-09-12 17:12:54 -0700 | [diff] [blame] | 238 | build, suite_name, suite_job_id, |
Allen Li | 0b675b6 | 2017-07-05 13:38:04 -0700 | [diff] [blame] | 239 | return_code_function=run_suite._ReturnCodeComputer()) |
David Riley | dcd1a64 | 2017-03-01 23:15:08 -0800 | [diff] [blame] | 240 | child_views, retry_counts, missing_results = ( |
| 241 | collector._fetch_test_views_of_child_jobs()) |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 242 | # child_views should contain tests 21, 22, 23 |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 243 | child_views = sorted(child_views, key=lambda view: view['test_idx']) |
| 244 | # Verify that the SERVER_JOB has been renamed properly |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 245 | self.assertEqual(child_views[1].get_testname(), |
| 246 | 'test_ServerJobFail_SERVER_JOB') |
David Riley | dcd1a64 | 2017-03-01 23:15:08 -0800 | [diff] [blame] | 247 | self.assertEqual(missing_results, {'test.104': [104]}) |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 248 | # Verify that failed SERVER_JOB and actual invalid tests are included, |
| 249 | expected = [good_job_test['test_idx'], bad_job_server_job['test_idx'], |
| 250 | bad_job_test['test_idx']] |
| 251 | child_view_ids = [v['test_idx'] for v in child_views] |
| 252 | self.assertEqual(child_view_ids, expected) |
| 253 | self.afe.get_jobs.assert_called_once_with( |
| 254 | parent_job_id=suite_job_id) |
| 255 | # Verify the retry_counts is calculated correctly |
| 256 | self.assertEqual(len(retry_counts), 1) |
| 257 | self.assertEqual(retry_counts[21], 1) |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 258 | |
| 259 | |
| 260 | def testGenerateLinks(self): |
| 261 | """Test that it generates correct web and buildbot links.""" |
| 262 | suite_job_id = 100 |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 263 | suite_name = 'my_suite' |
| 264 | build = 'lumpy-release/R36-5788.0.0' |
MK Ryu | 977a975 | 2014-10-21 11:58:09 -0700 | [diff] [blame] | 265 | board = 'lumpy' |
Fang Deng | f850353 | 2014-06-12 18:21:55 -0700 | [diff] [blame] | 266 | fake_job = mock.MagicMock() |
| 267 | fake_job.parent = suite_job_id |
Dan Shi | 9b620c2 | 2017-10-10 10:58:37 -0700 | [diff] [blame] | 268 | test_sponge_url = 'http://test_url' |
| 269 | job_keyvals = {'sponge_url': test_sponge_url} |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 270 | suite_job_view = run_suite.TestView( |
| 271 | self._build_view( |
Dan Shi | 9b620c2 | 2017-10-10 10:58:37 -0700 | [diff] [blame] | 272 | 20, 'Suite job', '----', 'GOOD', suite_job_id, |
| 273 | job_keyvals=job_keyvals), |
Simran Basi | 01984f5 | 2015-10-12 15:36:45 -0700 | [diff] [blame] | 274 | fake_job, suite_name, build, 'chromeos-test') |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 275 | good_test = run_suite.TestView( |
| 276 | self._build_view( |
Dan Shi | 9b620c2 | 2017-10-10 10:58:37 -0700 | [diff] [blame] | 277 | 21, 'test_Pass', 'fake/subdir', 'GOOD', 101, |
| 278 | job_keyvals=job_keyvals), |
Simran Basi | 01984f5 | 2015-10-12 15:36:45 -0700 | [diff] [blame] | 279 | fake_job, suite_name, build, 'chromeos-test') |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 280 | bad_test = run_suite.TestView( |
| 281 | self._build_view( |
Dan Shi | 9b620c2 | 2017-10-10 10:58:37 -0700 | [diff] [blame] | 282 | 23, 'test_Fail', 'fake/subdir', 'FAIL', 102, |
| 283 | job_keyvals=job_keyvals), |
Simran Basi | 01984f5 | 2015-10-12 15:36:45 -0700 | [diff] [blame] | 284 | fake_job, suite_name, build, 'chromeos-test') |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 285 | |
| 286 | collector = run_suite.ResultCollector( |
| 287 | 'fake_server', self.afe, self.tko, |
Richard Barnette | ed5115a | 2018-09-12 17:12:54 -0700 | [diff] [blame] | 288 | build, suite_name, suite_job_id, user='chromeos-test', |
Allen Li | 0b675b6 | 2017-07-05 13:38:04 -0700 | [diff] [blame] | 289 | return_code_function=run_suite._ReturnCodeComputer()) |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 290 | collector._suite_views = [suite_job_view] |
| 291 | collector._test_views = [suite_job_view, good_test, bad_test] |
| 292 | collector._max_testname_width = max( |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 293 | [len(v.get_testname()) for v in collector._test_views]) + 3 |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 294 | collector._generate_web_and_buildbot_links() |
Allen Li | dc2c69a | 2016-09-14 19:05:47 -0700 | [diff] [blame] | 295 | URL_PATTERN = run_suite._URL_PATTERN |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 296 | # expected_web_links is list of (anchor, url) tuples we |
| 297 | # are expecting. |
| 298 | expected_web_links = [ |
Allen Li | 3461324 | 2016-09-02 11:52:34 -0700 | [diff] [blame] | 299 | (v.get_testname(), |
Prathmesh Prabhu | cd246f5 | 2018-01-03 13:45:48 -0800 | [diff] [blame] | 300 | URL_PATTERN % ('http://fake_server', |
| 301 | '%s-%s' % (v['afe_job_id'], 'chromeos-test')), |
Dan Shi | 9b620c2 | 2017-10-10 10:58:37 -0700 | [diff] [blame] | 302 | test_sponge_url) |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 303 | for v in collector._test_views] |
| 304 | # Verify web links are generated correctly. |
| 305 | for i in range(len(collector._web_links)): |
| 306 | expect = expected_web_links[i] |
| 307 | self.assertEqual(collector._web_links[i].anchor, expect[0]) |
| 308 | self.assertEqual(collector._web_links[i].url, expect[1]) |
Dan Shi | 9b620c2 | 2017-10-10 10:58:37 -0700 | [diff] [blame] | 309 | self.assertEqual(collector._web_links[i].sponge_url, expect[2]) |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 310 | |
| 311 | expected_buildbot_links = [ |
Allen Li | 3461324 | 2016-09-02 11:52:34 -0700 | [diff] [blame] | 312 | (v.get_testname(), |
Prathmesh Prabhu | cd246f5 | 2018-01-03 13:45:48 -0800 | [diff] [blame] | 313 | URL_PATTERN % ('http://fake_server', |
| 314 | '%s-%s' % (v['afe_job_id'], 'chromeos-test'))) |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 315 | for v in collector._test_views if v['status'] != 'GOOD'] |
| 316 | # Verify buildbot links are generated correctly. |
Allen Li | 28be064 | 2017-07-10 15:16:26 -0700 | [diff] [blame] | 317 | for i in range(len(collector.buildbot_links)): |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 318 | expect = expected_buildbot_links[i] |
Allen Li | 28be064 | 2017-07-10 15:16:26 -0700 | [diff] [blame] | 319 | self.assertEqual(collector.buildbot_links[i].anchor, expect[0]) |
| 320 | self.assertEqual(collector.buildbot_links[i].url, expect[1]) |
| 321 | self.assertEqual(collector.buildbot_links[i].retry_count, 0) |
Shuhei Takahashi | 18f5649 | 2017-11-14 16:23:46 +0900 | [diff] [blame] | 322 | # Assert that a retry dashboard link is created. |
Simran Basi | 7203d4e | 2015-02-03 15:50:18 -0800 | [diff] [blame] | 323 | self.assertNotEqual( |
Shuhei Takahashi | 18f5649 | 2017-11-14 16:23:46 +0900 | [diff] [blame] | 324 | collector.buildbot_links[i].GenerateRetryLink(), '') |
David Riley | a0cd1c2 | 2017-07-10 11:15:57 -0700 | [diff] [blame] | 325 | self.assertNotEqual( |
Shuhei Takahashi | 18f5649 | 2017-11-14 16:23:46 +0900 | [diff] [blame] | 326 | collector.buildbot_links[i].GenerateHistoryLink(), '') |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 327 | |
| 328 | |
Fang Deng | 5a43be6 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 329 | def _end_to_end_test_helper( |
| 330 | self, include_bad_test=False, include_warn_test=False, |
Allen Li | e6236ec | 2017-07-05 12:52:36 -0700 | [diff] [blame] | 331 | include_timeout_test=False, |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 332 | include_self_aborted_test=False, |
Fang Deng | f850353 | 2014-06-12 18:21:55 -0700 | [diff] [blame] | 333 | include_aborted_by_suite_test=False, |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 334 | include_good_retry=False, include_bad_retry=False, |
Brian Norris | 58ae516 | 2017-09-26 13:34:56 -0700 | [diff] [blame] | 335 | include_good_test=True, |
Fang Deng | f850353 | 2014-06-12 18:21:55 -0700 | [diff] [blame] | 336 | suite_job_timed_out=False, suite_job_status='GOOD'): |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 337 | """A helper method for testing ResultCollector end-to-end. |
| 338 | |
| 339 | This method mocks the retrieving of required test views, |
| 340 | and call ResultCollector.run() to collect the results. |
| 341 | |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 342 | @param include_bad_test: |
| 343 | If True, include a view of a test which has status 'FAIL'. |
| 344 | @param include_warn_test: |
| 345 | If True, include a view of a test which has status 'WARN' |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 346 | @param include_timeout_test: |
| 347 | If True, include a view of a test which was aborted before |
| 348 | started. |
| 349 | @param include_self_aborted_test: |
| 350 | If True, include a view of test which was aborted after |
Fang Deng | f850353 | 2014-06-12 18:21:55 -0700 | [diff] [blame] | 351 | started and hit hits own timeout. |
| 352 | @param include_self_aborted_by_suite_test: |
| 353 | If True, include a view of test which was aborted after |
| 354 | started but has not hit its own timeout. |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 355 | @param include_good_retry: |
| 356 | If True, include a test that passed after retry. |
| 357 | @param include_bad_retry: |
| 358 | If True, include a test that failed after retry. |
Brian Norris | 58ae516 | 2017-09-26 13:34:56 -0700 | [diff] [blame] | 359 | @param include_good_test: |
| 360 | If True, include a test that passed. If False, pretend no tests |
| 361 | (including the parent suite job) came back with any test |
| 362 | results. |
Fang Deng | f850353 | 2014-06-12 18:21:55 -0700 | [diff] [blame] | 363 | @param suite_job_status: One of 'GOOD' 'FAIL' 'ABORT' 'RUNNING' |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 364 | |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 365 | @returns: A ResultCollector instance. |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 366 | """ |
| 367 | suite_job_id = 100 |
| 368 | good_job_id = 101 |
| 369 | bad_job_id = 102 |
Fang Deng | 5a43be6 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 370 | warn_job_id = 102 |
Fang Deng | 5a43be6 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 371 | timeout_job_id = 100 |
| 372 | self_aborted_job_id = 104 |
Fang Deng | f850353 | 2014-06-12 18:21:55 -0700 | [diff] [blame] | 373 | aborted_by_suite_job_id = 105 |
| 374 | good_retry_job_id = 106 |
| 375 | bad_retry_job_id = 107 |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 376 | invalid_job_id_1 = 90 |
| 377 | invalid_job_id_2 = 91 |
| 378 | suite_name = 'dummy' |
| 379 | build = 'lumpy-release/R27-3888.0.0' |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 380 | suite_job_keyvals = { |
| 381 | constants.DOWNLOAD_STARTED_TIME: '2014-04-29 13:14:20', |
| 382 | constants.PAYLOAD_FINISHED_TIME: '2014-04-29 13:14:25', |
| 383 | constants.ARTIFACT_FINISHED_TIME: '2014-04-29 13:14:30'} |
| 384 | |
Fang Deng | f850353 | 2014-06-12 18:21:55 -0700 | [diff] [blame] | 385 | suite_job_started_time = '2014-04-29 13:14:37' |
| 386 | if suite_job_timed_out: |
| 387 | suite_job_keyvals['aborted_by'] = 'test_user' |
| 388 | suite_job_finished_time = '2014-04-29 13:25:37' |
| 389 | suite_job_status = 'ABORT' |
| 390 | else: |
| 391 | suite_job_finished_time = '2014-04-29 13:23:37' |
| 392 | |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 393 | server_job_view = self._build_view( |
Fang Deng | 5a43be6 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 394 | 10, 'SERVER_JOB', '----', suite_job_status, suite_job_id, |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 395 | 'lumpy-release/R27-3888.0.0-test_suites/control.dummy', |
| 396 | '', suite_job_keyvals, '2014-04-29 13:14:37', |
Fang Deng | f850353 | 2014-06-12 18:21:55 -0700 | [diff] [blame] | 397 | '2014-04-29 13:20:27', job_started_time=suite_job_started_time, |
| 398 | job_finished_time=suite_job_finished_time) |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 399 | good_test = self._build_view( |
| 400 | 11, 'dummy_Pass', '101-user/host/dummy_Pass', 'GOOD', |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 401 | good_job_id, 'lumpy-release/R27-3888.0.0/dummy/dummy_Pass', |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 402 | '', {}, '2014-04-29 13:15:35', '2014-04-29 13:15:36') |
| 403 | bad_test = self._build_view( |
| 404 | 12, 'dummy_Fail.Fail', '102-user/host/dummy_Fail.Fail', 'FAIL', |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 405 | bad_job_id, 'lumpy-release/R27-3888.0.0/dummy/dummy_Fail.Fail', |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 406 | 'always fail', {}, '2014-04-29 13:16:00', |
| 407 | '2014-04-29 13:16:02') |
| 408 | warn_test = self._build_view( |
Fang Deng | 5a43be6 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 409 | 13, 'dummy_Fail.Warn', '102-user/host/dummy_Fail.Warn', 'WARN', |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 410 | warn_job_id, 'lumpy-release/R27-3888.0.0/dummy/dummy_Fail.Warn', |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 411 | 'always warn', {}, '2014-04-29 13:16:00', |
| 412 | '2014-04-29 13:16:02') |
Fang Deng | 5a43be6 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 413 | timeout_test = self._build_view( |
| 414 | 15, 'dummy_Timeout', '', 'ABORT', |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 415 | timeout_job_id, |
| 416 | 'lumpy-release/R27-3888.0.0/dummy/dummy_Timeout', |
Fang Deng | 5a43be6 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 417 | 'child job did not run', {}, '2014-04-29 13:15:37', |
| 418 | '2014-04-29 13:15:38') |
| 419 | self_aborted_test = self._build_view( |
| 420 | 16, 'dummy_Abort', '104-user/host/dummy_Abort', 'ABORT', |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 421 | self_aborted_job_id, |
| 422 | 'lumpy-release/R27-3888.0.0/dummy/dummy_Abort', |
Fang Deng | f850353 | 2014-06-12 18:21:55 -0700 | [diff] [blame] | 423 | 'child job aborted', {'aborted_by': 'test_user'}, |
| 424 | '2014-04-29 13:15:39', '2014-04-29 13:15:40', |
| 425 | job_started_time='2014-04-29 13:15:39', |
| 426 | job_finished_time='2014-04-29 13:25:40') |
| 427 | aborted_by_suite = self._build_view( |
| 428 | 17, 'dummy_AbortBySuite', '105-user/host/dummy_AbortBySuite', |
| 429 | 'RUNNING', aborted_by_suite_job_id, |
| 430 | 'lumpy-release/R27-3888.0.0/dummy/dummy_Abort', |
| 431 | 'aborted by suite', {'aborted_by': 'test_user'}, |
| 432 | '2014-04-29 13:15:39', '2014-04-29 13:15:40', |
| 433 | job_started_time='2014-04-29 13:15:39', |
| 434 | job_finished_time='2014-04-29 13:15:40') |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 435 | good_retry = self._build_view( |
Fang Deng | f850353 | 2014-06-12 18:21:55 -0700 | [diff] [blame] | 436 | 18, 'dummy_RetryPass', '106-user/host/dummy_RetryPass', 'GOOD', |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 437 | good_retry_job_id, |
| 438 | 'lumpy-release/R27-3888.0.0/dummy/dummy_RetryPass', |
| 439 | '', {'retry_original_job_id': invalid_job_id_1}, |
| 440 | '2014-04-29 13:15:37', |
| 441 | '2014-04-29 13:15:38', invalidates_test_idx=1) |
| 442 | bad_retry = self._build_view( |
Fang Deng | f850353 | 2014-06-12 18:21:55 -0700 | [diff] [blame] | 443 | 19, 'dummy_RetryFail', '107-user/host/dummy_RetryFail', 'FAIL', |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 444 | bad_retry_job_id, |
| 445 | 'lumpy-release/R27-3888.0.0/dummy/dummy_RetryFail', |
| 446 | 'retry failed', {'retry_original_job_id': invalid_job_id_2}, |
| 447 | '2014-04-29 13:15:39', '2014-04-29 13:15:40', |
| 448 | invalidates_test_idx=2) |
| 449 | invalid_test_1 = self._build_view( |
| 450 | 1, 'dummy_RetryPass', '90-user/host/dummy_RetryPass', 'GOOD', |
| 451 | invalid_job_id_1, |
| 452 | 'lumpy-release/R27-3888.0.0/dummy/dummy_RetryPass', |
| 453 | 'original test failed', {}, '2014-04-29 13:10:00', |
| 454 | '2014-04-29 13:10:01') |
| 455 | invalid_test_2 = self._build_view( |
| 456 | 2, 'dummy_RetryFail', '91-user/host/dummy_RetryFail', 'FAIL', |
| 457 | invalid_job_id_2, |
| 458 | 'lumpy-release/R27-3888.0.0/dummy/dummy_RetryFail', |
| 459 | 'original test failed', {}, |
| 460 | '2014-04-29 13:10:03', '2014-04-29 13:10:04') |
| 461 | |
Brian Norris | 58ae516 | 2017-09-26 13:34:56 -0700 | [diff] [blame] | 462 | test_views = [] |
| 463 | child_jobs = set() |
| 464 | missing_results = [] |
| 465 | if include_good_test: |
| 466 | test_views.append(server_job_view) |
| 467 | test_views.append(good_test) |
| 468 | child_jobs.add(good_job_id) |
| 469 | # Emulate missing even the parent/suite job. |
| 470 | else: |
| 471 | missing_results.append(suite_job_id) |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 472 | if include_bad_test: |
| 473 | test_views.append(bad_test) |
Fang Deng | 5a43be6 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 474 | child_jobs.add(bad_job_id) |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 475 | if include_warn_test: |
| 476 | test_views.append(warn_test) |
Fang Deng | 5a43be6 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 477 | child_jobs.add(warn_job_id) |
Fang Deng | 5a43be6 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 478 | if include_timeout_test: |
| 479 | test_views.append(timeout_test) |
Fang Deng | 5a43be6 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 480 | if include_self_aborted_test: |
| 481 | test_views.append(self_aborted_test) |
| 482 | child_jobs.add(self_aborted_job_id) |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 483 | if include_good_retry: |
| 484 | test_views.extend([good_retry, invalid_test_1]) |
| 485 | child_jobs.add(good_retry_job_id) |
| 486 | if include_bad_retry: |
| 487 | test_views.extend([bad_retry, invalid_test_2]) |
| 488 | child_jobs.add(bad_retry_job_id) |
Fang Deng | f850353 | 2014-06-12 18:21:55 -0700 | [diff] [blame] | 489 | if include_aborted_by_suite_test: |
| 490 | test_views.append(aborted_by_suite) |
| 491 | child_jobs.add(aborted_by_suite_job_id) |
Brian Norris | 58ae516 | 2017-09-26 13:34:56 -0700 | [diff] [blame] | 492 | self._mock_tko_get_detailed_test_views(test_views, |
| 493 | missing_results=missing_results) |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 494 | self._mock_afe_get_jobs(suite_job_id, child_jobs) |
| 495 | collector = run_suite.ResultCollector( |
| 496 | 'fake_server', self.afe, self.tko, |
Richard Barnette | ed5115a | 2018-09-12 17:12:54 -0700 | [diff] [blame] | 497 | 'lumpy-release/R36-5788.0.0', 'dummy', suite_job_id, |
Allen Li | 0b675b6 | 2017-07-05 13:38:04 -0700 | [diff] [blame] | 498 | return_code_function=run_suite._ReturnCodeComputer()) |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 499 | collector.run() |
Brian Norris | 58ae516 | 2017-09-26 13:34:56 -0700 | [diff] [blame] | 500 | collector.output_results() |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 501 | return collector |
| 502 | |
| 503 | |
Brian Norris | 58ae516 | 2017-09-26 13:34:56 -0700 | [diff] [blame] | 504 | def testEndToEndSuiteEmpty(self): |
| 505 | """Test it returns code INFRA_FAILURE when no tests report back.""" |
| 506 | collector = self._end_to_end_test_helper(include_good_test=False) |
| 507 | self.assertEqual(collector.return_result.return_code, |
Xixuan Wu | 888ee7a | 2018-04-24 10:27:27 -0700 | [diff] [blame] | 508 | run_suite_common.RETURN_CODES.INFRA_FAILURE) |
Brian Norris | 58ae516 | 2017-09-26 13:34:56 -0700 | [diff] [blame] | 509 | |
| 510 | |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 511 | def testEndToEndSuitePass(self): |
| 512 | """Test it returns code OK when all test pass.""" |
| 513 | collector = self._end_to_end_test_helper() |
Allen Li | cc20549 | 2017-07-10 17:26:04 -0700 | [diff] [blame] | 514 | self.assertEqual(collector.return_result.return_code, |
Xixuan Wu | 888ee7a | 2018-04-24 10:27:27 -0700 | [diff] [blame] | 515 | run_suite_common.RETURN_CODES.OK) |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 516 | |
| 517 | |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 518 | def testEndToEndSuiteWarn(self): |
| 519 | """Test it returns code WARNING when there is a test that warns.""" |
| 520 | collector = self._end_to_end_test_helper(include_warn_test=True) |
Allen Li | cc20549 | 2017-07-10 17:26:04 -0700 | [diff] [blame] | 521 | self.assertEqual(collector.return_result.return_code, |
Xixuan Wu | 888ee7a | 2018-04-24 10:27:27 -0700 | [diff] [blame] | 522 | run_suite_common.RETURN_CODES.WARNING) |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 523 | |
| 524 | |
| 525 | def testEndToEndSuiteFail(self): |
| 526 | """Test it returns code ERROR when there is a test that fails.""" |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 527 | collector = self._end_to_end_test_helper(include_bad_test=True) |
Allen Li | cc20549 | 2017-07-10 17:26:04 -0700 | [diff] [blame] | 528 | self.assertEqual(collector.return_result.return_code, |
Xixuan Wu | 888ee7a | 2018-04-24 10:27:27 -0700 | [diff] [blame] | 529 | run_suite_common.RETURN_CODES.ERROR) |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 530 | |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 531 | |
Fang Deng | 5a43be6 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 532 | def testEndToEndSuiteJobFail(self): |
| 533 | """Test it returns code SUITE_FAILURE when only the suite job failed.""" |
| 534 | collector = self._end_to_end_test_helper(suite_job_status='ABORT') |
| 535 | self.assertEqual( |
Allen Li | cc20549 | 2017-07-10 17:26:04 -0700 | [diff] [blame] | 536 | collector.return_result.return_code, |
Xixuan Wu | 888ee7a | 2018-04-24 10:27:27 -0700 | [diff] [blame] | 537 | run_suite_common.RETURN_CODES.INFRA_FAILURE) |
Fang Deng | 5a43be6 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 538 | |
| 539 | collector = self._end_to_end_test_helper(suite_job_status='ERROR') |
| 540 | self.assertEqual( |
Allen Li | cc20549 | 2017-07-10 17:26:04 -0700 | [diff] [blame] | 541 | collector.return_result.return_code, |
Xixuan Wu | 888ee7a | 2018-04-24 10:27:27 -0700 | [diff] [blame] | 542 | run_suite_common.RETURN_CODES.INFRA_FAILURE) |
Fang Deng | 5a43be6 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 543 | |
| 544 | |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 545 | def testEndToEndRetry(self): |
| 546 | """Test it returns correct code when a test was retried.""" |
| 547 | collector = self._end_to_end_test_helper(include_good_retry=True) |
| 548 | self.assertEqual( |
Allen Li | cc20549 | 2017-07-10 17:26:04 -0700 | [diff] [blame] | 549 | collector.return_result.return_code, |
Xixuan Wu | 888ee7a | 2018-04-24 10:27:27 -0700 | [diff] [blame] | 550 | run_suite_common.RETURN_CODES.WARNING) |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 551 | |
| 552 | collector = self._end_to_end_test_helper(include_good_retry=True, |
| 553 | include_self_aborted_test=True) |
| 554 | self.assertEqual( |
Allen Li | cc20549 | 2017-07-10 17:26:04 -0700 | [diff] [blame] | 555 | collector.return_result.return_code, |
Xixuan Wu | 888ee7a | 2018-04-24 10:27:27 -0700 | [diff] [blame] | 556 | run_suite_common.RETURN_CODES.ERROR) |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 557 | |
| 558 | collector = self._end_to_end_test_helper(include_good_retry=True, |
| 559 | include_bad_test=True) |
| 560 | self.assertEqual( |
Allen Li | cc20549 | 2017-07-10 17:26:04 -0700 | [diff] [blame] | 561 | collector.return_result.return_code, |
Xixuan Wu | 888ee7a | 2018-04-24 10:27:27 -0700 | [diff] [blame] | 562 | run_suite_common.RETURN_CODES.ERROR) |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 563 | |
| 564 | collector = self._end_to_end_test_helper(include_bad_retry=True) |
| 565 | self.assertEqual( |
Allen Li | cc20549 | 2017-07-10 17:26:04 -0700 | [diff] [blame] | 566 | collector.return_result.return_code, |
Xixuan Wu | 888ee7a | 2018-04-24 10:27:27 -0700 | [diff] [blame] | 567 | run_suite_common.RETURN_CODES.ERROR) |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 568 | |
| 569 | |
Fang Deng | 5a43be6 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 570 | def testEndToEndSuiteTimeout(self): |
| 571 | """Test it returns correct code when a child job timed out.""" |
Fang Deng | f850353 | 2014-06-12 18:21:55 -0700 | [diff] [blame] | 572 | # a child job timed out before started, none failed. |
Fang Deng | 5a43be6 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 573 | collector = self._end_to_end_test_helper(include_timeout_test=True) |
| 574 | self.assertEqual( |
Allen Li | cc20549 | 2017-07-10 17:26:04 -0700 | [diff] [blame] | 575 | collector.return_result.return_code, |
Xixuan Wu | 888ee7a | 2018-04-24 10:27:27 -0700 | [diff] [blame] | 576 | run_suite_common.RETURN_CODES.SUITE_TIMEOUT) |
Fang Deng | 5a43be6 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 577 | |
Fang Deng | f850353 | 2014-06-12 18:21:55 -0700 | [diff] [blame] | 578 | # a child job timed out before started, and one test failed. |
Fang Deng | 5a43be6 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 579 | collector = self._end_to_end_test_helper( |
| 580 | include_bad_test=True, include_timeout_test=True) |
Allen Li | cc20549 | 2017-07-10 17:26:04 -0700 | [diff] [blame] | 581 | self.assertEqual(collector.return_result.return_code, |
Xixuan Wu | 888ee7a | 2018-04-24 10:27:27 -0700 | [diff] [blame] | 582 | run_suite_common.RETURN_CODES.ERROR) |
Fang Deng | 5a43be6 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 583 | |
Fang Deng | f850353 | 2014-06-12 18:21:55 -0700 | [diff] [blame] | 584 | # a child job timed out before started, and one test warned. |
Fang Deng | 5a43be6 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 585 | collector = self._end_to_end_test_helper( |
| 586 | include_warn_test=True, include_timeout_test=True) |
Allen Li | cc20549 | 2017-07-10 17:26:04 -0700 | [diff] [blame] | 587 | self.assertEqual(collector.return_result.return_code, |
Xixuan Wu | 888ee7a | 2018-04-24 10:27:27 -0700 | [diff] [blame] | 588 | run_suite_common.RETURN_CODES.SUITE_TIMEOUT) |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 589 | |
Fang Deng | f850353 | 2014-06-12 18:21:55 -0700 | [diff] [blame] | 590 | # a child job timed out before started, and one test was retried. |
Fang Deng | aeab617 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 591 | collector = self._end_to_end_test_helper(include_good_retry=True, |
| 592 | include_timeout_test=True) |
Fang Deng | 5a43be6 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 593 | self.assertEqual( |
Allen Li | cc20549 | 2017-07-10 17:26:04 -0700 | [diff] [blame] | 594 | collector.return_result.return_code, |
Xixuan Wu | 888ee7a | 2018-04-24 10:27:27 -0700 | [diff] [blame] | 595 | run_suite_common.RETURN_CODES.SUITE_TIMEOUT) |
Fang Deng | 5a43be6 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 596 | |
Fang Deng | f850353 | 2014-06-12 18:21:55 -0700 | [diff] [blame] | 597 | # a child jot was aborted because suite timed out. |
| 598 | collector = self._end_to_end_test_helper( |
| 599 | include_aborted_by_suite_test=True) |
| 600 | self.assertEqual( |
Prathmesh Prabhu | 316180c | 2017-12-19 16:06:44 -0800 | [diff] [blame] | 601 | collector.return_result.return_code, |
Xixuan Wu | 888ee7a | 2018-04-24 10:27:27 -0700 | [diff] [blame] | 602 | run_suite_common.RETURN_CODES.SUITE_TIMEOUT) |
Fang Deng | f850353 | 2014-06-12 18:21:55 -0700 | [diff] [blame] | 603 | |
| 604 | # suite job timed out. |
| 605 | collector = self._end_to_end_test_helper(suite_job_timed_out=True) |
| 606 | self.assertEqual( |
Allen Li | cc20549 | 2017-07-10 17:26:04 -0700 | [diff] [blame] | 607 | collector.return_result.return_code, |
Xixuan Wu | 888ee7a | 2018-04-24 10:27:27 -0700 | [diff] [blame] | 608 | run_suite_common.RETURN_CODES.SUITE_TIMEOUT) |
Fang Deng | f850353 | 2014-06-12 18:21:55 -0700 | [diff] [blame] | 609 | |
Fang Deng | 5a43be6 | 2014-05-07 17:17:04 -0700 | [diff] [blame] | 610 | |
Ningning Xia | bd911bd | 2016-04-19 14:06:03 -0700 | [diff] [blame] | 611 | class LogLinkUnittests(unittest.TestCase): |
| 612 | """Test the LogLink""" |
| 613 | |
| 614 | def testGenerateBuildbotLinks(self): |
| 615 | """Test LogLink GenerateBuildbotLinks""" |
| 616 | log_link_a = run_suite.LogLink('mock_anchor', 'mock_server', |
| 617 | 'mock_job_string', |
| 618 | bug_info=('mock_bug_id', 1), |
| 619 | reason='mock_reason', |
| 620 | retry_count=1, |
| 621 | testname='mock_testname') |
| 622 | # Generate a bug link and a log link when bug_info is present |
Allen Li | e082ced | 2016-09-14 15:19:20 -0700 | [diff] [blame] | 623 | self.assertTrue(len(list(log_link_a.GenerateBuildbotLinks())) == 2) |
Ningning Xia | bd911bd | 2016-04-19 14:06:03 -0700 | [diff] [blame] | 624 | |
| 625 | log_link_b = run_suite.LogLink('mock_anchor', 'mock_server', |
| 626 | 'mock_job_string_b', |
| 627 | reason='mock_reason', |
| 628 | retry_count=1, |
| 629 | testname='mock_testname') |
| 630 | # Generate a log link when there is no bug_info |
Allen Li | e082ced | 2016-09-14 15:19:20 -0700 | [diff] [blame] | 631 | self.assertTrue(len(list(log_link_b.GenerateBuildbotLinks())) == 1) |
Ningning Xia | bd911bd | 2016-04-19 14:06:03 -0700 | [diff] [blame] | 632 | |
| 633 | |
Prashanth B | a7be207 | 2014-07-15 15:03:21 -0700 | [diff] [blame] | 634 | class SimpleTimerUnittests(unittest.TestCase): |
| 635 | """Test the simple timer.""" |
| 636 | |
| 637 | def testPoll(self): |
| 638 | """Test polling the timer.""" |
| 639 | interval_hours = 0.0001 |
| 640 | t = diagnosis_utils.SimpleTimer(interval_hours=interval_hours) |
| 641 | deadline = t.deadline |
| 642 | self.assertTrue(deadline is not None and |
| 643 | t.interval_hours == interval_hours) |
| 644 | min_deadline = (datetime.now() + |
| 645 | datetime_base.timedelta(hours=interval_hours)) |
| 646 | time.sleep(interval_hours * 3600) |
| 647 | self.assertTrue(t.poll()) |
| 648 | self.assertTrue(t.deadline >= min_deadline) |
| 649 | |
| 650 | |
| 651 | def testBadInterval(self): |
| 652 | """Test a bad interval.""" |
| 653 | t = diagnosis_utils.SimpleTimer(interval_hours=-1) |
| 654 | self.assertTrue(t.deadline is None and t.poll() == False) |
| 655 | t._reset() |
| 656 | self.assertTrue(t.deadline is None and t.poll() == False) |
| 657 | |
| 658 | |
Fang Deng | 0454e63 | 2014-04-07 15:39:47 -0700 | [diff] [blame] | 659 | if __name__ == '__main__': |
| 660 | unittest.main() |