blob: 52dfc9d1b748f83d62af5d324ac3be71d9266cba [file] [log] [blame]
Fang Deng0454e632014-04-07 15:39:47 -07001#!/usr/bin/python
2# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
Allen Licc205492017-07-10 17:26:04 -07006import collections
Prashanth Ba7be2072014-07-15 15:03:21 -07007import datetime as datetime_base
8from datetime import datetime
Fang Deng0454e632014-04-07 15:39:47 -07009import mock
Prashanth Ba7be2072014-07-15 15:03:21 -070010import time
Fang Deng0454e632014-04-07 15:39:47 -070011import unittest
12
13import common
14
15from autotest_lib.server.cros.dynamic_suite import constants
16from autotest_lib.site_utils import run_suite
Prashanth Ba7be2072014-07-15 15:03:21 -070017from autotest_lib.site_utils import diagnosis_utils
Fang Deng0454e632014-04-07 15:39:47 -070018
19
Allen Licc205492017-07-10 17:26:04 -070020class ReturnResultUnittest(unittest.TestCase):
21 """_ReturnResult tests."""
22
23 def setUp(self):
24 super(ReturnResultUnittest, self).setUp()
25 patcher = mock.patch.object(run_suite, '_RETURN_RESULTS',
26 collections.OrderedDict())
27 self.results = results = patcher.start()
28 self.addCleanup(patcher.stop)
29 results['small'] = run_suite._ReturnResult(0, 'small')
30 results['big'] = run_suite._ReturnResult(1, 'big')
31
32 patcher = mock.patch.object(run_suite, '_RETURN_RESULTS_LIST',
33 list(results.values()))
34 patcher.start()
35 self.addCleanup(patcher.stop)
36
37 def test_equal(self):
38 """Test _ReturnResult equal."""
39 self.assertEqual(self.results['small'], self.results['small'])
40
41 def test_unequal(self):
42 """Test _ReturnResult unequal."""
43 self.assertNotEqual(self.results['big'], self.results['small'])
44
45 def test_greater_than(self):
46 """Test _ReturnResult greater than."""
47 self.assertGreater(self.results['big'], self.results['small'])
48
49 def test_bitwise_or(self):
50 """Test _ReturnResult bitwise or."""
51 self.assertEqual(self.results['big'],
52 self.results['big'] | self.results['small'])
53
54
Fang Deng0454e632014-04-07 15:39:47 -070055class ResultCollectorUnittest(unittest.TestCase):
56 """Runsuite unittest"""
57
Fang Dengf8503532014-06-12 18:21:55 -070058 JOB_MAX_RUNTIME_MINS = 10
59
Fang Deng0454e632014-04-07 15:39:47 -070060 def setUp(self):
61 """Set up test."""
62 self.afe = mock.MagicMock()
63 self.tko = mock.MagicMock()
64
65
66 def _build_view(self, test_idx, test_name, subdir, status, afe_job_id,
67 job_name='fake_job_name', reason='fake reason',
68 job_keyvals=None, test_started_time=None,
Fang Dengf8503532014-06-12 18:21:55 -070069 test_finished_time=None, invalidates_test_idx=None,
70 job_started_time=None, job_finished_time=None):
Fang Deng0454e632014-04-07 15:39:47 -070071 """Build a test view using the given fields.
72
73 @param test_idx: An integer representing test_idx.
74 @param test_name: A string, e.g. 'dummy_Pass'
75 @param subdir: A string representing the subdir field of the test view.
76 e.g. 'dummy_Pass'.
77 @param status: A string representing the test status.
78 e.g. 'FAIL', 'PASS'
79 @param afe_job_id: An integer representing the afe job id.
80 @param job_name: A string representing the job name.
81 @param reason: A string representing the reason field of the test view.
82 @param job_keyvals: A dictionary stroing the job keyvals.
83 @param test_started_time: A string, e.g. '2014-04-12 12:35:33'
84 @param test_finished_time: A string, e.g. '2014-04-12 12:35:33'
Fang Dengaeab6172014-05-07 17:17:04 -070085 @param invalidates_test_idx: An integer, representing the idx of the
86 test that has been retried.
Fang Dengf8503532014-06-12 18:21:55 -070087 @param job_started_time: A string, e.g. '2014-04-12 12:35:33'
88 @param job_finished_time: A string, e.g. '2014-04-12 12:35:33'
Fang Deng0454e632014-04-07 15:39:47 -070089
90 @reutrn: A dictionary representing a test view.
91
92 """
93 if job_keyvals is None:
94 job_keyvals = {}
95 return {'test_idx': test_idx, 'test_name': test_name, 'subdir':subdir,
96 'status': status, 'afe_job_id': afe_job_id,
97 'job_name': job_name, 'reason': reason,
98 'job_keyvals': job_keyvals,
99 'test_started_time': test_started_time,
Fang Dengaeab6172014-05-07 17:17:04 -0700100 'test_finished_time': test_finished_time,
Fang Dengf8503532014-06-12 18:21:55 -0700101 'invalidates_test_idx': invalidates_test_idx,
102 'job_started_time': job_started_time,
103 'job_finished_time': job_finished_time}
Fang Deng0454e632014-04-07 15:39:47 -0700104
105
David Rileydcd1a642017-03-01 23:15:08 -0800106 def _mock_tko_get_detailed_test_views(self, test_views,
107 missing_results=[]):
Fang Deng0454e632014-04-07 15:39:47 -0700108 """Mock tko method get_detailed_test_views call.
109
110 @param test_views: A list of test views that will be returned
111 by get_detailed_test_views.
112 """
113 return_values = {}
114 for v in test_views:
115 views_of_job = return_values.setdefault(
116 ('get_detailed_test_views', v['afe_job_id']), [])
117 views_of_job.append(v)
David Rileydcd1a642017-03-01 23:15:08 -0800118 for job_id in missing_results:
119 views_of_job = return_values.setdefault(
120 ('get_detailed_test_views', job_id), [])
Fang Deng0454e632014-04-07 15:39:47 -0700121
122 def side_effect(*args, **kwargs):
123 """Maps args and kwargs to the mocked return values."""
124 key = (kwargs['call'], kwargs['afe_job_id'])
125 return return_values[key]
126
127 self.tko.run = mock.MagicMock(side_effect=side_effect)
128
129
130 def _mock_afe_get_jobs(self, suite_job_id, child_job_ids):
131 """Mock afe get_jobs call.
132
133 @param suite_job_id: The afe job id of the suite job.
134 @param child_job_ids: A list of job ids of the child jobs.
135
136 """
Fang Dengf8503532014-06-12 18:21:55 -0700137 suite_job = mock.MagicMock()
138 suite_job.id = suite_job_id
139 suite_job.max_runtime_mins = 10
140 suite_job.parent_job = None
141
Fang Deng0454e632014-04-07 15:39:47 -0700142 return_values = {suite_job_id: []}
143 for job_id in child_job_ids:
144 new_job = mock.MagicMock()
145 new_job.id = job_id
David Rileydcd1a642017-03-01 23:15:08 -0800146 new_job.name = 'test.%d' % job_id
Fang Dengf8503532014-06-12 18:21:55 -0700147 new_job.max_runtime_mins = self.JOB_MAX_RUNTIME_MINS
148 new_job.parent_job = suite_job
Fang Deng0454e632014-04-07 15:39:47 -0700149 return_values[suite_job_id].append(new_job)
150
151 def side_effect(*args, **kwargs):
152 """Maps args and kwargs to the mocked return values."""
Fang Dengf8503532014-06-12 18:21:55 -0700153 if kwargs.get('id') == suite_job_id:
154 return [suite_job]
Fang Deng0454e632014-04-07 15:39:47 -0700155 return return_values[kwargs['parent_job_id']]
156
157 self.afe.get_jobs = mock.MagicMock(side_effect=side_effect)
158
159
160 def testFetchSuiteTestView(self):
161 """Test that it fetches the correct suite test views."""
162 suite_job_id = 100
Fang Dengaeab6172014-05-07 17:17:04 -0700163 suite_name = 'dummy'
164 build = 'R23-1.1.1.1'
Fang Deng0454e632014-04-07 15:39:47 -0700165 server_job_view = self._build_view(
166 10, 'SERVER_JOB', '----', 'GOOD', suite_job_id)
167 test_to_ignore = self._build_view(
168 11, 'dummy_Pass', '101-user/host/dummy_Pass',
169 'GOOD', suite_job_id)
170 test_to_include = self._build_view(
171 12, 'dummy_Pass.bluetooth', None, 'TEST_NA', suite_job_id)
David Rileydcd1a642017-03-01 23:15:08 -0800172 test_missing = self._build_view(
173 13, 'dummy_Missing', None, 'ABORT', suite_job_id)
Fang Dengf8503532014-06-12 18:21:55 -0700174 self._mock_afe_get_jobs(suite_job_id, [])
Fang Deng0454e632014-04-07 15:39:47 -0700175 self._mock_tko_get_detailed_test_views(
David Rileydcd1a642017-03-01 23:15:08 -0800176 [server_job_view, test_to_ignore, test_to_include,
177 test_missing])
Fang Deng0454e632014-04-07 15:39:47 -0700178 collector = run_suite.ResultCollector(
179 'fake_server', self.afe, self.tko,
Dan Shi9b620c22017-10-10 10:58:37 -0700180 build=build, board='fake', suite_name=suite_name,
Allen Li0b675b62017-07-05 13:38:04 -0700181 suite_job_id=suite_job_id,
182 return_code_function=run_suite._ReturnCodeComputer())
David Rileydcd1a642017-03-01 23:15:08 -0800183 collector._missing_results = {
184 test_missing['test_name']: [14, 15],
185 }
Fang Deng0454e632014-04-07 15:39:47 -0700186 suite_views = collector._fetch_relevant_test_views_of_suite()
187 suite_views = sorted(suite_views, key=lambda view: view['test_idx'])
Shuqian Zhaoc085abb2016-02-24 11:27:26 -0800188 # Verify that SERVER_JOB is renamed to 'Suite job'
Fang Dengaeab6172014-05-07 17:17:04 -0700189 self.assertEqual(suite_views[0].get_testname(),
Shuqian Zhaoc085abb2016-02-24 11:27:26 -0800190 run_suite.TestView.SUITE_JOB)
Fang Deng0454e632014-04-07 15:39:47 -0700191 # Verify that the test with a subidr is not included.
Fang Dengaeab6172014-05-07 17:17:04 -0700192 self.assertEqual(suite_views[0]['test_idx'], 10)
193 self.assertEqual(suite_views[1]['test_idx'], 12)
David Rileydcd1a642017-03-01 23:15:08 -0800194 self.assertEqual(suite_views[1]['afe_job_id'], suite_job_id)
195 # Verify that the test with missing results had it's AFE job id
196 # replaced.
197 self.assertEqual(suite_views[2]['test_idx'], 13)
198 self.assertEqual(suite_views[2]['afe_job_id'], 14)
Fang Deng0454e632014-04-07 15:39:47 -0700199
200
201 def testFetchTestViewOfChildJobs(self):
202 """Test that it fetches the correct child test views."""
203 build = 'lumpy-release/R36-5788.0.0'
MK Ryu977a9752014-10-21 11:58:09 -0700204 board = 'lumpy'
Fang Deng0454e632014-04-07 15:39:47 -0700205 suite_name = 'my_suite'
206 suite_job_id = 100
Fang Dengaeab6172014-05-07 17:17:04 -0700207 invalid_job_id = 101
208 invalid_job_name = '%s/%s/test_Pass' % (build, suite_name)
209 good_job_id = 102
Fang Deng0454e632014-04-07 15:39:47 -0700210 good_job_name = '%s/%s/test_Pass' % (build, suite_name)
Fang Dengaeab6172014-05-07 17:17:04 -0700211 bad_job_id = 103
Fang Deng0454e632014-04-07 15:39:47 -0700212 bad_job_name = '%s/%s/test_ServerJobFail' % (build, suite_name)
David Rileydcd1a642017-03-01 23:15:08 -0800213 missing_job_id = 104
Fang Deng0454e632014-04-07 15:39:47 -0700214
Fang Dengaeab6172014-05-07 17:17:04 -0700215 invalid_test = self._build_view(
216 19, 'test_Pass_Old', 'fake/subdir',
217 'FAIL', invalid_job_id, invalid_job_name)
Fang Deng0454e632014-04-07 15:39:47 -0700218 good_job_server_job = self._build_view(
219 20, 'SERVER_JOB', '----', 'GOOD', good_job_id, good_job_name)
220 good_job_test = self._build_view(
Fang Dengaeab6172014-05-07 17:17:04 -0700221 21, 'test_Pass', 'fake/subdir', 'GOOD',
222 good_job_id, good_job_name,
223 job_keyvals={'retry_original_job_id': invalid_job_id})
Fang Deng0454e632014-04-07 15:39:47 -0700224 bad_job_server_job = self._build_view(
225 22, 'SERVER_JOB', '----', 'FAIL', bad_job_id, bad_job_name)
226 bad_job_test = self._build_view(
227 23, 'test_ServerJobFail', 'fake/subdir', 'GOOD',
228 bad_job_id, bad_job_name)
229 self._mock_tko_get_detailed_test_views(
230 [good_job_server_job, good_job_test,
David Rileydcd1a642017-03-01 23:15:08 -0800231 bad_job_server_job, bad_job_test, invalid_test],
232 [missing_job_id])
233 self._mock_afe_get_jobs(suite_job_id,
234 [good_job_id, bad_job_id, missing_job_id])
Fang Deng0454e632014-04-07 15:39:47 -0700235 collector = run_suite.ResultCollector(
236 'fake_server', self.afe, self.tko,
Allen Li0b675b62017-07-05 13:38:04 -0700237 build, board, suite_name, suite_job_id,
238 return_code_function=run_suite._ReturnCodeComputer())
David Rileydcd1a642017-03-01 23:15:08 -0800239 child_views, retry_counts, missing_results = (
240 collector._fetch_test_views_of_child_jobs())
Fang Dengaeab6172014-05-07 17:17:04 -0700241 # child_views should contain tests 21, 22, 23
Fang Deng0454e632014-04-07 15:39:47 -0700242 child_views = sorted(child_views, key=lambda view: view['test_idx'])
243 # Verify that the SERVER_JOB has been renamed properly
Fang Dengaeab6172014-05-07 17:17:04 -0700244 self.assertEqual(child_views[1].get_testname(),
245 'test_ServerJobFail_SERVER_JOB')
David Rileydcd1a642017-03-01 23:15:08 -0800246 self.assertEqual(missing_results, {'test.104': [104]})
Fang Dengaeab6172014-05-07 17:17:04 -0700247 # Verify that failed SERVER_JOB and actual invalid tests are included,
248 expected = [good_job_test['test_idx'], bad_job_server_job['test_idx'],
249 bad_job_test['test_idx']]
250 child_view_ids = [v['test_idx'] for v in child_views]
251 self.assertEqual(child_view_ids, expected)
252 self.afe.get_jobs.assert_called_once_with(
253 parent_job_id=suite_job_id)
254 # Verify the retry_counts is calculated correctly
255 self.assertEqual(len(retry_counts), 1)
256 self.assertEqual(retry_counts[21], 1)
Fang Deng0454e632014-04-07 15:39:47 -0700257
258
259 def testGenerateLinks(self):
260 """Test that it generates correct web and buildbot links."""
261 suite_job_id = 100
Fang Dengaeab6172014-05-07 17:17:04 -0700262 suite_name = 'my_suite'
263 build = 'lumpy-release/R36-5788.0.0'
MK Ryu977a9752014-10-21 11:58:09 -0700264 board = 'lumpy'
Fang Dengf8503532014-06-12 18:21:55 -0700265 fake_job = mock.MagicMock()
266 fake_job.parent = suite_job_id
Dan Shi9b620c22017-10-10 10:58:37 -0700267 test_sponge_url = 'http://test_url'
268 job_keyvals = {'sponge_url': test_sponge_url}
Fang Dengaeab6172014-05-07 17:17:04 -0700269 suite_job_view = run_suite.TestView(
270 self._build_view(
Dan Shi9b620c22017-10-10 10:58:37 -0700271 20, 'Suite job', '----', 'GOOD', suite_job_id,
272 job_keyvals=job_keyvals),
Simran Basi01984f52015-10-12 15:36:45 -0700273 fake_job, suite_name, build, 'chromeos-test')
Fang Dengaeab6172014-05-07 17:17:04 -0700274 good_test = run_suite.TestView(
275 self._build_view(
Dan Shi9b620c22017-10-10 10:58:37 -0700276 21, 'test_Pass', 'fake/subdir', 'GOOD', 101,
277 job_keyvals=job_keyvals),
Simran Basi01984f52015-10-12 15:36:45 -0700278 fake_job, suite_name, build, 'chromeos-test')
Fang Dengaeab6172014-05-07 17:17:04 -0700279 bad_test = run_suite.TestView(
280 self._build_view(
Dan Shi9b620c22017-10-10 10:58:37 -0700281 23, 'test_Fail', 'fake/subdir', 'FAIL', 102,
282 job_keyvals=job_keyvals),
Simran Basi01984f52015-10-12 15:36:45 -0700283 fake_job, suite_name, build, 'chromeos-test')
Fang Deng0454e632014-04-07 15:39:47 -0700284
285 collector = run_suite.ResultCollector(
286 'fake_server', self.afe, self.tko,
Allen Li0b675b62017-07-05 13:38:04 -0700287 build, board, suite_name, suite_job_id, user='chromeos-test',
288 return_code_function=run_suite._ReturnCodeComputer())
Fang Deng0454e632014-04-07 15:39:47 -0700289 collector._suite_views = [suite_job_view]
290 collector._test_views = [suite_job_view, good_test, bad_test]
291 collector._max_testname_width = max(
Fang Dengaeab6172014-05-07 17:17:04 -0700292 [len(v.get_testname()) for v in collector._test_views]) + 3
Fang Deng0454e632014-04-07 15:39:47 -0700293 collector._generate_web_and_buildbot_links()
Allen Lidc2c69a2016-09-14 19:05:47 -0700294 URL_PATTERN = run_suite._URL_PATTERN
Fang Deng0454e632014-04-07 15:39:47 -0700295 # expected_web_links is list of (anchor, url) tuples we
296 # are expecting.
297 expected_web_links = [
Allen Li34613242016-09-02 11:52:34 -0700298 (v.get_testname(),
Fang Deng0454e632014-04-07 15:39:47 -0700299 URL_PATTERN % ('fake_server',
Dan Shi9b620c22017-10-10 10:58:37 -0700300 '%s-%s' % (v['afe_job_id'], 'chromeos-test')),
301 test_sponge_url)
Fang Deng0454e632014-04-07 15:39:47 -0700302 for v in collector._test_views]
303 # Verify web links are generated correctly.
304 for i in range(len(collector._web_links)):
305 expect = expected_web_links[i]
306 self.assertEqual(collector._web_links[i].anchor, expect[0])
307 self.assertEqual(collector._web_links[i].url, expect[1])
Dan Shi9b620c22017-10-10 10:58:37 -0700308 self.assertEqual(collector._web_links[i].sponge_url, expect[2])
Fang Deng0454e632014-04-07 15:39:47 -0700309
310 expected_buildbot_links = [
Allen Li34613242016-09-02 11:52:34 -0700311 (v.get_testname(),
Fang Deng0454e632014-04-07 15:39:47 -0700312 URL_PATTERN % ('fake_server',
Simran Basi01984f52015-10-12 15:36:45 -0700313 '%s-%s' % (v['afe_job_id'], 'chromeos-test')))
Fang Deng0454e632014-04-07 15:39:47 -0700314 for v in collector._test_views if v['status'] != 'GOOD']
315 # Verify buildbot links are generated correctly.
Allen Li28be0642017-07-10 15:16:26 -0700316 for i in range(len(collector.buildbot_links)):
Fang Deng0454e632014-04-07 15:39:47 -0700317 expect = expected_buildbot_links[i]
Allen Li28be0642017-07-10 15:16:26 -0700318 self.assertEqual(collector.buildbot_links[i].anchor, expect[0])
319 self.assertEqual(collector.buildbot_links[i].url, expect[1])
320 self.assertEqual(collector.buildbot_links[i].retry_count, 0)
Shuhei Takahashi18f56492017-11-14 16:23:46 +0900321 # Assert that a retry dashboard link is created.
Simran Basi7203d4e2015-02-03 15:50:18 -0800322 self.assertNotEqual(
Shuhei Takahashi18f56492017-11-14 16:23:46 +0900323 collector.buildbot_links[i].GenerateRetryLink(), '')
David Rileya0cd1c22017-07-10 11:15:57 -0700324 self.assertNotEqual(
Shuhei Takahashi18f56492017-11-14 16:23:46 +0900325 collector.buildbot_links[i].GenerateHistoryLink(), '')
Fang Deng0454e632014-04-07 15:39:47 -0700326
327
Fang Deng5a43be62014-05-07 17:17:04 -0700328 def _end_to_end_test_helper(
329 self, include_bad_test=False, include_warn_test=False,
Allen Lie6236ec2017-07-05 12:52:36 -0700330 include_timeout_test=False,
Fang Dengaeab6172014-05-07 17:17:04 -0700331 include_self_aborted_test=False,
Fang Dengf8503532014-06-12 18:21:55 -0700332 include_aborted_by_suite_test=False,
Fang Dengaeab6172014-05-07 17:17:04 -0700333 include_good_retry=False, include_bad_retry=False,
Brian Norris58ae5162017-09-26 13:34:56 -0700334 include_good_test=True,
Fang Dengf8503532014-06-12 18:21:55 -0700335 suite_job_timed_out=False, suite_job_status='GOOD'):
Fang Deng0454e632014-04-07 15:39:47 -0700336 """A helper method for testing ResultCollector end-to-end.
337
338 This method mocks the retrieving of required test views,
339 and call ResultCollector.run() to collect the results.
340
Fang Dengaeab6172014-05-07 17:17:04 -0700341 @param include_bad_test:
342 If True, include a view of a test which has status 'FAIL'.
343 @param include_warn_test:
344 If True, include a view of a test which has status 'WARN'
Fang Dengaeab6172014-05-07 17:17:04 -0700345 @param include_timeout_test:
346 If True, include a view of a test which was aborted before
347 started.
348 @param include_self_aborted_test:
349 If True, include a view of test which was aborted after
Fang Dengf8503532014-06-12 18:21:55 -0700350 started and hit hits own timeout.
351 @param include_self_aborted_by_suite_test:
352 If True, include a view of test which was aborted after
353 started but has not hit its own timeout.
Fang Dengaeab6172014-05-07 17:17:04 -0700354 @param include_good_retry:
355 If True, include a test that passed after retry.
356 @param include_bad_retry:
357 If True, include a test that failed after retry.
Brian Norris58ae5162017-09-26 13:34:56 -0700358 @param include_good_test:
359 If True, include a test that passed. If False, pretend no tests
360 (including the parent suite job) came back with any test
361 results.
Fang Dengf8503532014-06-12 18:21:55 -0700362 @param suite_job_status: One of 'GOOD' 'FAIL' 'ABORT' 'RUNNING'
Fang Deng0454e632014-04-07 15:39:47 -0700363
Fang Dengaeab6172014-05-07 17:17:04 -0700364 @returns: A ResultCollector instance.
Fang Deng0454e632014-04-07 15:39:47 -0700365 """
366 suite_job_id = 100
367 good_job_id = 101
368 bad_job_id = 102
Fang Deng5a43be62014-05-07 17:17:04 -0700369 warn_job_id = 102
Fang Deng5a43be62014-05-07 17:17:04 -0700370 timeout_job_id = 100
371 self_aborted_job_id = 104
Fang Dengf8503532014-06-12 18:21:55 -0700372 aborted_by_suite_job_id = 105
373 good_retry_job_id = 106
374 bad_retry_job_id = 107
Fang Dengaeab6172014-05-07 17:17:04 -0700375 invalid_job_id_1 = 90
376 invalid_job_id_2 = 91
377 suite_name = 'dummy'
378 build = 'lumpy-release/R27-3888.0.0'
Fang Deng0454e632014-04-07 15:39:47 -0700379 suite_job_keyvals = {
380 constants.DOWNLOAD_STARTED_TIME: '2014-04-29 13:14:20',
381 constants.PAYLOAD_FINISHED_TIME: '2014-04-29 13:14:25',
382 constants.ARTIFACT_FINISHED_TIME: '2014-04-29 13:14:30'}
383
Fang Dengf8503532014-06-12 18:21:55 -0700384 suite_job_started_time = '2014-04-29 13:14:37'
385 if suite_job_timed_out:
386 suite_job_keyvals['aborted_by'] = 'test_user'
387 suite_job_finished_time = '2014-04-29 13:25:37'
388 suite_job_status = 'ABORT'
389 else:
390 suite_job_finished_time = '2014-04-29 13:23:37'
391
Fang Deng0454e632014-04-07 15:39:47 -0700392 server_job_view = self._build_view(
Fang Deng5a43be62014-05-07 17:17:04 -0700393 10, 'SERVER_JOB', '----', suite_job_status, suite_job_id,
Fang Deng0454e632014-04-07 15:39:47 -0700394 'lumpy-release/R27-3888.0.0-test_suites/control.dummy',
395 '', suite_job_keyvals, '2014-04-29 13:14:37',
Fang Dengf8503532014-06-12 18:21:55 -0700396 '2014-04-29 13:20:27', job_started_time=suite_job_started_time,
397 job_finished_time=suite_job_finished_time)
Fang Deng0454e632014-04-07 15:39:47 -0700398 good_test = self._build_view(
399 11, 'dummy_Pass', '101-user/host/dummy_Pass', 'GOOD',
Fang Dengaeab6172014-05-07 17:17:04 -0700400 good_job_id, 'lumpy-release/R27-3888.0.0/dummy/dummy_Pass',
Fang Deng0454e632014-04-07 15:39:47 -0700401 '', {}, '2014-04-29 13:15:35', '2014-04-29 13:15:36')
402 bad_test = self._build_view(
403 12, 'dummy_Fail.Fail', '102-user/host/dummy_Fail.Fail', 'FAIL',
Fang Dengaeab6172014-05-07 17:17:04 -0700404 bad_job_id, 'lumpy-release/R27-3888.0.0/dummy/dummy_Fail.Fail',
Fang Deng0454e632014-04-07 15:39:47 -0700405 'always fail', {}, '2014-04-29 13:16:00',
406 '2014-04-29 13:16:02')
407 warn_test = self._build_view(
Fang Deng5a43be62014-05-07 17:17:04 -0700408 13, 'dummy_Fail.Warn', '102-user/host/dummy_Fail.Warn', 'WARN',
Fang Dengaeab6172014-05-07 17:17:04 -0700409 warn_job_id, 'lumpy-release/R27-3888.0.0/dummy/dummy_Fail.Warn',
Fang Deng0454e632014-04-07 15:39:47 -0700410 'always warn', {}, '2014-04-29 13:16:00',
411 '2014-04-29 13:16:02')
Fang Deng5a43be62014-05-07 17:17:04 -0700412 timeout_test = self._build_view(
413 15, 'dummy_Timeout', '', 'ABORT',
Fang Dengaeab6172014-05-07 17:17:04 -0700414 timeout_job_id,
415 'lumpy-release/R27-3888.0.0/dummy/dummy_Timeout',
Fang Deng5a43be62014-05-07 17:17:04 -0700416 'child job did not run', {}, '2014-04-29 13:15:37',
417 '2014-04-29 13:15:38')
418 self_aborted_test = self._build_view(
419 16, 'dummy_Abort', '104-user/host/dummy_Abort', 'ABORT',
Fang Dengaeab6172014-05-07 17:17:04 -0700420 self_aborted_job_id,
421 'lumpy-release/R27-3888.0.0/dummy/dummy_Abort',
Fang Dengf8503532014-06-12 18:21:55 -0700422 'child job aborted', {'aborted_by': 'test_user'},
423 '2014-04-29 13:15:39', '2014-04-29 13:15:40',
424 job_started_time='2014-04-29 13:15:39',
425 job_finished_time='2014-04-29 13:25:40')
426 aborted_by_suite = self._build_view(
427 17, 'dummy_AbortBySuite', '105-user/host/dummy_AbortBySuite',
428 'RUNNING', aborted_by_suite_job_id,
429 'lumpy-release/R27-3888.0.0/dummy/dummy_Abort',
430 'aborted by suite', {'aborted_by': 'test_user'},
431 '2014-04-29 13:15:39', '2014-04-29 13:15:40',
432 job_started_time='2014-04-29 13:15:39',
433 job_finished_time='2014-04-29 13:15:40')
Fang Dengaeab6172014-05-07 17:17:04 -0700434 good_retry = self._build_view(
Fang Dengf8503532014-06-12 18:21:55 -0700435 18, 'dummy_RetryPass', '106-user/host/dummy_RetryPass', 'GOOD',
Fang Dengaeab6172014-05-07 17:17:04 -0700436 good_retry_job_id,
437 'lumpy-release/R27-3888.0.0/dummy/dummy_RetryPass',
438 '', {'retry_original_job_id': invalid_job_id_1},
439 '2014-04-29 13:15:37',
440 '2014-04-29 13:15:38', invalidates_test_idx=1)
441 bad_retry = self._build_view(
Fang Dengf8503532014-06-12 18:21:55 -0700442 19, 'dummy_RetryFail', '107-user/host/dummy_RetryFail', 'FAIL',
Fang Dengaeab6172014-05-07 17:17:04 -0700443 bad_retry_job_id,
444 'lumpy-release/R27-3888.0.0/dummy/dummy_RetryFail',
445 'retry failed', {'retry_original_job_id': invalid_job_id_2},
446 '2014-04-29 13:15:39', '2014-04-29 13:15:40',
447 invalidates_test_idx=2)
448 invalid_test_1 = self._build_view(
449 1, 'dummy_RetryPass', '90-user/host/dummy_RetryPass', 'GOOD',
450 invalid_job_id_1,
451 'lumpy-release/R27-3888.0.0/dummy/dummy_RetryPass',
452 'original test failed', {}, '2014-04-29 13:10:00',
453 '2014-04-29 13:10:01')
454 invalid_test_2 = self._build_view(
455 2, 'dummy_RetryFail', '91-user/host/dummy_RetryFail', 'FAIL',
456 invalid_job_id_2,
457 'lumpy-release/R27-3888.0.0/dummy/dummy_RetryFail',
458 'original test failed', {},
459 '2014-04-29 13:10:03', '2014-04-29 13:10:04')
460
Brian Norris58ae5162017-09-26 13:34:56 -0700461 test_views = []
462 child_jobs = set()
463 missing_results = []
464 if include_good_test:
465 test_views.append(server_job_view)
466 test_views.append(good_test)
467 child_jobs.add(good_job_id)
468 # Emulate missing even the parent/suite job.
469 else:
470 missing_results.append(suite_job_id)
Fang Deng0454e632014-04-07 15:39:47 -0700471 if include_bad_test:
472 test_views.append(bad_test)
Fang Deng5a43be62014-05-07 17:17:04 -0700473 child_jobs.add(bad_job_id)
Fang Deng0454e632014-04-07 15:39:47 -0700474 if include_warn_test:
475 test_views.append(warn_test)
Fang Deng5a43be62014-05-07 17:17:04 -0700476 child_jobs.add(warn_job_id)
Fang Deng5a43be62014-05-07 17:17:04 -0700477 if include_timeout_test:
478 test_views.append(timeout_test)
Fang Deng5a43be62014-05-07 17:17:04 -0700479 if include_self_aborted_test:
480 test_views.append(self_aborted_test)
481 child_jobs.add(self_aborted_job_id)
Fang Dengaeab6172014-05-07 17:17:04 -0700482 if include_good_retry:
483 test_views.extend([good_retry, invalid_test_1])
484 child_jobs.add(good_retry_job_id)
485 if include_bad_retry:
486 test_views.extend([bad_retry, invalid_test_2])
487 child_jobs.add(bad_retry_job_id)
Fang Dengf8503532014-06-12 18:21:55 -0700488 if include_aborted_by_suite_test:
489 test_views.append(aborted_by_suite)
490 child_jobs.add(aborted_by_suite_job_id)
Brian Norris58ae5162017-09-26 13:34:56 -0700491 self._mock_tko_get_detailed_test_views(test_views,
492 missing_results=missing_results)
Fang Deng0454e632014-04-07 15:39:47 -0700493 self._mock_afe_get_jobs(suite_job_id, child_jobs)
494 collector = run_suite.ResultCollector(
495 'fake_server', self.afe, self.tko,
Allen Li0b675b62017-07-05 13:38:04 -0700496 'lumpy-release/R36-5788.0.0', 'lumpy', 'dummy', suite_job_id,
497 return_code_function=run_suite._ReturnCodeComputer())
Fang Deng0454e632014-04-07 15:39:47 -0700498 collector.run()
Brian Norris58ae5162017-09-26 13:34:56 -0700499 collector.output_results()
Fang Deng0454e632014-04-07 15:39:47 -0700500 return collector
501
502
Brian Norris58ae5162017-09-26 13:34:56 -0700503 def testEndToEndSuiteEmpty(self):
504 """Test it returns code INFRA_FAILURE when no tests report back."""
505 collector = self._end_to_end_test_helper(include_good_test=False)
506 self.assertEqual(collector.return_result.return_code,
507 run_suite.RETURN_CODES.INFRA_FAILURE)
508
509
Fang Deng0454e632014-04-07 15:39:47 -0700510 def testEndToEndSuitePass(self):
511 """Test it returns code OK when all test pass."""
512 collector = self._end_to_end_test_helper()
Allen Licc205492017-07-10 17:26:04 -0700513 self.assertEqual(collector.return_result.return_code,
514 run_suite.RETURN_CODES.OK)
Fang Deng0454e632014-04-07 15:39:47 -0700515
516
Fang Deng0454e632014-04-07 15:39:47 -0700517 def testEndToEndSuiteWarn(self):
518 """Test it returns code WARNING when there is a test that warns."""
519 collector = self._end_to_end_test_helper(include_warn_test=True)
Allen Licc205492017-07-10 17:26:04 -0700520 self.assertEqual(collector.return_result.return_code,
521 run_suite.RETURN_CODES.WARNING)
Fang Deng0454e632014-04-07 15:39:47 -0700522
523
524 def testEndToEndSuiteFail(self):
525 """Test it returns code ERROR when there is a test that fails."""
Fang Deng0454e632014-04-07 15:39:47 -0700526 collector = self._end_to_end_test_helper(include_bad_test=True)
Allen Licc205492017-07-10 17:26:04 -0700527 self.assertEqual(collector.return_result.return_code,
528 run_suite.RETURN_CODES.ERROR)
Fang Deng0454e632014-04-07 15:39:47 -0700529
Fang Deng0454e632014-04-07 15:39:47 -0700530
Fang Deng5a43be62014-05-07 17:17:04 -0700531 def testEndToEndSuiteJobFail(self):
532 """Test it returns code SUITE_FAILURE when only the suite job failed."""
533 collector = self._end_to_end_test_helper(suite_job_status='ABORT')
534 self.assertEqual(
Allen Licc205492017-07-10 17:26:04 -0700535 collector.return_result.return_code,
536 run_suite.RETURN_CODES.INFRA_FAILURE)
Fang Deng5a43be62014-05-07 17:17:04 -0700537
538 collector = self._end_to_end_test_helper(suite_job_status='ERROR')
539 self.assertEqual(
Allen Licc205492017-07-10 17:26:04 -0700540 collector.return_result.return_code,
541 run_suite.RETURN_CODES.INFRA_FAILURE)
Fang Deng5a43be62014-05-07 17:17:04 -0700542
543
Fang Dengaeab6172014-05-07 17:17:04 -0700544 def testEndToEndRetry(self):
545 """Test it returns correct code when a test was retried."""
546 collector = self._end_to_end_test_helper(include_good_retry=True)
547 self.assertEqual(
Allen Licc205492017-07-10 17:26:04 -0700548 collector.return_result.return_code,
549 run_suite.RETURN_CODES.WARNING)
Fang Dengaeab6172014-05-07 17:17:04 -0700550
551 collector = self._end_to_end_test_helper(include_good_retry=True,
552 include_self_aborted_test=True)
553 self.assertEqual(
Allen Licc205492017-07-10 17:26:04 -0700554 collector.return_result.return_code,
555 run_suite.RETURN_CODES.ERROR)
Fang Dengaeab6172014-05-07 17:17:04 -0700556
557 collector = self._end_to_end_test_helper(include_good_retry=True,
558 include_bad_test=True)
559 self.assertEqual(
Allen Licc205492017-07-10 17:26:04 -0700560 collector.return_result.return_code,
561 run_suite.RETURN_CODES.ERROR)
Fang Dengaeab6172014-05-07 17:17:04 -0700562
563 collector = self._end_to_end_test_helper(include_bad_retry=True)
564 self.assertEqual(
Allen Licc205492017-07-10 17:26:04 -0700565 collector.return_result.return_code,
566 run_suite.RETURN_CODES.ERROR)
Fang Dengaeab6172014-05-07 17:17:04 -0700567
568
Fang Deng5a43be62014-05-07 17:17:04 -0700569 def testEndToEndSuiteTimeout(self):
570 """Test it returns correct code when a child job timed out."""
Fang Dengf8503532014-06-12 18:21:55 -0700571 # a child job timed out before started, none failed.
Fang Deng5a43be62014-05-07 17:17:04 -0700572 collector = self._end_to_end_test_helper(include_timeout_test=True)
573 self.assertEqual(
Allen Licc205492017-07-10 17:26:04 -0700574 collector.return_result.return_code,
575 run_suite.RETURN_CODES.SUITE_TIMEOUT)
Fang Deng5a43be62014-05-07 17:17:04 -0700576
Fang Dengf8503532014-06-12 18:21:55 -0700577 # a child job timed out before started, and one test failed.
Fang Deng5a43be62014-05-07 17:17:04 -0700578 collector = self._end_to_end_test_helper(
579 include_bad_test=True, include_timeout_test=True)
Allen Licc205492017-07-10 17:26:04 -0700580 self.assertEqual(collector.return_result.return_code,
581 run_suite.RETURN_CODES.ERROR)
Fang Deng5a43be62014-05-07 17:17:04 -0700582
Fang Dengf8503532014-06-12 18:21:55 -0700583 # a child job timed out before started, and one test warned.
Fang Deng5a43be62014-05-07 17:17:04 -0700584 collector = self._end_to_end_test_helper(
585 include_warn_test=True, include_timeout_test=True)
Allen Licc205492017-07-10 17:26:04 -0700586 self.assertEqual(collector.return_result.return_code,
Fang Dengaeab6172014-05-07 17:17:04 -0700587 run_suite.RETURN_CODES.SUITE_TIMEOUT)
588
Fang Dengf8503532014-06-12 18:21:55 -0700589 # a child job timed out before started, and one test was retried.
Fang Dengaeab6172014-05-07 17:17:04 -0700590 collector = self._end_to_end_test_helper(include_good_retry=True,
591 include_timeout_test=True)
Fang Deng5a43be62014-05-07 17:17:04 -0700592 self.assertEqual(
Allen Licc205492017-07-10 17:26:04 -0700593 collector.return_result.return_code,
594 run_suite.RETURN_CODES.SUITE_TIMEOUT)
Fang Deng5a43be62014-05-07 17:17:04 -0700595
Fang Dengf8503532014-06-12 18:21:55 -0700596 # a child jot was aborted because suite timed out.
597 collector = self._end_to_end_test_helper(
598 include_aborted_by_suite_test=True)
599 self.assertEqual(
Allen Licc205492017-07-10 17:26:04 -0700600 collector.return_result.return_code, run_suite.RETURN_CODES.OK)
Fang Dengf8503532014-06-12 18:21:55 -0700601
602 # suite job timed out.
603 collector = self._end_to_end_test_helper(suite_job_timed_out=True)
604 self.assertEqual(
Allen Licc205492017-07-10 17:26:04 -0700605 collector.return_result.return_code,
606 run_suite.RETURN_CODES.SUITE_TIMEOUT)
Fang Dengf8503532014-06-12 18:21:55 -0700607
Fang Deng5a43be62014-05-07 17:17:04 -0700608
Ningning Xiabd911bd2016-04-19 14:06:03 -0700609class LogLinkUnittests(unittest.TestCase):
610 """Test the LogLink"""
611
612 def testGenerateBuildbotLinks(self):
613 """Test LogLink GenerateBuildbotLinks"""
614 log_link_a = run_suite.LogLink('mock_anchor', 'mock_server',
615 'mock_job_string',
616 bug_info=('mock_bug_id', 1),
617 reason='mock_reason',
618 retry_count=1,
619 testname='mock_testname')
620 # Generate a bug link and a log link when bug_info is present
Allen Lie082ced2016-09-14 15:19:20 -0700621 self.assertTrue(len(list(log_link_a.GenerateBuildbotLinks())) == 2)
Ningning Xiabd911bd2016-04-19 14:06:03 -0700622
623 log_link_b = run_suite.LogLink('mock_anchor', 'mock_server',
624 'mock_job_string_b',
625 reason='mock_reason',
626 retry_count=1,
627 testname='mock_testname')
628 # Generate a log link when there is no bug_info
Allen Lie082ced2016-09-14 15:19:20 -0700629 self.assertTrue(len(list(log_link_b.GenerateBuildbotLinks())) == 1)
Ningning Xiabd911bd2016-04-19 14:06:03 -0700630
631
Prashanth Ba7be2072014-07-15 15:03:21 -0700632class SimpleTimerUnittests(unittest.TestCase):
633 """Test the simple timer."""
634
635 def testPoll(self):
636 """Test polling the timer."""
637 interval_hours = 0.0001
638 t = diagnosis_utils.SimpleTimer(interval_hours=interval_hours)
639 deadline = t.deadline
640 self.assertTrue(deadline is not None and
641 t.interval_hours == interval_hours)
642 min_deadline = (datetime.now() +
643 datetime_base.timedelta(hours=interval_hours))
644 time.sleep(interval_hours * 3600)
645 self.assertTrue(t.poll())
646 self.assertTrue(t.deadline >= min_deadline)
647
648
649 def testBadInterval(self):
650 """Test a bad interval."""
651 t = diagnosis_utils.SimpleTimer(interval_hours=-1)
652 self.assertTrue(t.deadline is None and t.poll() == False)
653 t._reset()
654 self.assertTrue(t.deadline is None and t.poll() == False)
655
656
Fang Deng0454e632014-04-07 15:39:47 -0700657if __name__ == '__main__':
658 unittest.main()