AUTOTEST: Check recently ran tests for health.
complete_failures.py will now only check tests that have been ran recently.
BUG=chromium:253599
DEPLOY=none
TEST=Unittests plus ran script and manually verified.
Change-Id: I475a16fd1f4d4fb2330eb94884eea4b3285ef965
Reviewed-on: https://gerrit.chromium.org/gerrit/60113
Reviewed-by: Dennis Jeffrey <dennisjeffrey@chromium.org>
Commit-Queue: Keyar Hood <keyar@chromium.org>
Tested-by: Keyar Hood <keyar@chromium.org>
diff --git a/frontend/health/complete_failures.py b/frontend/health/complete_failures.py
index c5790ec..64159d0 100644
--- a/frontend/health/complete_failures.py
+++ b/frontend/health/complete_failures.py
@@ -18,7 +18,10 @@
_STORAGE_FILE = 'failure_storage'
+# Mark a test as failing too long if it has not passed in this many days
_DAYS_TO_BE_FAILING_TOO_LONG = 60
+# Ignore any tests that have not ran in this many days
+_DAYS_NOT_RUNNING_CUTOFF = 60
_TEST_PASS_STATUS_INDEX = 6
_MAIL_RESULTS_FROM = 'chromeos-test-health@google.com'
_MAIL_RESULTS_TO = 'chromeos-lab-infrastructure@google.com'
@@ -86,22 +89,25 @@
for name in valid_test_names}
-def get_all_test_names():
+def get_recently_ran_test_names():
"""
- Get all the test names from the database.
+ Get all the test names from the database that have been recently ran.
- @return a list of all the test names.
+ @return a set of the recently ran tests.
"""
- results = tko_models.Test.objects.values('test').distinct()
+ cutoff_delta = datetime.timedelta(_DAYS_NOT_RUNNING_CUTOFF)
+ cutoff_date = datetime.datetime.today() - cutoff_delta
+ results = tko_models.Test.objects.filter(
+ started_time__gte=cutoff_date).values('test').distinct()
test_names = [test['test'] for test in results]
valid_test_names = filter(is_valid_test_name, test_names)
- return [test.encode('utf8') for test in valid_test_names]
+ return {test.encode('utf8') for test in valid_test_names}
def get_tests_to_analyze():
"""
- Get all the tests as well as the last time they have passed.
+ Get all the recently ran tests as well as the last time they have passed.
The minimum datetime is given as last pass time for tests that have never
passed.
@@ -109,11 +115,17 @@
@return the dict of test_name:last_finish_time pairs.
"""
+ recent_test_names = get_recently_ran_test_names()
last_passes = get_last_pass_times()
- all_test_names = get_all_test_names()
- failures_names = (set(all_test_names) - set(last_passes.keys()))
+
+ running_passes = {}
+ for test, pass_time in last_passes.items():
+ if test in recent_test_names:
+ running_passes[test] = pass_time
+
+ failures_names = recent_test_names.difference(running_passes)
always_failed = {test: datetime.datetime.min for test in failures_names}
- return dict(always_failed.items() + last_passes.items())
+ return dict(always_failed.items() + running_passes.items())
def email_about_test_failure(tests, storage):
diff --git a/frontend/health/complete_failures_functional_test.py b/frontend/health/complete_failures_functional_test.py
index b6c8585..45d7277 100755
--- a/frontend/health/complete_failures_functional_test.py
+++ b/frontend/health/complete_failures_functional_test.py
@@ -28,13 +28,13 @@
GOOD_STATUS_IDX = 6
FAIL_STATUS_IDX = 4
-# During the tests there is a point where a type check is done on
-# datetime.datetime. Unfortunately this means when datetime is mocked it
-# horrible failures happen when Django tries to do this check. It is necesarry
+# During the tests there is a point where Django does a type check on
+# datetime.datetime. Unfortunately this means when datetime is mocked out,
+# horrible failures happen when Django tries to do this check. The solution
+# chosen is to create a pure Python class that inheirits from datetime.datetime
+# so that the today class method can be directly mocked out. It is necesarry
# to mock out datetime.datetime completely as it a C class and so cannot have
-# parts of itself mocked out. The solution chosen is to create a pure Python
-# class that inheirits from datetime.datetime so that the today class method
-# can be directly mocked out.
+# parts of itself mocked out.
class MockDatetime(datetime.datetime):
"""Used to mock out parts of datetime.datetime."""
pass
@@ -92,13 +92,14 @@
failing_test = models.Test(job=job, status=fail_status,
kernel=kernel, machine=machine,
test='test2',
- started_time=self.datetime.min)
+ started_time=self.datetime(2012,1,1))
failing_test.save()
complete_failures._DAYS_TO_BE_FAILING_TOO_LONG = 10
storage = {}
complete_failures.load_storage().AndReturn(storage)
MockDatetime.today().AndReturn(self.datetime(2012, 1, 21))
+ MockDatetime.today().AndReturn(self.datetime(2012, 1, 21))
mail.send('chromeos-test-health@google.com',
['chromeos-lab-infrastructure@google.com'],
[],
diff --git a/frontend/health/complete_failures_unittest.py b/frontend/health/complete_failures_unittest.py
index a08344c..bce06e7 100755
--- a/frontend/health/complete_failures_unittest.py
+++ b/frontend/health/complete_failures_unittest.py
@@ -22,6 +22,12 @@
GOOD_STATUS_IDX = 6
FAIL_STATUS_IDX = 4
+# See complte_failurs_functional_tests.py for why we need this.
+class MockDatetime(datetime.datetime):
+ """Used to mock out parts of datetime.datetime."""
+ pass
+
+
class EmailAboutTestFailureTests(mox.MoxTestBase):
"""
Test the core logic of the comlete_failures.py script.
@@ -38,11 +44,11 @@
# emails will be sent out during tests.
self.mox.StubOutWithMock(mail, 'send')
- self._orignal_too_late = complete_failures._DAYS_TO_BE_FAILING_TOO_LONG
+ self._orig_too_long = complete_failures._DAYS_TO_BE_FAILING_TOO_LONG
def tearDown(self):
- complete_failures._DAYS_TO_BE_FAILING_TOO_LONG = self._orignal_too_late
+ complete_failures._DAYS_TO_BE_FAILING_TOO_LONG = self._orig_too_long
super(EmailAboutTestFailureTests, self).tearDown()
@@ -272,40 +278,50 @@
self.assertTrue(not results)
-class GetAllTestNamesTests(mox.MoxTestBase, test.TestCase):
- """Tests the get_all_test_names function."""
+class GetRecentlyRanTestNamesTests(mox.MoxTestBase, test.TestCase):
+ """Tests the get_recently_ran_test_names function."""
def setUp(self):
- super(GetAllTestNamesTests, self).setUp()
+ super(GetRecentlyRanTestNamesTests, self).setUp()
+ self.mox.StubOutWithMock(MockDatetime, 'today')
+ self.datetime = datetime.datetime
+ datetime.datetime = MockDatetime
setup_test_environment.set_up()
+ self._orig_cutoff = complete_failures._DAYS_NOT_RUNNING_CUTOFF
def tearDown(self):
+ datetime.datetime = self.datetime
+ complete_failures._DAYS_NOT_RUNNING_CUTOFF = self._orig_cutoff
setup_test_environment.tear_down()
- super(GetAllTestNamesTests, self).tearDown()
+ super(GetRecentlyRanTestNamesTests, self).tearDown()
- def test_return_all_tests(self):
+ def test_return_all_recently_ran_tests(self):
"""Test that the function does as it says it does."""
job = models.Job(job_idx=1)
kernel = models.Kernel(kernel_idx=1)
machine = models.Machine(machine_idx=1)
success_status = models.Status(status_idx=GOOD_STATUS_IDX)
- test1 = models.Test(job=job, status=success_status,
- kernel=kernel, machine=machine,
- test='test1',
- started_time=datetime.datetime(2012, 1, 1))
- test1.save()
- test2 = models.Test(job=job, status=success_status,
- kernel=kernel, machine=machine,
- test='test2',
- started_time=datetime.datetime(2012, 1, 2))
- test2.save()
+ recent = models.Test(job=job, status=success_status,
+ kernel=kernel, machine=machine,
+ test='recent',
+ started_time=self.datetime(2012, 1, 1))
+ recent.save()
+ old = models.Test(job=job, status=success_status,
+ kernel=kernel, machine=machine,
+ test='old',
+ started_time=self.datetime(2011, 1, 2))
+ old.save()
- results = complete_failures.get_all_test_names()
+ datetime.datetime.today().AndReturn(self.datetime(2012, 1, 4))
+ complete_failures._DAYS_NOT_RUNNING_CUTOFF = 60
- self.assertEqual(set(results), set(['test1', 'test2']))
+ self.mox.ReplayAll()
+ results = complete_failures.get_recently_ran_test_names()
+
+ self.assertEqual(set(results), set(['recent']))
def test_returns_no_duplicate_names(self):
@@ -318,15 +334,19 @@
test = models.Test(job=job, status=success_status,
kernel=kernel, machine=machine,
test='test',
- started_time=datetime.datetime(2012, 1, 1))
+ started_time=self.datetime(2012, 1, 1))
test.save()
duplicate = models.Test(job=job, status=success_status,
kernel=kernel, machine=machine,
test='test',
- started_time=datetime.datetime(2012, 1, 2))
+ started_time=self.datetime(2012, 1, 2))
duplicate.save()
- results = complete_failures.get_all_test_names()
+ datetime.datetime.today().AndReturn(self.datetime(2012, 1, 3))
+ complete_failures._DAYS_NOT_RUNNING_CUTOFF = 60
+
+ self.mox.ReplayAll()
+ results = complete_failures.get_recently_ran_test_names()
self.assertEqual(len(results), 1)
@@ -341,10 +361,14 @@
invalid_test = models.Test(job=job, status=success_status,
kernel=kernel, machine=machine,
test='invalid_test/name',
- started_time=datetime.datetime(2012, 1, 1))
+ started_time=self.datetime(2012, 1, 1))
invalid_test.save()
- results = complete_failures.get_all_test_names()
+ datetime.datetime.today().AndReturn(self.datetime(2012, 1, 2))
+ complete_failures._DAYS_NOT_RUNNING_CUTOFF = 60
+
+ self.mox.ReplayAll()
+ results = complete_failures.get_recently_ran_test_names()
self.assertTrue(not results)
@@ -352,29 +376,34 @@
class GetTestsToAnalyzeTests(mox.MoxTestBase):
"""Tests the get_tests_to_analyze function."""
- def test_returns_all_test_names(self):
+ def test_returns_recent_test_names(self):
"""Test should return all the test names in the database."""
self.mox.StubOutWithMock(complete_failures, 'get_last_pass_times')
- self.mox.StubOutWithMock(complete_failures, 'get_all_test_names')
+ self.mox.StubOutWithMock(complete_failures,
+ 'get_recently_ran_test_names')
complete_failures.get_last_pass_times().AndReturn({'passing_test':
- datetime.datetime(2012, 1 ,1)})
- complete_failures.get_all_test_names().AndReturn(['passing_test',
- 'failing_test'])
+ datetime.datetime(2012, 1 ,1),
+ 'old_passing_test': datetime.datetime(2011, 1, 1)})
+ complete_failures.get_recently_ran_test_names().AndReturn(
+ {'passing_test',
+ 'failing_test'})
self.mox.ReplayAll()
results = complete_failures.get_tests_to_analyze()
- self.assertEqual(set(results.keys()),
- set(['passing_test', 'failing_test']))
+ self.assertEqual(results,
+ {'passing_test': datetime.datetime(2012, 1, 1),
+ 'failing_test': datetime.datetime.min})
def test_returns_failing_tests_with_min_datetime(self):
"""Test that never-passed tests are paired with datetime.min."""
self.mox.StubOutWithMock(complete_failures, 'get_last_pass_times')
- self.mox.StubOutWithMock(complete_failures, 'get_all_test_names')
+ self.mox.StubOutWithMock(complete_failures,
+ 'get_recently_ran_test_names')
complete_failures.get_last_pass_times().AndReturn({})
- complete_failures.get_all_test_names().AndReturn(['test'])
+ complete_failures.get_recently_ran_test_names().AndReturn({'test'})
self.mox.ReplayAll()
results = complete_failures.get_tests_to_analyze()