Introduce a mechanism for retrying tests at the control file level.
If a test returns unsuccessfully and --retry-test > 0 is specified the test
will be rerun. Options were added to autoserv and are passed through to the
client side job by modifying the base_job.
Tests that fail and then succeed within the --retry-test limits will have a
test keyval set with the number of attempts |test_retries_before_success|.
--retry-test can be utilized immediately in run_remote_test but will need to
be piped through the RPC system and down to the scheduler to provide the proper
--retry-test value from the scheduler's perspective.
Fixed random pylint errors consisting mostly of unused imports.
TEST=unittests
created a randomly failing client and server side test and ran it on devices
BUG=chromium-os:37158
Change-Id: Ibec3935b5f6fd28fc1b6eb7be55de27a571ad777
Reviewed-on: https://gerrit.chromium.org/gerrit/42043
Commit-Queue: Scott Zawalski <scottz@chromium.org>
Reviewed-by: Scott Zawalski <scottz@chromium.org>
Tested-by: Scott Zawalski <scottz@chromium.org>
diff --git a/client/common_lib/test_unittest.py b/client/common_lib/test_unittest.py
index d2218ba..65b5785 100755
--- a/client/common_lib/test_unittest.py
+++ b/client/common_lib/test_unittest.py
@@ -5,9 +5,8 @@
__author__ = 'gps@google.com (Gregory P. Smith)'
import unittest
-from cStringIO import StringIO
import common
-from autotest_lib.client.common_lib import error, test
+from autotest_lib.client.common_lib import test
from autotest_lib.client.common_lib.test_utils import mock
class TestTestCase(unittest.TestCase):
@@ -24,6 +23,7 @@
self.job = MockJob()
self.job.default_profile_only = False
self.job.profilers = MockProfilerManager()
+ self.job.test_retry = 0
self._new_keyval = False
self.iteration = 0
self.before_iteration_hooks = []
@@ -93,6 +93,78 @@
self.god.check_playback()
+ def _setup_failed_test_calls(self, fail_count, error):
+ """
+ Set up failed test calls for use with call_run_once_with_retry.
+
+ @param fail_count: The amount of times to mock a failure.
+ @param error: The error to raise while failing.
+ """
+ self.god.stub_function(self.test.job, 'record')
+ self.god.stub_function(self.test, '_call_run_once')
+ # tests the test._call_run_once implementation
+ for run in xrange(0, fail_count):
+ self.test._call_run_once.expect_call([], False, None, (1, 2),
+ {'arg': 'val'}).and_raises(
+ error)
+ info_str = 'Run %s failed with %s' % (run, error)
+ # On the final run we do not emit this message.
+ if run != self.test.job.test_retry:
+ self.test.job.record.expect_call('INFO', None, None, info_str)
+
+
+ def test_call_run_once_with_retry_exception(self):
+ """
+ Test call_run_once_with_retry duplicating a test that will always fail.
+ """
+ self.test.job.test_retry = 5
+ self.god.stub_function(self.test, 'drop_caches_between_iterations')
+ self.god.stub_function(self.test, 'run_once')
+ before_hook = self.god.create_mock_function('before_hook')
+ after_hook = self.god.create_mock_function('after_hook')
+ self.test.register_before_iteration_hook(before_hook)
+ self.test.register_after_iteration_hook(after_hook)
+ error = Exception('fail')
+ self._setup_failed_test_calls(self.test.job.test_retry+1, error)
+ try:
+ self.test._call_run_once_with_retry([], False, None, (1, 2),
+ {'arg': 'val'})
+ except Exception as err:
+ if err != error:
+ raise
+ self.god.check_playback()
+
+
+ def test_call_run_once_with_retry_exception_and_pass(self):
+ """
+ Test call_run_once_with_retry duplicating a test that fails at first
+ and later passes.
+ """
+ # Stubbed out for the write_keyval call.
+ self.test.outputdir = '/tmp'
+ self.test.job._tap = None
+
+ num_to_fail = 2
+ self.test.job.test_retry = 5
+ self.god.stub_function(self.test, 'drop_caches_between_iterations')
+ self.god.stub_function(self.test, 'run_once')
+ before_hook = self.god.create_mock_function('before_hook')
+ after_hook = self.god.create_mock_function('after_hook')
+ self.god.stub_function(self.test, '_call_run_once')
+ self.test.register_before_iteration_hook(before_hook)
+ self.test.register_after_iteration_hook(after_hook)
+ self.god.stub_function(self.test.job, 'record')
+ # tests the test._call_run_once implementation
+ error = Exception('fail')
+ self._setup_failed_test_calls(num_to_fail, error)
+ # Passing call
+ self.test._call_run_once.expect_call([], False, None, (1, 2),
+ {'arg': 'val'})
+ self.test._call_run_once_with_retry([], False, None, (1, 2),
+ {'arg': 'val'})
+ self.god.check_playback()
+
+
def _expect_call_run_once(self):
self.test._call_run_once.expect_call((), False, None, (), {})