[autotest] Add retrying to autotest frontend RPCs
Add a decorator that will retry a function if it raises an exception,
exponentially backing off until a deadline. Create classes that wrap
frontend.AFE and frontend.TKO, applying the decorator to their run()
method.
BUG=chromium-os:26419
TEST=unit
TEST=use atest to schedule a suite and restart the afe while it runs
STATUS=Fixed
Change-Id: I8a8c48b4418f55933ab168fc92f19f03ca7e9fc3
Reviewed-on: https://gerrit.chromium.org/gerrit/15960
Commit-Ready: Chris Masone <cmasone@chromium.org>
Reviewed-by: Chris Masone <cmasone@chromium.org>
Tested-by: Chris Masone <cmasone@chromium.org>
diff --git a/server/cros/frontend_wrappers_unittest.py b/server/cros/frontend_wrappers_unittest.py
new file mode 100644
index 0000000..7b3b8ff
--- /dev/null
+++ b/server/cros/frontend_wrappers_unittest.py
@@ -0,0 +1,73 @@
+# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Unit tests for server/cros/frontend_wrappers.py."""
+
+import logging
+import mox
+import time
+import unittest
+
+from autotest_lib.server.cros import frontend_wrappers
+from autotest_lib.server import frontend
+
+class FrontendWrappersTest(mox.MoxTestBase):
+ """Unit tests for dynamic_suite.Reimager.
+
+ @var _FLAKY_FLAG: for use in tests that need to simulate random failures.
+ """
+
+ _FLAKY_FLAG = None
+
+ def setUp(self):
+ super(FrontendWrappersTest, self).setUp()
+ self._FLAKY_FLAG = False
+
+
+ def testRetryDecoratorSucceeds(self):
+ """Tests that a wrapped function succeeds without retrying."""
+ timeout_min = .1
+ timeout_sec = timeout_min * 60
+ @frontend_wrappers.retry(Exception,
+ timeout_min=timeout_min,
+ delay_sec=1)
+ def succeed():
+ return True
+
+ deadline = time.time() + timeout_sec
+ self.assertTrue(succeed())
+ self.assertTrue(time.time() < deadline)
+
+
+ def testRetryDecoratorFlakySucceeds(self):
+ """Tests that a wrapped function can retry and succeed."""
+ timeout_min = .1
+ timeout_sec = timeout_min * 60
+ @frontend_wrappers.retry(Exception,
+ timeout_min=timeout_min,
+ delay_sec=1)
+ def flaky_succeed():
+ if self._FLAKY_FLAG:
+ return True
+ self._FLAKY_FLAG = True
+ raise Exception
+
+ deadline = time.time() + timeout_sec
+ self.assertTrue(flaky_succeed())
+ self.assertTrue(time.time() < deadline)
+
+
+ def testRetryDecoratorFailss(self):
+ """Tests that a wrapped function retries til the timeout, then fails."""
+ timeout_min = .01
+ timeout_sec = timeout_min * 60
+ @frontend_wrappers.retry(Exception,
+ timeout_min=timeout_min,
+ delay_sec=1)
+ def fail():
+ raise Exception()
+
+ deadline = time.time() + timeout_sec
+ self.assertRaises(Exception, fail)
+ self.assertTrue(time.time() >= deadline)