commit 8ac6671d261161f21df60175e4d0f77472bf81e3
author Chris Masone <cmasone@chromium.org> Wed Feb 15 14:21:02 2012 -0800
committer Gerrit <chrome-bot@google.com> Thu Feb 16 18:31:01 2012 -0800
tree 3c96d733cbddc21b06b6381606706f3b92501331
parent 3b8f2e85f1b5e4622114c9b2c7d8b44dc2a01cc4

[autotest] Add retrying to autotest frontend RPCs

Add a decorator that will retry a function if it raises an exception,
exponentially backing off until a deadline.  Create classes that wrap
frontend.AFE and frontend.TKO, applying the decorator to their run()
method.

BUG=chromium-os:26419
TEST=unit
TEST=use atest to schedule a suite and restart the afe while it runs
STATUS=Fixed

Change-Id: I8a8c48b4418f55933ab168fc92f19f03ca7e9fc3
Reviewed-on: https://gerrit.chromium.org/gerrit/15960
Commit-Ready: Chris Masone <cmasone@chromium.org>
Reviewed-by: Chris Masone <cmasone@chromium.org>
Tested-by: Chris Masone <cmasone@chromium.org>

server/cros/dynamic_suite.py

diff --git a/server/cros/dynamic_suite.py b/server/cros/dynamic_suite.py
index ecf1b7b..3852644 100644
--- a/server/cros/dynamic_suite.py
+++ b/server/cros/dynamic_suite.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
+# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
@@ -6,7 +6,7 @@
 import compiler, logging, os, random, re, time
 from autotest_lib.client.common_lib import control_data, global_config, error
 from autotest_lib.client.common_lib import utils
-from autotest_lib.server.cros import control_file_getter
+from autotest_lib.server.cros import control_file_getter, frontend_wrappers
 from autotest_lib.server import frontend
 
 
@@ -54,8 +54,12 @@
         @param afe: an instance of AFE as defined in server/frontend.py.
         @param tko: an instance of TKO as defined in server/frontend.py.
         """
-        self._afe = afe or frontend.AFE(debug=False)
-        self._tko = tko or frontend.TKO(debug=False)
+        self._afe = afe or frontend_wrappers.RetryingAFE(timeout_min=30,
+                                                         delay_sec=10,
+                                                         debug=False)
+        self._tko = tko or frontend_wrappers.RetryingTKO(timeout_min=30,
+                                                         delay_sec=10,
+                                                         debug=False)
         self._cf_getter = control_file_getter.FileSystemGetter(
             [os.path.join(autotest_dir, 'server/site_tests')])
 
@@ -239,8 +243,12 @@
         """
         self._predicate = predicate
         self._tag = tag
-        self._afe = afe or frontend.AFE(debug=False)
-        self._tko = tko or frontend.TKO(debug=False)
+        self._afe = afe or frontend_wrappers.RetryingAFE(timeout_min=30,
+                                                         delay_sec=10,
+                                                         debug=False)
+        self._tko = tko or frontend_wrappers.RetryingTKO(timeout_min=30,
+                                                         delay_sec=10,
+                                                         debug=False)
         self._jobs = []
 
         self._cf_getter = Suite.create_fs_getter(autotest_dir)