[autotest] Retry with backoff for RPCs
Use chromite.lib.retry_util to run RPCs with exponential
backoff. The chromite library cannot be safely imported into
client/common_lib, preventing us from adding to the retry module.
So, we use chromite's retry_util directly when issuing RPCs to
get the desired backoff behaviour.
BUG=chromium:493219
Test=Tested that test suites were runnable with these changes on
a moblab. Tested the new retry logic with a stubbed function to
retry.
Change-Id: Ia5aac84d3e475a022867b4ca43d8f6bb0fb05d7a
Reviewed-on: https://chromium-review.googlesource.com/275699
Tested-by: Matthew Sartori <msartori@chromium.org>
Reviewed-by: David James <davidjames@chromium.org>
Commit-Queue: Matthew Sartori <msartori@chromium.org>
diff --git a/client/common_lib/cros/retry.py b/client/common_lib/cros/retry.py
index 380ea92..d8512de 100644
--- a/client/common_lib/cros/retry.py
+++ b/client/common_lib/cros/retry.py
@@ -2,16 +2,10 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
-import logging, math, random, signal, sys, time
+import logging, random, signal, sys, time
from autotest_lib.client.common_lib import error
-try:
- from chromite.lib import retry_util
-except ImportError:
- logging.warn('Unable to import chromite for retry_util.')
- retry_util = None
-
def handler(signum, frame):
"""
@@ -184,86 +178,3 @@
return func_retry # true decorator
return deco_retry
-
-
-def retry_exponential(ExceptionToCheck, timeout_min=1.0, delay_sec=3,
- backoff_factor=2, blacklist=None):
- """Retry calling the decorated function using an exponential backoff.
-
- Present an interface consistent with the existing retry function, but
- use instead the chromite retry_util functions to provide exponential
- backoff.
-
- @param ExceptionToCheck: See retry.
- @param timeout_min: See retry.
- @param delay_sec: See retry.
- @param backoff_factor: The base used for exponential backoff. A simpler
- backoff method is used if backoff_factor is not
- greater than 1.
- @param blacklist: See retry.
- """
- def deco_retry(func):
- """The outer decorator.
-
- @param func: The function we are decorating.
- """
- exception_tuple = () if blacklist is None else tuple(blacklist)
-
- # Check the backoff_factor. If backoff is greater than 1,
- # then we use exponential backoff, else, simple backoff.
- backoff = backoff_factor if backoff_factor >= 1 else 1
-
- # Chromite retry_util uses:
- # max_retry: The number of retry attempts to make.
- # sleep: The multiplier for how long to sleep between attempts.
- total_sleep = timeout_min * 60.0
- sleep = abs(delay_sec) if delay_sec != 0 else 1
-
- # Estimate the max_retry in the case of simple backoff:
- # => total_sleep = sleep*sum(1..max_retry)
- # => total_sleep/sleep = max_retry(max_retry+1)/2
- # => max_retry = -1/2 + sqrt(1+8K)/2 where K = total_sleep/sleep
- max_retry = int(math.ceil(-1 + math.sqrt(
- 1+8*math.ceil(total_sleep/sleep))/2.0))
-
- # Estimate the max_retry in the case of exponential backoff:
- # => total_sleep = sleep*sum(r=0..max_retry-1, backoff^r)
- # => total_sleep = sleep( (1-backoff^max_retry) / (1-backoff) )
- # => max_retry*ln(backoff) = ln(1-(total_sleep/sleep)*(1-backoff))
- # => max_retry = ln(1-(total_sleep/sleep)*(1-backoff))/ln(backoff)
- if backoff > 1:
- numerator = math.log10(1-(total_sleep/sleep)*(1-backoff))
- denominator = math.log10(backoff)
- max_retry = int(math.ceil(numerator/denominator))
-
- def handler(exc):
- """Check if exc is an ExceptionToCheck or if it's blacklisted.
-
- @param exc: An exception.
-
- @return: True if exc is an ExceptionToCheck and is not
- blacklisted. False otherwise.
- """
- is_exc_to_check = isinstance(exc, ExceptionToCheck)
- is_blacklisted = isinstance(exc, exception_tuple)
- return is_exc_to_check and not is_blacklisted
-
- def func_retry(*args, **kwargs):
- """The actual function decorator.
-
- @params args: The arguments to the function.
- @params kwargs: The keyword arguments to the function.
- """
- # Set keyword arguments
- kwargs['sleep'] = sleep
- kwargs['backoff_factor'] = backoff
-
- if retry_util is None:
- logging.warn('Failed to decorate with retry_exponential.')
- return func
- return retry_util.GenericRetry(handler, max_retry, func,
- *args, **kwargs)
-
- return func_retry
-
- return deco_retry