[autotest] Set limit on how many retries happen in a suite

In catastrophic cases where all the tests fail, having a lot of
retries can hurt the lab. This CL allows set a ceiling on
the number of retries that happen.

BUG=chromium:426177
TEST=Add JOB_RETRIES=2 to dummy_Fail. Run dummy suite with
1) --retry='True' -> retried twice
3) --retry='True' and --max_retries=0 -> No retry
2) --retry='True' and --max_retries=1 -> only 1 retry is run
3) --retry='True' and --max_retries=3 -> retried twice
4) --retry='False' and --max_retries=1 --> invalid option
DEPLOY=apache

Change-Id: I93af1dfe0744a21d45500e420a9e13b4c5680685
Reviewed-on: https://chromium-review.googlesource.com/238421
Reviewed-by: Fang Deng <fdeng@chromium.org>
Commit-Queue: Fang Deng <fdeng@chromium.org>
Trybot-Ready: Fang Deng <fdeng@chromium.org>
Tested-by: Fang Deng <fdeng@chromium.org>
diff --git a/frontend/afe/site_rpc_interface.py b/frontend/afe/site_rpc_interface.py
index c042c3e..71d9c21 100644
--- a/frontend/afe/site_rpc_interface.py
+++ b/frontend/afe/site_rpc_interface.py
@@ -117,7 +117,8 @@
                      check_hosts=True, num=None, file_bugs=False, timeout=24,
                      timeout_mins=None, priority=priorities.Priority.DEFAULT,
                      suite_args=None, wait_for_results=True, job_retry=False,
-                     max_runtime_mins=None, suite_min_duts=0, **kwargs):
+                     max_retries=None, max_runtime_mins=None, suite_min_duts=0,
+                     **kwargs):
     """
     Create a job to run a test suite on the given device with the given image.
 
@@ -144,6 +145,8 @@
     @param wait_for_results: Set to False to run the suite job without waiting
                              for test jobs to finish. Default is True.
     @param job_retry: Set to True to enable job-level retry. Default is False.
+    @param max_retries: Integer, maximum job retries allowed at suite level.
+                        None for no max.
     @param max_runtime_mins: Maximum amount of time a job can be running in
                              minutes.
     @param suite_min_duts: Integer. Scheduler will prioritize getting the
@@ -196,6 +199,7 @@
                    'suite_args' : suite_args,
                    'wait_for_results': wait_for_results,
                    'job_retry': job_retry,
+                   'max_retries': max_retries,
                    'max_runtime_mins': max_runtime_mins,
                    }
 
@@ -466,4 +470,4 @@
              of CROS.stable_cros_version if stable_versinos table does not have
              entry of board DEFAULT.
     """
-    return stable_version_utils.get_version(board)
\ No newline at end of file
+    return stable_version_utils.get_version(board)