[autotest] Retry when staging a control files fails in run-suite.

The cl also adds some stats for reporting run_suite failures, and
sneaks in some autoserv testing changes.

TEST=Ran run_suite against a bad devserver, raised the exception.
BUG=chromium:418928, chromium:359741, chromium:371644

Change-Id: If49a3b96c053432cb26a01a02e160176be37c037
Reviewed-on: https://chromium-review.googlesource.com/220973
Reviewed-by: Prashanth B <beeps@chromium.org>
Tested-by: Prashanth B <beeps@chromium.org>
Commit-Queue: Prashanth B <beeps@chromium.org>
diff --git a/server/autoserv b/server/autoserv
index b627c44..c0fb4d6 100755
--- a/server/autoserv
+++ b/server/autoserv
@@ -17,7 +17,7 @@
 
 
 # Number of seconds to wait before returning if testing mode is enabled
-TESTING_MODE_SLEEP_SECS = 10
+TESTING_MODE_SLEEP_SECS = 1
 
 
 try:
@@ -314,14 +314,26 @@
     except control_data.ControlVariableException as e:
         logging.error(str(e))
     exit_code = 0
+    # TODO(beeps): Extend this to cover different failure modes.
+    # Testing exceptions are matched against labels sent to autoserv. Eg,
+    # to allow only the hostless job to run, specify
+    # testing_exceptions: test_suite in the shadow_config. To allow both
+    # the hostless job and dummy_Pass to run, specify
+    # testing_exceptions: test_suite,dummy_Pass. You can figure out
+    # what label autoserv is invoked with by looking through the logs of a test
+    # for the autoserv command's -l option.
+    testing_exceptions = global_config.global_config.get_config_value(
+            'AUTOSERV', 'testing_exceptions', type=list, default=[])
+    test_mode = global_config.global_config.get_config_value(
+            'AUTOSERV', 'testing_mode', type=bool, default=False)
+    test_mode = test_mode and not any([ex in parser.options.label
+                                       for ex in testing_exceptions])
     try:
         try:
-            if not global_config.global_config.get_config_value(
-                        'AUTOSERV', 'testing_mode', type=bool, default=False):
-                run_autoserv(pid_file_manager, results, parser)
-            else:
-                # TODO(beeps): Extend this to cover different failure modes.
+            if test_mode:
                 time.sleep(TESTING_MODE_SLEEP_SECS)
+            else:
+                run_autoserv(pid_file_manager, results, parser)
         except SystemExit as e:
             exit_code = e.code
             if exit_code: