autotest: verify all the DUTs at the beginning of testing push

Verify all the DUTs at the beginning of testing push to ensure that
every DUT is in a good state.

BUG=None
TEST=unittest and also test in testing server.

Change-Id: I77fc502886ae3f020866fd7761fd7790b094e1fe
Reviewed-on: https://chromium-review.googlesource.com/418490
Reviewed-by: Shuqian Zhao <shuqianz@chromium.org>
Reviewed-by: Prathmesh Prabhu <pprabhu@chromium.org>
Tested-by: Shuqian Zhao <shuqianz@chromium.org>
Commit-Queue: Shuqian Zhao <shuqianz@chromium.org>
diff --git a/site_utils/test_push.py b/site_utils/test_push.py
index 48f97ac..ff249eb 100755
--- a/site_utils/test_push.py
+++ b/site_utils/test_push.py
@@ -41,6 +41,7 @@
     pass
 from autotest_lib.client.common_lib import global_config
 from autotest_lib.client.common_lib import priorities
+from autotest_lib.client.common_lib.cros import retry
 from autotest_lib.server import site_utils
 from autotest_lib.server import utils
 from autotest_lib.server.cros import provision
@@ -134,6 +135,7 @@
     pass
 
 
+@retry.retry(TestPushException, timeout_min=5, delay_sec=30)
 def check_dut_inventory(required_num_duts):
     """Check DUT inventory for each board.
 
@@ -141,6 +143,7 @@
                               requires in order to finish push tests.
     @raise TestPushException: if number of DUTs are less than the requirement.
     """
+    print 'Checking DUT inventory...'
     hosts = AFE.run('get_hosts', status='Ready', locked=False)
     platforms = [host['platform'] for host in hosts]
     current_inventory = {p : platforms.count(p) for p in platforms}
@@ -186,6 +189,7 @@
 
     @param pool: Name of the pool used by test_push.
     """
+    print 'Reverifying DUTs in pool %s' % pool
     pool_label = constants.Labels.POOL_PREFIX + pool
     hosts = [h.hostname for h in AFE.get_hosts(label=pool_label)]
     AFE.reverify_hosts(hostnames=hosts)
@@ -576,6 +580,9 @@
     try:
         # Use daemon flag will kill child processes when parent process fails.
         use_daemon = not arguments.continue_on_failure
+        # Verify all the DUTs at the beginning of testing push.
+        reverify_all_push_duts(arguments.pool)
+        time.sleep(15) # Wait 15 secs for the verify test to start.
         check_dut_inventory(arguments.num_duts)
         queue = multiprocessing.Queue()
 
diff --git a/site_utils/test_push_unittest.py b/site_utils/test_push_unittest.py
index 5b100dc..a517e96 100755
--- a/site_utils/test_push_unittest.py
+++ b/site_utils/test_push_unittest.py
@@ -13,11 +13,13 @@
 from autotest_lib.server import site_utils
 from autotest_lib.server.cros.dynamic_suite import reporting
 # Mock the retry.retry used in the test_push before import it.
-def mock_retry(ExceptionToCheck, timeout_min, exception_to_raise=None):
+def mock_retry(ExceptionToCheck, timeout_min, delay_sec,
+               exception_to_raise=None):
     """A mock retry decorator to use in place of the actual one for testing.
 
     @param ExceptionToCheck: the exception to check.
     @param timeout_mins: Amount of time in mins to wait before timing out.
+    @param delay_sec: Amount of time in secs to wait before retry.
     @param exception_to_raise: Ignored
 
     """