autotest: add a tko connection retry counters

BUG=chromium:852083
TEST=None

Change-Id: I4c9470af2ae85e54a3b28f3c36298d2cad5ee20a
Reviewed-on: https://chromium-review.googlesource.com/1136054
Commit-Ready: Aviv Keshet <akeshet@chromium.org>
Tested-by: Aviv Keshet <akeshet@chromium.org>
Reviewed-by: Prathmesh Prabhu <pprabhu@chromium.org>
diff --git a/tko/db.py b/tko/db.py
index 1e8af23..34c07eb 100644
--- a/tko/db.py
+++ b/tko/db.py
@@ -25,9 +25,16 @@
 
 import common
 from autotest_lib.client.common_lib import global_config
+from autotest_lib.client.common_lib import utils
 from autotest_lib.client.common_lib.cros import retry
 from autotest_lib.frontend import database_settings_helper
 
+try:
+    from chromite.lib import metrics
+except ImportError:
+    metrics = utils.metrics_mock
+
+
 def _log_error(msg):
     """Log an error message.
 
@@ -51,6 +58,11 @@
     pass
 
 
+def _connection_retry_callback():
+    """Callback method used to increment a retry metric."""
+    metrics.Counter('chromeos/autotest/tko/connection_retries').increment()
+
+
 class db_sql(object):
     """Data access."""
 
@@ -155,7 +167,7 @@
 
 
     @retry.retry(driver.OperationalError, timeout_min=10,
-                 delay_sec=5)
+                 delay_sec=5, callback=_connection_retry_callback)
     def connect(self, host, database, user, password, port):
         """Open and return a connection to mysql database."""
         connection_args = {