commit e8e0c0520cf1cb5ab2c4198c5344a237f2a04850
author Dan Shi <dshi@chromium.org> Tue Sep 01 00:27:27 2015 -0700
committer chrome-bot <chrome-bot@chromium.org> Tue Sep 01 16:10:46 2015 -0700
tree 6bfefe3b2c25d3ed424b7c84c1f4ddb5a9b314a4
parent 08cd8021a3f1f9a22bbf3d80e697078333e1b716

[autotest] Add retry to tko/db.commit

suite.py uses RetryingTKO for querying test results, however TKO class does not
use RPC to call get_job_test_statuses_from_db. It is a direct call to db.
In that method, db.connect, db.select are both retried, but not db.commit.

This change adds retry to db.commit.

BUG=chromium:526879
TEST=local test by restarting CloudSQL instance.

Change-Id: Iff2b404275571bf94e846ea646bfdcc3159515f2
Reviewed-on: https://chromium-review.googlesource.com/296475
Commit-Ready: Dan Shi <dshi@chromium.org>
Tested-by: Dan Shi <dshi@chromium.org>
Reviewed-by: Fang Deng <fdeng@chromium.org>

server/frontend.py

diff --git a/server/frontend.py b/server/frontend.py
index 6e8f817..122a2cb 100644
--- a/server/frontend.py
+++ b/server/frontend.py
@@ -13,12 +13,18 @@
     http://docs.djangoproject.com/en/dev/ref/models/querysets/#queryset-api
 """
 
-import getpass, os, time, traceback, re
+import getpass
+import os
+import re
+import time
+import traceback
+
 import common
 from autotest_lib.frontend.afe import rpc_client_lib
+from autotest_lib.client.common_lib import control_data
 from autotest_lib.client.common_lib import global_config
 from autotest_lib.client.common_lib import utils
-from autotest_lib.client.common_lib import control_data
+from autotest_lib.client.common_lib.cros.graphite import autotest_stats
 from autotest_lib.tko import db
 
 
@@ -31,6 +37,8 @@
 GLOBAL_CONFIG = global_config.global_config
 DEFAULT_SERVER = 'autotest'
 
+_tko_timer = autotest_stats.Timer('tko')
+
 def dump_object(header, obj):
     """
     Standard way to print out the frontend objects (eg job, host, acl, label)
@@ -129,6 +137,7 @@
         self._db = None
 
 
+    @_tko_timer.decorate
     def get_job_test_statuses_from_db(self, job_id):
         """Get job test statuses from the database.
 
@@ -141,7 +150,7 @@
         @returns a TestStatus object of the resulting information.
         """
         if self._db is None:
-          self._db = db.db()
+            self._db = db.db()
         fields = ['status', 'test_name', 'subdir', 'reason',
                   'test_started_time', 'test_finished_time', 'afe_job_id',
                   'job_owner', 'hostname', 'job_tag']

tko/db.py

diff --git a/tko/db.py b/tko/db.py
index 80ea5c9..4c611f2 100644
--- a/tko/db.py
+++ b/tko/db.py
@@ -6,6 +6,7 @@
 
 import common
 from autotest_lib.client.common_lib import global_config
+from autotest_lib.client.common_lib.cros.graphite import autotest_stats
 from autotest_lib.frontend import database_settings_helper
 from autotest_lib.server import site_utils
 from autotest_lib.tko import utils
@@ -142,6 +143,7 @@
                     try:
                         self._random_delay()
                         self._init_db()
+                        autotest_stats.Counter('tko_db_error').increment()
                     except OperationalError, e:
                         self._log_operational_error(e)
             else:
@@ -161,8 +163,17 @@
             sys.stdout.write('SQL: ' + str(value) + '\n')
 
 
+    def _commit(self):
+        """Private method for function commit to call for retry.
+        """
+        return self.con.commit()
+
+
     def commit(self):
-        self.con.commit()
+        if self.autocommit:
+            return self.run_with_retry(self._commit)
+        else:
+            return self._commit()
 
 
     def rollback(self):