[autotest] Add timers to scheduler cleanup.
The peaks of scheduler cleanup are also starting to get high enough that
I'd like to take a look at trying to lessen them a bit. I'm guessing
there's some indexing work to be done here, but I don't know which
queries to target.
BUG=chromium:305408
DEPLOY=scheduler
TEST=Ran scheduler
Change-Id: Ibc1924a90c2502b38e6c7e1058f675f497877369
Reviewed-on: https://chromium-review.googlesource.com/172320
Tested-by: Alex Miller <milleral@chromium.org>
Reviewed-by: Dan Shi <dshi@chromium.org>
Commit-Queue: Alex Miller <milleral@chromium.org>
diff --git a/scheduler/monitor_db_cleanup.py b/scheduler/monitor_db_cleanup.py
index 47cbb36..1a10759 100644
--- a/scheduler/monitor_db_cleanup.py
+++ b/scheduler/monitor_db_cleanup.py
@@ -8,6 +8,7 @@
from autotest_lib.frontend.afe import models
from autotest_lib.scheduler import email_manager, scheduler_config
from autotest_lib.client.common_lib import host_protections
+from autotest_lib.site_utils.graphite import stats
class PeriodicCleanup(object):
@@ -42,6 +43,7 @@
"""User cleanup that is controlled by the global config variable
clean_interval in the SCHEDULER section.
"""
+ timer = stats.Timer('monitor_db_cleanup.user_cleanup')
def __init__(self, db, clean_interval_minutes):
@@ -49,6 +51,7 @@
self._last_reverify_time = time.time()
+ @timer.decorate
def _cleanup(self):
logging.info('Running periodic cleanup')
self._abort_timed_out_jobs()
@@ -58,6 +61,7 @@
self._reverify_dead_hosts()
+ @timer.decorate
def _abort_timed_out_jobs(self):
msg = 'Aborting all jobs that have timed out and are not complete'
logging.info(msg)
@@ -68,6 +72,7 @@
job.abort()
+ @timer.decorate
def _abort_jobs_past_max_runtime(self):
"""
Abort executions that have started and are past the job's max runtime.
@@ -87,6 +92,7 @@
queue_entry.abort()
+ @timer.decorate
def _check_for_db_inconsistencies(self):
logging.info('Cleaning db inconsistencies')
self._check_all_invalid_related_objects()
@@ -142,6 +148,7 @@
email_manager.manager.enqueue_notify_email(subject, message)
+ @timer.decorate
def _clear_inactive_blocks(self):
msg = 'Clear out blocks for all completed jobs.'
logging.info(msg)
@@ -171,6 +178,7 @@
return sorted(hosts)
+ @timer.decorate
def _reverify_dead_hosts(self):
if not self._should_reverify_hosts_now():
return
@@ -200,6 +208,7 @@
"""Cleanup that runs at the startup of monitor_db and every subsequent
twenty four hours.
"""
+ timer = stats.Timer('monitor_db_cleanup.twentyfourhour_cleanup')
def __init__(self, db, run_at_initialize=True):
@@ -208,12 +217,14 @@
db, clean_interval, run_at_initialize=run_at_initialize)
+ @timer.decorate
def _cleanup(self):
logging.info('Running 24 hour clean up')
self._django_session_cleanup()
self._check_for_uncleanable_db_inconsistencies()
+ @timer.decorate
def _django_session_cleanup(self):
"""Clean up django_session since django doesn't for us.
http://www.djangoproject.com/documentation/0.96/sessions/
@@ -223,6 +234,7 @@
self._db.execute(sql)
+ @timer.decorate
def _check_for_uncleanable_db_inconsistencies(self):
logging.info('Checking for uncleanable DB inconsistencies')
self._check_for_active_and_complete_queue_entries()
@@ -231,6 +243,7 @@
self._check_for_multiple_atomic_group_hosts()
+ @timer.decorate
def _check_for_active_and_complete_queue_entries(self):
query = models.HostQueueEntry.objects.filter(active=True, complete=True)
if query.count() != 0:
@@ -248,6 +261,7 @@
self._send_inconsistency_message(subject, lines)
+ @timer.decorate
def _check_for_multiple_platform_hosts(self):
rows = self._db.execute("""
SELECT afe_hosts.id, hostname, COUNT(1) AS platform_count,
@@ -267,6 +281,7 @@
self._send_inconsistency_message(subject, lines)
+ @timer.decorate
def _check_for_no_platform_hosts(self):
rows = self._db.execute("""
SELECT hostname
@@ -281,6 +296,7 @@
', '.join(row[0] for row in rows))
+ @timer.decorate
def _check_for_multiple_atomic_group_hosts(self):
rows = self._db.execute("""
SELECT afe_hosts.id, hostname,