[autotest] Allow for multiple pre-hqe special tasks.
Previously, every special task would call |queue_entry.on_pending()|
once it finished. This means that if you queued up multiple special
tasks to run before a host queue entry starts running, only the first
would actually run.
Now, each special task checks to see if it is the last one before it
makes the call to |queue_entry.on_pending()|, so that if we have
multiple special tasks before a queue entry, all of them will get run.
BUG=chromium:249437
DEPLOY=scheduler
TEST=Manually created a job and host queue entry and two special tasks
(one verify and one pre-job cleanup) and verified that both of them ran
before the host queue entry started.
Change-Id: Id00296192388ee256cf3071a572f8a019959c158
Reviewed-on: https://gerrit.chromium.org/gerrit/58381
Reviewed-by: Dan Shi <dshi@chromium.org>
Reviewed-by: Aviv Keshet <akeshet@chromium.org>
Reviewed-by: Scott Zawalski <scottz@chromium.org>
Commit-Queue: Alex Miller <milleral@chromium.org>
Tested-by: Alex Miller <milleral@chromium.org>
diff --git a/scheduler/monitor_db.py b/scheduler/monitor_db.py
index a50668a..93c65c4 100755
--- a/scheduler/monitor_db.py
+++ b/scheduler/monitor_db.py
@@ -1676,6 +1676,32 @@
requested_by=self.task.requested_by)
+ def _should_pending(self):
+ """
+ Decide if we should call the host queue entry's on_pending method.
+ We should if:
+ 1) There exists an associated host queue entry.
+ 2) The current special task completed successfully.
+ 3) There do not exist any more special tasks to be run before the
+ host queue entry starts.
+
+ @returns: True if we should call pending, false if not.
+
+ """
+ if not self.queue_entry or not self.success:
+ return False
+
+ # We know if this is the last one when we create it, so we could add
+ # another column to the database to keep track of this information, but
+ # I expect the overhead of querying here to be minimal.
+ queue_entry = models.HostQueueEntry.objects.get(id=self.queue_entry.id)
+ queued = models.SpecialTask.objects.filter(
+ host__id=self.host.id, is_active=False,
+ is_complete=False, queue_entry=queue_entry)
+ queued = queued.exclude(id=self.task.id)
+ return queued.count() == 0
+
+
class VerifyTask(PreJobTask):
TASK_TYPE = models.SpecialTask.Task.VERIFY
@@ -1706,7 +1732,7 @@
def epilog(self):
super(VerifyTask, self).epilog()
if self.success:
- if self.queue_entry:
+ if self._should_pending():
self.queue_entry.on_pending()
else:
self.host.set_status(models.Host.Status.READY)
@@ -1748,7 +1774,8 @@
queue_entry=entry,
task=models.SpecialTask.Task.VERIFY)
else:
- self.queue_entry.on_pending()
+ if self._should_pending():
+ self.queue_entry.on_pending()
def epilog(self):
diff --git a/scheduler/scheduler_models.py b/scheduler/scheduler_models.py
index 512cfa5..bc8da6f 100644
--- a/scheduler/scheduler_models.py
+++ b/scheduler/scheduler_models.py
@@ -1202,30 +1202,46 @@
return self.run_verify
- def schedule_pre_job_tasks(self, queue_entry):
+ def _queue_special_task(self, queue_entry, task):
"""
- Get a list of tasks to perform before the host_queue_entry
- may be used to run this Job (such as Cleanup & Verify).
+ Create a special task and associate it with a host queue entry.
- @returns A list of tasks to be done to the given queue_entry before
- it should be considered be ready to run this job. The last
- task in the list calls HostQueueEntry.on_pending(), which
- continues the flow of the job.
+ @param queue_entry: The queue entry this special task should be
+ associated with.
+ @param task: One of the members of the enum models.SpecialTask.Task.
+ @returns: None
+
"""
- if self._should_run_cleanup(queue_entry):
- task = models.SpecialTask.Task.CLEANUP
- elif self._should_run_verify(queue_entry):
- task = models.SpecialTask.Task.VERIFY
- else:
- queue_entry.on_pending()
- return
-
- queue_entry = models.HostQueueEntry.objects.get(id=queue_entry.id)
models.SpecialTask.objects.create(
host=models.Host.objects.get(id=queue_entry.host_id),
queue_entry=queue_entry, task=task)
+ def schedule_pre_job_tasks(self, queue_entry):
+ """
+ Queue all of the special tasks that need to be run before a host
+ queue entry may run.
+
+ If no special taskes need to be scheduled, then |on_pending| will be
+ called directly.
+
+ @returns None
+
+ """
+ task_queued = False
+ hqe_model = models.HostQueueEntry.objects.get(id=queue_entry.id)
+
+ if self._should_run_cleanup(queue_entry):
+ self._queue_special_task(hqe_model, models.SpecialTask.Task.CLEANUP)
+ task_queued = True
+ if self._should_run_verify(queue_entry):
+ self._queue_special_task(hqe_model, models.SpecialTask.Task.VERIFY)
+ task_queued = True
+
+ if not task_queued:
+ queue_entry.on_pending()
+
+
def _assign_new_group(self, queue_entries, group_name=''):
if len(queue_entries) == 1:
group_subdir_name = queue_entries[0].host.hostname