[autotest] Allow for multiple pre-hqe special tasks.

Previously, every special task would call |queue_entry.on_pending()|
once it finished.  This means that if you queued up multiple special
tasks to run before a host queue entry starts running, only the first
would actually run.

Now, each special task checks to see if it is the last one before it
makes the call to |queue_entry.on_pending()|, so that if we have
multiple special tasks before a queue entry, all of them will get run.

BUG=chromium:249437
DEPLOY=scheduler
TEST=Manually created a job and host queue entry and two special tasks
(one verify and one pre-job cleanup) and verified that both of them ran
before the host queue entry started.

Change-Id: Id00296192388ee256cf3071a572f8a019959c158
Reviewed-on: https://gerrit.chromium.org/gerrit/58381
Reviewed-by: Dan Shi <dshi@chromium.org>
Reviewed-by: Aviv Keshet <akeshet@chromium.org>
Reviewed-by: Scott Zawalski <scottz@chromium.org>
Commit-Queue: Alex Miller <milleral@chromium.org>
Tested-by: Alex Miller <milleral@chromium.org>
diff --git a/scheduler/monitor_db.py b/scheduler/monitor_db.py
index a50668a..93c65c4 100755
--- a/scheduler/monitor_db.py
+++ b/scheduler/monitor_db.py
@@ -1676,6 +1676,32 @@
                 requested_by=self.task.requested_by)
 
 
+    def _should_pending(self):
+        """
+        Decide if we should call the host queue entry's on_pending method.
+        We should if:
+        1) There exists an associated host queue entry.
+        2) The current special task completed successfully.
+        3) There do not exist any more special tasks to be run before the
+           host queue entry starts.
+
+        @returns: True if we should call pending, false if not.
+
+        """
+        if not self.queue_entry or not self.success:
+            return False
+
+        # We know if this is the last one when we create it, so we could add
+        # another column to the database to keep track of this information, but
+        # I expect the overhead of querying here to be minimal.
+        queue_entry = models.HostQueueEntry.objects.get(id=self.queue_entry.id)
+        queued = models.SpecialTask.objects.filter(
+                host__id=self.host.id, is_active=False,
+                is_complete=False, queue_entry=queue_entry)
+        queued = queued.exclude(id=self.task.id)
+        return queued.count() == 0
+
+
 class VerifyTask(PreJobTask):
     TASK_TYPE = models.SpecialTask.Task.VERIFY
 
@@ -1706,7 +1732,7 @@
     def epilog(self):
         super(VerifyTask, self).epilog()
         if self.success:
-            if self.queue_entry:
+            if self._should_pending():
                 self.queue_entry.on_pending()
             else:
                 self.host.set_status(models.Host.Status.READY)
@@ -1748,7 +1774,8 @@
                     queue_entry=entry,
                     task=models.SpecialTask.Task.VERIFY)
         else:
-            self.queue_entry.on_pending()
+            if self._should_pending():
+                self.queue_entry.on_pending()
 
 
     def epilog(self):