[autotest] Scheduler, drone_manager, drone_utility stats.

Adds some useful stats to drone manager, handle agents and
drone utility. These stats should help us track processes,
figure out where the drone_manager latency is coming from
and draw correlations between the number of agents scheduled
and drone refresh time.

This cl also moves site_drone_utility's kill_process method
into drone_utility, and modifies the nuke_pids function to only
wait on and kill processes that haven't already died.

TEST=Ran suites.
BUG=chromium:400486
DEPLOY=scheduler

Change-Id: I56e6ee05fa2ae1935435dbc2055d7f99a9a89e5e
Reviewed-on: https://chromium-review.googlesource.com/211769
Reviewed-by: Prashanth B <beeps@chromium.org>
Commit-Queue: Prashanth B <beeps@chromium.org>
Tested-by: Prashanth B <beeps@chromium.org>
diff --git a/scheduler/drone_manager_unittest.py b/scheduler/drone_manager_unittest.py
index e9f1f53..a4fd0fe 100755
--- a/scheduler/drone_manager_unittest.py
+++ b/scheduler/drone_manager_unittest.py
@@ -345,7 +345,8 @@
     _DRONE_HOST = ssh_host.SSHHost
 
 
-    def create_drone(self, drone_hostname, mock_hostname):
+    def create_drone(self, drone_hostname, mock_hostname,
+                     timestamp_remote_calls=False):
         """Create and initialize a Remote Drone.
 
         @return: A remote drone instance.
@@ -355,7 +356,8 @@
         drones.drone_utility.create_host.expect_call(drone_hostname).and_return(
                 mock_host)
         mock_host.is_up.expect_call().and_return(True)
-        return self._DRONE_CLASS(drone_hostname)
+        return self._DRONE_CLASS(drone_hostname,
+                                 timestamp_remote_calls=timestamp_remote_calls)
 
 
     def create_fake_pidfile_info(self, tag='tag', name='name'):
@@ -473,14 +475,16 @@
     _DRONE_HOST = local_host.LocalHost
 
 
-    def create_drone(self, drone_hostname, mock_hostname):
+    def create_drone(self, drone_hostname, mock_hostname,
+                     timestamp_remote_calls=False):
         """Create and initialize a Remote Drone.
 
         @return: A remote drone instance.
         """
         mock_host = self.god.create_mock_class(self._DRONE_HOST, mock_hostname)
         self.god.stub_function(drones.drone_utility, 'create_host')
-        local_drone = self._DRONE_CLASS()
+        local_drone = self._DRONE_CLASS(
+                timestamp_remote_calls=timestamp_remote_calls)
         self.god.stub_with(local_drone, '_host', mock_host)
         return local_drone