blob: 836e7f18bb4adb903ae74ca90deee0cda718ca7c [file] [log] [blame]
Prashanth B340fd1e2014-06-22 12:44:10 -07001import heapq
2import os
Prashanth B340fd1e2014-06-22 12:44:10 -07003import logging
Michael Liangda8c60a2014-06-03 13:24:51 -07004
5import common
Dan Shi5e2efb72017-02-07 11:40:23 -08006from autotest_lib.client.common_lib import error
7from autotest_lib.client.common_lib import global_config
8from autotest_lib.client.common_lib import utils
MK Ryu7911ad52015-12-18 11:40:04 -08009from autotest_lib.scheduler import drone_task_queue
Allen Li036b3a62017-02-06 15:19:43 -080010from autotest_lib.scheduler import drone_utility
11from autotest_lib.scheduler import drones
showard324bf812009-01-20 23:23:38 +000012from autotest_lib.scheduler import scheduler_config
Prashanth B340fd1e2014-06-22 12:44:10 -070013from autotest_lib.scheduler import thread_lib
showard170873e2009-01-07 00:22:26 +000014
Dan Shi5e2efb72017-02-07 11:40:23 -080015try:
16 from chromite.lib import metrics
17except ImportError:
18 metrics = utils.metrics_mock
19
showard170873e2009-01-07 00:22:26 +000020
showardc75fded2009-10-14 16:20:02 +000021# results on drones will be placed under the drone_installation_directory in a
22# directory with this name
23_DRONE_RESULTS_DIR_SUFFIX = 'results'
24
showarded2afea2009-07-07 20:54:07 +000025WORKING_DIRECTORY = object() # see execute_command()
26
showard8d3dbca2009-09-25 20:29:38 +000027
jamesrenc44ae992010-02-19 00:12:54 +000028AUTOSERV_PID_FILE = '.autoserv_execute'
29CRASHINFO_PID_FILE = '.collect_crashinfo_execute'
30PARSER_PID_FILE = '.parser_execute'
31ARCHIVER_PID_FILE = '.archiver_execute'
32
33ALL_PIDFILE_NAMES = (AUTOSERV_PID_FILE, CRASHINFO_PID_FILE, PARSER_PID_FILE,
34 ARCHIVER_PID_FILE)
35
MK Ryu7911ad52015-12-18 11:40:04 -080036_THREADED_DRONE_MANAGER = global_config.global_config.get_config_value(
37 scheduler_config.CONFIG_SECTION, 'threaded_drone_manager',
38 type=bool, default=True)
39
Allen Li036b3a62017-02-06 15:19:43 -080040HOSTS_JOB_SUBDIR = 'hosts/'
41PARSE_LOG = '.parse.log'
42ENABLE_ARCHIVING = global_config.global_config.get_config_value(
43 scheduler_config.CONFIG_SECTION, 'enable_archiving', type=bool)
44
jamesrenc44ae992010-02-19 00:12:54 +000045
showard170873e2009-01-07 00:22:26 +000046class DroneManagerError(Exception):
47 pass
48
49
50class CustomEquals(object):
51 def _id(self):
52 raise NotImplementedError
53
54
55 def __eq__(self, other):
56 if not isinstance(other, type(self)):
57 return NotImplemented
58 return self._id() == other._id()
59
60
61 def __ne__(self, other):
62 return not self == other
63
64
65 def __hash__(self):
66 return hash(self._id())
67
68
69class Process(CustomEquals):
70 def __init__(self, hostname, pid, ppid=None):
71 self.hostname = hostname
72 self.pid = pid
73 self.ppid = ppid
74
75 def _id(self):
76 return (self.hostname, self.pid)
77
78
79 def __str__(self):
80 return '%s/%s' % (self.hostname, self.pid)
81
82
83 def __repr__(self):
84 return super(Process, self).__repr__() + '<%s>' % self
85
86
87class PidfileId(CustomEquals):
88 def __init__(self, path):
89 self.path = path
90
91
92 def _id(self):
93 return self.path
94
95
96 def __str__(self):
97 return str(self.path)
98
99
showardd1195652009-12-08 22:21:02 +0000100class _PidfileInfo(object):
101 age = 0
102 num_processes = None
103
104
showard170873e2009-01-07 00:22:26 +0000105class PidfileContents(object):
106 process = None
107 exit_status = None
108 num_tests_failed = None
109
110 def is_invalid(self):
111 return False
112
113
showardd1195652009-12-08 22:21:02 +0000114 def is_running(self):
115 return self.process and not self.exit_status
116
117
showard170873e2009-01-07 00:22:26 +0000118class InvalidPidfile(object):
Simran Basi899f9fe2013-02-27 11:58:49 -0800119 process = None
120 exit_status = None
121 num_tests_failed = None
Simran Basi4d7bca22013-02-27 10:57:04 -0800122
123
Simran Basi899f9fe2013-02-27 11:58:49 -0800124 def __init__(self, error):
showard170873e2009-01-07 00:22:26 +0000125 self.error = error
126
127
128 def is_invalid(self):
129 return True
130
131
showardd1195652009-12-08 22:21:02 +0000132 def is_running(self):
133 return False
134
135
showard170873e2009-01-07 00:22:26 +0000136 def __str__(self):
137 return self.error
138
139
showard418785b2009-11-23 20:19:59 +0000140class _DroneHeapWrapper(object):
141 """Wrapper to compare drones based on used_capacity().
142
143 These objects can be used to keep a heap of drones by capacity.
144 """
145 def __init__(self, drone):
146 self.drone = drone
147
148
149 def __cmp__(self, other):
150 assert isinstance(other, _DroneHeapWrapper)
151 return cmp(self.drone.used_capacity(), other.drone.used_capacity())
152
153
Allen Li036b3a62017-02-06 15:19:43 -0800154class DroneManager(object):
showard170873e2009-01-07 00:22:26 +0000155 """
156 This class acts as an interface from the scheduler to drones, whether it be
157 only a single "drone" for localhost or multiple remote drones.
158
159 All paths going into and out of this class are relative to the full results
160 directory, except for those returns by absolute_path().
161 """
Fang Deng9a0c6c32013-09-04 15:34:55 -0700162
163
164 # Minimum time to wait before next email
165 # about a drone hitting process limit is sent.
166 NOTIFY_INTERVAL = 60 * 60 * 24 # one day
Prashanth B340fd1e2014-06-22 12:44:10 -0700167 _STATS_KEY = 'drone_manager'
Fang Deng9a0c6c32013-09-04 15:34:55 -0700168
Aviv Keshet99e6adb2016-07-14 16:35:32 -0700169
Fang Deng9a0c6c32013-09-04 15:34:55 -0700170
showard170873e2009-01-07 00:22:26 +0000171 def __init__(self):
showardd1195652009-12-08 22:21:02 +0000172 # absolute path of base results dir
showard170873e2009-01-07 00:22:26 +0000173 self._results_dir = None
showardd1195652009-12-08 22:21:02 +0000174 # holds Process objects
showard170873e2009-01-07 00:22:26 +0000175 self._process_set = set()
Alex Millere76e2252013-08-15 09:24:27 -0700176 # holds the list of all processes running on all drones
177 self._all_processes = {}
showardd1195652009-12-08 22:21:02 +0000178 # maps PidfileId to PidfileContents
showard170873e2009-01-07 00:22:26 +0000179 self._pidfiles = {}
showardd1195652009-12-08 22:21:02 +0000180 # same as _pidfiles
showard170873e2009-01-07 00:22:26 +0000181 self._pidfiles_second_read = {}
showardd1195652009-12-08 22:21:02 +0000182 # maps PidfileId to _PidfileInfo
183 self._registered_pidfile_info = {}
184 # used to generate unique temporary paths
showard170873e2009-01-07 00:22:26 +0000185 self._temporary_path_counter = 0
showardd1195652009-12-08 22:21:02 +0000186 # maps hostname to Drone object
showard170873e2009-01-07 00:22:26 +0000187 self._drones = {}
188 self._results_drone = None
showardd1195652009-12-08 22:21:02 +0000189 # maps results dir to dict mapping file path to contents
showard170873e2009-01-07 00:22:26 +0000190 self._attached_files = {}
showard418785b2009-11-23 20:19:59 +0000191 # heapq of _DroneHeapWrappers
showard170873e2009-01-07 00:22:26 +0000192 self._drone_queue = []
Prashanth B340fd1e2014-06-22 12:44:10 -0700193 # A threaded task queue used to refresh drones asynchronously.
MK Ryu7911ad52015-12-18 11:40:04 -0800194 if _THREADED_DRONE_MANAGER:
195 self._refresh_task_queue = thread_lib.ThreadedTaskQueue(
196 name='%s.refresh_queue' % self._STATS_KEY)
197 else:
198 self._refresh_task_queue = drone_task_queue.DroneTaskQueue()
showard170873e2009-01-07 00:22:26 +0000199
200
201 def initialize(self, base_results_dir, drone_hostnames,
202 results_repository_hostname):
203 self._results_dir = base_results_dir
showard170873e2009-01-07 00:22:26 +0000204
205 for hostname in drone_hostnames:
Eric Li861b2d52011-02-04 14:50:35 -0800206 self._add_drone(hostname)
showard170873e2009-01-07 00:22:26 +0000207
208 if not self._drones:
209 # all drones failed to initialize
210 raise DroneManagerError('No valid drones found')
211
showard324bf812009-01-20 23:23:38 +0000212 self.refresh_drone_configs()
showardc5afc462009-01-13 00:09:39 +0000213
showard4460ee82009-07-07 20:54:29 +0000214 logging.info('Using results repository on %s',
showardb18134f2009-03-20 20:52:18 +0000215 results_repository_hostname)
showard170873e2009-01-07 00:22:26 +0000216 self._results_drone = drones.get_drone(results_repository_hostname)
showardac5b0002009-10-19 18:34:00 +0000217 results_installation_dir = global_config.global_config.get_config_value(
218 scheduler_config.CONFIG_SECTION,
219 'results_host_installation_directory', default=None)
220 if results_installation_dir:
221 self._results_drone.set_autotest_install_dir(
222 results_installation_dir)
showard170873e2009-01-07 00:22:26 +0000223 # don't initialize() the results drone - we don't want to clear out any
showardd1195652009-12-08 22:21:02 +0000224 # directories and we don't need to kill any processes
showard170873e2009-01-07 00:22:26 +0000225
226
227 def reinitialize_drones(self):
Prathmesh Prabhu36238622016-11-22 18:41:06 -0800228 for drone in self.get_drones():
229 with metrics.SecondsTimer(
230 'chromeos/autotest/drone_manager/'
231 'reinitialize_drones_duration',
232 fields={'drone': drone.hostname}):
233 drone.call('initialize', self._results_dir)
showard170873e2009-01-07 00:22:26 +0000234
235
236 def shutdown(self):
showard324bf812009-01-20 23:23:38 +0000237 for drone in self.get_drones():
showard170873e2009-01-07 00:22:26 +0000238 drone.shutdown()
239
240
showard8d3dbca2009-09-25 20:29:38 +0000241 def _get_max_pidfile_refreshes(self):
242 """
243 Normally refresh() is called on every monitor_db.Dispatcher.tick().
244
245 @returns: The number of refresh() calls before we forget a pidfile.
246 """
247 pidfile_timeout = global_config.global_config.get_config_value(
248 scheduler_config.CONFIG_SECTION, 'max_pidfile_refreshes',
249 type=int, default=2000)
250 return pidfile_timeout
251
252
showard170873e2009-01-07 00:22:26 +0000253 def _add_drone(self, hostname):
Allen Li036b3a62017-02-06 15:19:43 -0800254 """
255 Add drone.
256
257 Catches AutoservRunError if the drone fails initialization and does not
258 add it to the list of usable drones.
259
260 @param hostname: Hostname of the drone we are trying to add.
261 """
262 logging.info('Adding drone %s' % hostname)
showard170873e2009-01-07 00:22:26 +0000263 drone = drones.get_drone(hostname)
Eric Li861b2d52011-02-04 14:50:35 -0800264 if drone:
Allen Li036b3a62017-02-06 15:19:43 -0800265 try:
266 drone.call('initialize', self.absolute_path(''))
267 except error.AutoservRunError as e:
268 logging.error('Failed to initialize drone %s with error: %s',
269 hostname, e)
270 return
Eric Li861b2d52011-02-04 14:50:35 -0800271 self._drones[drone.hostname] = drone
showard170873e2009-01-07 00:22:26 +0000272
273
274 def _remove_drone(self, hostname):
275 self._drones.pop(hostname, None)
276
277
showard324bf812009-01-20 23:23:38 +0000278 def refresh_drone_configs(self):
showardc5afc462009-01-13 00:09:39 +0000279 """
showard324bf812009-01-20 23:23:38 +0000280 Reread global config options for all drones.
showardc5afc462009-01-13 00:09:39 +0000281 """
Dan Shib9144a42014-12-01 16:09:32 -0800282 # Import server_manager_utils is delayed rather than at the beginning of
283 # this module. The reason is that test_that imports drone_manager when
284 # importing autoserv_utils. The import is done before test_that setup
285 # django (test_that only setup django in setup_local_afe, since it's
286 # not needed when test_that runs the test in a lab duts through :lab:
287 # option. Therefore, if server_manager_utils is imported at the
288 # beginning of this module, test_that will fail since django is not
289 # setup yet.
290 from autotest_lib.site_utils import server_manager_utils
showard324bf812009-01-20 23:23:38 +0000291 config = global_config.global_config
292 section = scheduler_config.CONFIG_SECTION
293 config.parse_config_file()
showardc5afc462009-01-13 00:09:39 +0000294 for hostname, drone in self._drones.iteritems():
Dan Shib9144a42014-12-01 16:09:32 -0800295 if server_manager_utils.use_server_db():
296 server = server_manager_utils.get_servers(hostname=hostname)[0]
297 attributes = dict([(a.attribute, a.value)
298 for a in server.attributes.all()])
299 drone.enabled = (
300 int(attributes.get('disabled', 0)) == 0)
301 drone.max_processes = int(
302 attributes.get(
303 'max_processes',
304 scheduler_config.config.max_processes_per_drone))
305 allowed_users = attributes.get('users', None)
306 else:
307 disabled = config.get_config_value(
308 section, '%s_disabled' % hostname, default='')
309 drone.enabled = not bool(disabled)
Dan Shib9144a42014-12-01 16:09:32 -0800310 drone.max_processes = config.get_config_value(
311 section, '%s_max_processes' % hostname, type=int,
312 default=scheduler_config.config.max_processes_per_drone)
showard9bb960b2009-11-19 01:02:11 +0000313
Dan Shib9144a42014-12-01 16:09:32 -0800314 allowed_users = config.get_config_value(
315 section, '%s_users' % hostname, default=None)
316 if allowed_users:
317 drone.allowed_users = set(allowed_users.split())
318 else:
319 drone.allowed_users = None
320 logging.info('Drone %s.max_processes: %s', hostname,
321 drone.max_processes)
322 logging.info('Drone %s.enabled: %s', hostname, drone.enabled)
323 logging.info('Drone %s.allowed_users: %s', hostname,
324 drone.allowed_users)
showardc5afc462009-01-13 00:09:39 +0000325
showard418785b2009-11-23 20:19:59 +0000326 self._reorder_drone_queue() # max_processes may have changed
Fang Deng9a0c6c32013-09-04 15:34:55 -0700327 # Clear notification record about reaching max_processes limit.
328 self._notify_record = {}
showard418785b2009-11-23 20:19:59 +0000329
showardc5afc462009-01-13 00:09:39 +0000330
showard324bf812009-01-20 23:23:38 +0000331 def get_drones(self):
332 return self._drones.itervalues()
showardc5afc462009-01-13 00:09:39 +0000333
334
Dan Shic458f662015-04-29 12:12:38 -0700335 def cleanup_orphaned_containers(self):
336 """Queue cleanup_orphaned_containers call at each drone.
337 """
Dan Shi55d58992015-05-05 09:10:02 -0700338 for drone in self._drones.values():
339 logging.info('Queue cleanup_orphaned_containers at %s',
340 drone.hostname)
Dan Shic458f662015-04-29 12:12:38 -0700341 drone.queue_call('cleanup_orphaned_containers')
Dan Shic458f662015-04-29 12:12:38 -0700342
343
showard170873e2009-01-07 00:22:26 +0000344 def _get_drone_for_process(self, process):
showard170873e2009-01-07 00:22:26 +0000345 return self._drones[process.hostname]
346
347
348 def _get_drone_for_pidfile_id(self, pidfile_id):
349 pidfile_contents = self.get_pidfile_contents(pidfile_id)
Shuqian Zhao502a6ac2018-02-21 14:44:59 -0800350 if pidfile_contents.process is None:
351 raise DroneManagerError('Fail to get a drone due to empty pidfile')
showard170873e2009-01-07 00:22:26 +0000352 return self._get_drone_for_process(pidfile_contents.process)
353
354
Allen Liff7064f2017-09-13 15:11:31 -0700355 def get_drone_for_pidfile_id(self, pidfile_id):
356 """Public API for luciferlib.
357
358 @param pidfile_id: PidfileId instance.
359 """
360 return self._get_drone_for_pidfile_id(pidfile_id)
361
362
showard170873e2009-01-07 00:22:26 +0000363 def _drop_old_pidfiles(self):
showardd3496242009-12-10 21:41:43 +0000364 # use items() since the dict is modified in unregister_pidfile()
365 for pidfile_id, info in self._registered_pidfile_info.items():
showardd1195652009-12-08 22:21:02 +0000366 if info.age > self._get_max_pidfile_refreshes():
showardf85a0b72009-10-07 20:48:45 +0000367 logging.warning('dropping leaked pidfile %s', pidfile_id)
368 self.unregister_pidfile(pidfile_id)
showard170873e2009-01-07 00:22:26 +0000369 else:
showardd1195652009-12-08 22:21:02 +0000370 info.age += 1
showard170873e2009-01-07 00:22:26 +0000371
372
373 def _reset(self):
showard170873e2009-01-07 00:22:26 +0000374 self._process_set = set()
Alex Millere76e2252013-08-15 09:24:27 -0700375 self._all_processes = {}
showard170873e2009-01-07 00:22:26 +0000376 self._pidfiles = {}
377 self._pidfiles_second_read = {}
378 self._drone_queue = []
379
380
showard170873e2009-01-07 00:22:26 +0000381 def _parse_pidfile(self, drone, raw_contents):
Prashanth B340fd1e2014-06-22 12:44:10 -0700382 """Parse raw pidfile contents.
383
384 @param drone: The drone on which this pidfile was found.
385 @param raw_contents: The raw contents of a pidfile, eg:
386 "pid\nexit_staus\nnum_tests_failed\n".
387 """
showard170873e2009-01-07 00:22:26 +0000388 contents = PidfileContents()
389 if not raw_contents:
390 return contents
391 lines = raw_contents.splitlines()
392 if len(lines) > 3:
393 return InvalidPidfile('Corrupt pid file (%d lines):\n%s' %
394 (len(lines), lines))
395 try:
396 pid = int(lines[0])
397 contents.process = Process(drone.hostname, pid)
398 # if len(lines) == 2, assume we caught Autoserv between writing
399 # exit_status and num_failed_tests, so just ignore it and wait for
400 # the next cycle
401 if len(lines) == 3:
402 contents.exit_status = int(lines[1])
403 contents.num_tests_failed = int(lines[2])
404 except ValueError, exc:
405 return InvalidPidfile('Corrupt pid file: ' + str(exc.args))
406
407 return contents
408
409
410 def _process_pidfiles(self, drone, pidfiles, store_in_dict):
411 for pidfile_path, contents in pidfiles.iteritems():
412 pidfile_id = PidfileId(pidfile_path)
413 contents = self._parse_pidfile(drone, contents)
414 store_in_dict[pidfile_id] = contents
415
416
showard0205a3e2009-01-16 03:03:50 +0000417 def _add_process(self, drone, process_info):
418 process = Process(drone.hostname, int(process_info['pid']),
419 int(process_info['ppid']))
420 self._process_set.add(process)
showard0205a3e2009-01-16 03:03:50 +0000421
422
423 def _add_autoserv_process(self, drone, process_info):
424 assert process_info['comm'] == 'autoserv'
425 # only root autoserv processes have pgid == pid
426 if process_info['pgid'] != process_info['pid']:
427 return
showardd1195652009-12-08 22:21:02 +0000428 self._add_process(drone, process_info)
showard0205a3e2009-01-16 03:03:50 +0000429
430
showard324bf812009-01-20 23:23:38 +0000431 def _enqueue_drone(self, drone):
showard418785b2009-11-23 20:19:59 +0000432 heapq.heappush(self._drone_queue, _DroneHeapWrapper(drone))
433
434
435 def _reorder_drone_queue(self):
436 heapq.heapify(self._drone_queue)
showard324bf812009-01-20 23:23:38 +0000437
438
Allen Lifb0cb102018-06-20 14:31:51 -0700439 def reorder_drone_queue(self):
440 """Reorder drone queue according to modified process counts.
441
442 This public API is exposed for luciferlib to wrap.
443 """
444 self._reorder_drone_queue()
445
446
showardd1195652009-12-08 22:21:02 +0000447 def _compute_active_processes(self, drone):
448 drone.active_processes = 0
449 for pidfile_id, contents in self._pidfiles.iteritems():
450 is_running = contents.exit_status is None
451 on_this_drone = (contents.process
452 and contents.process.hostname == drone.hostname)
453 if is_running and on_this_drone:
454 info = self._registered_pidfile_info[pidfile_id]
455 if info.num_processes is not None:
456 drone.active_processes += info.num_processes
Paul Hobbse1416222017-08-20 21:41:29 -0700457
458 metrics.Gauge('chromeos/autotest/drone/active_processes').set(
Aviv Keshet99e6adb2016-07-14 16:35:32 -0700459 drone.active_processes,
460 fields={'drone_hostname': drone.hostname})
showardd1195652009-12-08 22:21:02 +0000461
462
Fang Deng9a0c6c32013-09-04 15:34:55 -0700463 def _check_drone_process_limit(self, drone):
464 """
465 Notify if the number of processes on |drone| is approaching limit.
466
467 @param drone: A Drone object.
468 """
Alex Millerda713d92013-12-06 10:02:43 -0800469 try:
470 percent = float(drone.active_processes) / drone.max_processes
471 except ZeroDivisionError:
472 percent = 100
Prathmesh Prabhu21e09712016-12-20 11:50:26 -0800473 metrics.Float('chromeos/autotest/drone/active_process_percentage'
Aviv Keshetc29b4c72016-12-14 22:27:35 -0800474 ).set(percent, fields={'drone_hostname': drone.hostname})
Fang Deng9a0c6c32013-09-04 15:34:55 -0700475
Prashanth B340fd1e2014-06-22 12:44:10 -0700476 def trigger_refresh(self):
477 """Triggers a drone manager refresh.
478
479 @raises DroneManagerError: If a drone has un-executed calls.
480 Since they will get clobbered when we queue refresh calls.
showard170873e2009-01-07 00:22:26 +0000481 """
482 self._reset()
showardbf9695d2009-07-06 20:22:24 +0000483 self._drop_old_pidfiles()
showardd1195652009-12-08 22:21:02 +0000484 pidfile_paths = [pidfile_id.path
485 for pidfile_id in self._registered_pidfile_info]
Prashanth B340fd1e2014-06-22 12:44:10 -0700486 drones = list(self.get_drones())
487 for drone in drones:
488 calls = drone.get_calls()
489 if calls:
490 raise DroneManagerError('Drone %s has un-executed calls: %s '
491 'which might get corrupted through '
492 'this invocation' %
493 (drone, [str(call) for call in calls]))
494 drone.queue_call('refresh', pidfile_paths)
495 logging.info("Invoking drone refresh.")
Aviv Keshet14cac442016-11-20 21:44:11 -0800496 with metrics.SecondsTimer(
497 'chromeos/autotest/drone_manager/trigger_refresh_duration'):
Prashanth B340fd1e2014-06-22 12:44:10 -0700498 self._refresh_task_queue.execute(drones, wait=False)
showard170873e2009-01-07 00:22:26 +0000499
Prashanth B340fd1e2014-06-22 12:44:10 -0700500
501 def sync_refresh(self):
502 """Complete the drone refresh started by trigger_refresh.
503
504 Waits for all drone threads then refreshes internal datastructures
505 with drone process information.
506 """
507
508 # This gives us a dictionary like what follows:
509 # {drone: [{'pidfiles': (raw contents of pidfile paths),
510 # 'autoserv_processes': (autoserv process info from ps),
511 # 'all_processes': (all process info from ps),
512 # 'parse_processes': (parse process infor from ps),
513 # 'pidfile_second_read': (pidfile contents, again),}]
514 # drone2: ...}
515 # The values of each drone are only a list because this adheres to the
516 # drone utility interface (each call is executed and its results are
517 # places in a list, but since we never couple the refresh calls with
518 # any other call, this list will always contain a single dict).
Aviv Keshet14cac442016-11-20 21:44:11 -0800519 with metrics.SecondsTimer(
520 'chromeos/autotest/drone_manager/sync_refresh_duration'):
Prashanth B340fd1e2014-06-22 12:44:10 -0700521 all_results = self._refresh_task_queue.get_results()
522 logging.info("Drones refreshed.")
523
524 # The loop below goes through and parses pidfile contents. Pidfiles
525 # are used to track autoserv execution, and will always contain < 3
526 # lines of the following: pid, exit code, number of tests. Each pidfile
527 # is identified by a PidfileId object, which contains a unique pidfile
528 # path (unique because it contains the job id) making it hashable.
529 # All pidfiles are stored in the drone managers _pidfiles dict as:
530 # {pidfile_id: pidfile_contents(Process(drone, pid),
531 # exit_code, num_tests_failed)}
532 # In handle agents, each agent knows its pidfile_id, and uses this
533 # to retrieve the refreshed contents of its pidfile via the
534 # PidfileRunMonitor (through its tick) before making decisions. If
535 # the agent notices that its process has exited, it unregisters the
536 # pidfile from the drone_managers._registered_pidfile_info dict
537 # through its epilog.
showard170873e2009-01-07 00:22:26 +0000538 for drone, results_list in all_results.iteritems():
539 results = results_list[0]
Alex Miller82d7a9f2014-05-16 14:43:32 -0700540 drone_hostname = drone.hostname.replace('.', '_')
showard0205a3e2009-01-16 03:03:50 +0000541
Aviv Keshet14cac442016-11-20 21:44:11 -0800542 for process_info in results['all_processes']:
543 if process_info['comm'] == 'autoserv':
544 self._add_autoserv_process(drone, process_info)
545 drone_pid = drone.hostname, int(process_info['pid'])
546 self._all_processes[drone_pid] = process_info
showard170873e2009-01-07 00:22:26 +0000547
Aviv Keshet14cac442016-11-20 21:44:11 -0800548 for process_info in results['parse_processes']:
549 self._add_process(drone, process_info)
Alex Miller6cbd7582014-05-14 19:06:52 -0700550
Aviv Keshet14cac442016-11-20 21:44:11 -0800551 self._process_pidfiles(drone, results['pidfiles'], self._pidfiles)
552 self._process_pidfiles(drone, results['pidfiles_second_read'],
553 self._pidfiles_second_read)
showard170873e2009-01-07 00:22:26 +0000554
showardd1195652009-12-08 22:21:02 +0000555 self._compute_active_processes(drone)
556 if drone.enabled:
557 self._enqueue_drone(drone)
Fang Deng9a0c6c32013-09-04 15:34:55 -0700558 self._check_drone_process_limit(drone)
showard170873e2009-01-07 00:22:26 +0000559
560
Prashanth B340fd1e2014-06-22 12:44:10 -0700561 def refresh(self):
562 """Refresh all drones."""
Aviv Keshet14cac442016-11-20 21:44:11 -0800563 with metrics.SecondsTimer(
564 'chromeos/autotest/drone_manager/refresh_duration'):
Prashanth B340fd1e2014-06-22 12:44:10 -0700565 self.trigger_refresh()
566 self.sync_refresh()
567
568
Prathmesh Prabhu36238622016-11-22 18:41:06 -0800569 @metrics.SecondsTimerDecorator(
570 'chromeos/autotest/drone_manager/execute_actions_duration')
showard170873e2009-01-07 00:22:26 +0000571 def execute_actions(self):
572 """
573 Called at the end of a scheduler cycle to execute all queued actions
574 on drones.
575 """
Prashanth B340fd1e2014-06-22 12:44:10 -0700576 # Invoke calls queued on all drones since the last call to execute
577 # and wait for them to return.
MK Ryu7911ad52015-12-18 11:40:04 -0800578 if _THREADED_DRONE_MANAGER:
579 thread_lib.ThreadedTaskQueue(
580 name='%s.execute_queue' % self._STATS_KEY).execute(
581 self._drones.values())
582 else:
583 drone_task_queue.DroneTaskQueue().execute(self._drones.values())
showard170873e2009-01-07 00:22:26 +0000584
585 try:
mbligh1ef218d2009-08-03 16:57:56 +0000586 self._results_drone.execute_queued_calls()
showard170873e2009-01-07 00:22:26 +0000587 except error.AutoservError:
Aviv Keshetc29b4c72016-12-14 22:27:35 -0800588 m = 'chromeos/autotest/errors/results_repository_failed'
589 metrics.Counter(m).increment(
590 fields={'drone_hostname': self._results_drone.hostname})
showard170873e2009-01-07 00:22:26 +0000591 self._results_drone.clear_call_queue()
592
593
594 def get_orphaned_autoserv_processes(self):
595 """
showardd3dc1992009-04-22 21:01:40 +0000596 Returns a set of Process objects for orphaned processes only.
showard170873e2009-01-07 00:22:26 +0000597 """
showardd3dc1992009-04-22 21:01:40 +0000598 return set(process for process in self._process_set
599 if process.ppid == 1)
showard170873e2009-01-07 00:22:26 +0000600
601
showard170873e2009-01-07 00:22:26 +0000602 def kill_process(self, process):
603 """
604 Kill the given process.
605 """
showardd3dc1992009-04-22 21:01:40 +0000606 logging.info('killing %s', process)
showard170873e2009-01-07 00:22:26 +0000607 drone = self._get_drone_for_process(process)
Allen Li036b3a62017-02-06 15:19:43 -0800608 drone.queue_kill_process(process)
showard170873e2009-01-07 00:22:26 +0000609
610
611 def _ensure_directory_exists(self, path):
612 if not os.path.exists(path):
613 os.makedirs(path)
614
615
showard324bf812009-01-20 23:23:38 +0000616 def total_running_processes(self):
617 return sum(drone.active_processes for drone in self.get_drones())
618
619
jamesren76fcf192010-04-21 20:39:50 +0000620 def max_runnable_processes(self, username, drone_hostnames_allowed):
showard324bf812009-01-20 23:23:38 +0000621 """
622 Return the maximum number of processes that can be run (in a single
623 execution) given the current load on drones.
showard9bb960b2009-11-19 01:02:11 +0000624 @param username: login of user to run a process. may be None.
jamesren76fcf192010-04-21 20:39:50 +0000625 @param drone_hostnames_allowed: list of drones that can be used. May be
626 None
showard324bf812009-01-20 23:23:38 +0000627 """
showard1b7142d2010-01-15 00:21:37 +0000628 usable_drone_wrappers = [wrapper for wrapper in self._drone_queue
jamesren76fcf192010-04-21 20:39:50 +0000629 if wrapper.drone.usable_by(username) and
630 (drone_hostnames_allowed is None or
631 wrapper.drone.hostname in
632 drone_hostnames_allowed)]
showard1b7142d2010-01-15 00:21:37 +0000633 if not usable_drone_wrappers:
634 # all drones disabled or inaccessible
showardde700d32009-02-25 00:12:42 +0000635 return 0
jamesren37b50452010-03-25 20:38:56 +0000636 runnable_processes = [
637 wrapper.drone.max_processes - wrapper.drone.active_processes
638 for wrapper in usable_drone_wrappers]
639 return max([0] + runnable_processes)
showard324bf812009-01-20 23:23:38 +0000640
641
showarde39ebe92009-06-18 23:14:48 +0000642 def _least_loaded_drone(self, drones):
Paul Hobbs7f3c6fa2017-08-20 21:43:53 -0700643 return min(drones, key=lambda d: d.used_capacity())
showarde39ebe92009-06-18 23:14:48 +0000644
645
Prathmesh Prabhu89931612018-08-11 18:16:22 -0700646 def pick_drone_to_use(self, num_processes=1):
Allen Liff7064f2017-09-13 15:11:31 -0700647 """Return a drone to use.
648
649 Various options can be passed to optimize drone selection.
650
651 num_processes is the number of processes the drone is intended
652 to run.
653
Allen Liff7064f2017-09-13 15:11:31 -0700654 This public API is exposed for luciferlib to wrap.
655
656 Returns a drone instance (see drones.py).
657 """
658 return self._choose_drone_for_execution(
659 num_processes=num_processes,
660 username=None, # Always allow all drones
661 drone_hostnames_allowed=None, # Always allow all drones
Allen Liff7064f2017-09-13 15:11:31 -0700662 )
663
664
jamesren76fcf192010-04-21 20:39:50 +0000665 def _choose_drone_for_execution(self, num_processes, username,
Prathmesh Prabhu89931612018-08-11 18:16:22 -0700666 drone_hostnames_allowed):
Dan Shiec1d47d2015-02-13 11:38:13 -0800667 """Choose a drone to execute command.
668
669 @param num_processes: Number of processes needed for execution.
670 @param username: Name of the user to execute the command.
671 @param drone_hostnames_allowed: A list of names of drone allowed.
Dan Shiec1d47d2015-02-13 11:38:13 -0800672
673 @return: A drone object to be used for execution.
674 """
showard324bf812009-01-20 23:23:38 +0000675 # cycle through drones is order of increasing used capacity until
676 # we find one that can handle these processes
677 checked_drones = []
jamesren37b50452010-03-25 20:38:56 +0000678 usable_drones = []
showard324bf812009-01-20 23:23:38 +0000679 drone_to_use = None
680 while self._drone_queue:
showard418785b2009-11-23 20:19:59 +0000681 drone = heapq.heappop(self._drone_queue).drone
showard324bf812009-01-20 23:23:38 +0000682 checked_drones.append(drone)
Eric Lie0493a42010-11-15 13:05:43 -0800683 logging.info('Checking drone %s', drone.hostname)
showard9bb960b2009-11-19 01:02:11 +0000684 if not drone.usable_by(username):
685 continue
jamesren76fcf192010-04-21 20:39:50 +0000686
687 drone_allowed = (drone_hostnames_allowed is None
688 or drone.hostname in drone_hostnames_allowed)
689 if not drone_allowed:
Eric Lie0493a42010-11-15 13:05:43 -0800690 logging.debug('Drone %s not allowed: ', drone.hostname)
jamesren76fcf192010-04-21 20:39:50 +0000691 continue
692
jamesren37b50452010-03-25 20:38:56 +0000693 usable_drones.append(drone)
jamesren76fcf192010-04-21 20:39:50 +0000694
showard324bf812009-01-20 23:23:38 +0000695 if drone.active_processes + num_processes <= drone.max_processes:
696 drone_to_use = drone
697 break
Eric Lie0493a42010-11-15 13:05:43 -0800698 logging.info('Drone %s has %d active + %s requested > %s max',
699 drone.hostname, drone.active_processes, num_processes,
700 drone.max_processes)
showard324bf812009-01-20 23:23:38 +0000701
jamesren76fcf192010-04-21 20:39:50 +0000702 if not drone_to_use and usable_drones:
Dan Shiec1d47d2015-02-13 11:38:13 -0800703 # Drones are all over loaded, pick the one with least load.
showard324bf812009-01-20 23:23:38 +0000704 drone_summary = ','.join('%s %s/%s' % (drone.hostname,
705 drone.active_processes,
706 drone.max_processes)
jamesren37b50452010-03-25 20:38:56 +0000707 for drone in usable_drones)
708 logging.error('No drone has capacity to handle %d processes (%s) '
709 'for user %s', num_processes, drone_summary, username)
710 drone_to_use = self._least_loaded_drone(usable_drones)
showarde39ebe92009-06-18 23:14:48 +0000711
showard324bf812009-01-20 23:23:38 +0000712 # refill _drone_queue
713 for drone in checked_drones:
714 self._enqueue_drone(drone)
715
showard170873e2009-01-07 00:22:26 +0000716 return drone_to_use
717
718
showarded2afea2009-07-07 20:54:07 +0000719 def _substitute_working_directory_into_command(self, command,
720 working_directory):
721 for i, item in enumerate(command):
722 if item is WORKING_DIRECTORY:
723 command[i] = working_directory
724
725
showardd3dc1992009-04-22 21:01:40 +0000726 def execute_command(self, command, working_directory, pidfile_name,
showard418785b2009-11-23 20:19:59 +0000727 num_processes, log_file=None, paired_with_pidfile=None,
jamesren76fcf192010-04-21 20:39:50 +0000728 username=None, drone_hostnames_allowed=None):
showard170873e2009-01-07 00:22:26 +0000729 """
730 Execute the given command, taken as an argv list.
731
showarded2afea2009-07-07 20:54:07 +0000732 @param command: command to execute as a list. if any item is
733 WORKING_DIRECTORY, the absolute path to the working directory
734 will be substituted for it.
735 @param working_directory: directory in which the pidfile will be written
736 @param pidfile_name: name of the pidfile this process will write
showardd1195652009-12-08 22:21:02 +0000737 @param num_processes: number of processes to account for from this
738 execution
showarded2afea2009-07-07 20:54:07 +0000739 @param log_file (optional): path (in the results repository) to hold
740 command output.
741 @param paired_with_pidfile (optional): a PidfileId for an
742 already-executed process; the new process will execute on the
743 same drone as the previous process.
showard9bb960b2009-11-19 01:02:11 +0000744 @param username (optional): login of the user responsible for this
745 process.
jamesren76fcf192010-04-21 20:39:50 +0000746 @param drone_hostnames_allowed (optional): hostnames of the drones that
747 this command is allowed to
748 execute on
showard170873e2009-01-07 00:22:26 +0000749 """
showarddb502762009-09-09 15:31:20 +0000750 abs_working_directory = self.absolute_path(working_directory)
showard170873e2009-01-07 00:22:26 +0000751 if not log_file:
752 log_file = self.get_temporary_path('execute')
753 log_file = self.absolute_path(log_file)
showard170873e2009-01-07 00:22:26 +0000754
showarded2afea2009-07-07 20:54:07 +0000755 self._substitute_working_directory_into_command(command,
showarddb502762009-09-09 15:31:20 +0000756 abs_working_directory)
showarded2afea2009-07-07 20:54:07 +0000757
showard170873e2009-01-07 00:22:26 +0000758 if paired_with_pidfile:
759 drone = self._get_drone_for_pidfile_id(paired_with_pidfile)
760 else:
Dan Shiec1d47d2015-02-13 11:38:13 -0800761 drone = self._choose_drone_for_execution(
Prathmesh Prabhu89931612018-08-11 18:16:22 -0700762 num_processes, username, drone_hostnames_allowed)
jamesren76fcf192010-04-21 20:39:50 +0000763
764 if not drone:
765 raise DroneManagerError('command failed; no drones available: %s'
766 % command)
767
Dan Shiec1d47d2015-02-13 11:38:13 -0800768 logging.info("command = %s", command)
769 logging.info('log file = %s:%s', drone.hostname, log_file)
showarddb502762009-09-09 15:31:20 +0000770 self._write_attached_files(working_directory, drone)
771 drone.queue_call('execute_command', command, abs_working_directory,
showard170873e2009-01-07 00:22:26 +0000772 log_file, pidfile_name)
showard418785b2009-11-23 20:19:59 +0000773 drone.active_processes += num_processes
774 self._reorder_drone_queue()
showard170873e2009-01-07 00:22:26 +0000775
showard42d44982009-10-12 20:34:03 +0000776 pidfile_path = os.path.join(abs_working_directory, pidfile_name)
showard170873e2009-01-07 00:22:26 +0000777 pidfile_id = PidfileId(pidfile_path)
778 self.register_pidfile(pidfile_id)
showardd1195652009-12-08 22:21:02 +0000779 self._registered_pidfile_info[pidfile_id].num_processes = num_processes
showard170873e2009-01-07 00:22:26 +0000780 return pidfile_id
781
782
showardd3dc1992009-04-22 21:01:40 +0000783 def get_pidfile_id_from(self, execution_tag, pidfile_name):
784 path = os.path.join(self.absolute_path(execution_tag), pidfile_name)
showard170873e2009-01-07 00:22:26 +0000785 return PidfileId(path)
786
787
788 def register_pidfile(self, pidfile_id):
789 """
790 Indicate that the DroneManager should look for the given pidfile when
791 refreshing.
792 """
showardd1195652009-12-08 22:21:02 +0000793 if pidfile_id not in self._registered_pidfile_info:
showard37399782009-08-20 23:32:20 +0000794 logging.info('monitoring pidfile %s', pidfile_id)
showardd1195652009-12-08 22:21:02 +0000795 self._registered_pidfile_info[pidfile_id] = _PidfileInfo()
showardc6fb6042010-01-25 21:48:20 +0000796 self._reset_pidfile_age(pidfile_id)
797
798
799 def _reset_pidfile_age(self, pidfile_id):
800 if pidfile_id in self._registered_pidfile_info:
801 self._registered_pidfile_info[pidfile_id].age = 0
showard170873e2009-01-07 00:22:26 +0000802
803
showardf85a0b72009-10-07 20:48:45 +0000804 def unregister_pidfile(self, pidfile_id):
showardd1195652009-12-08 22:21:02 +0000805 if pidfile_id in self._registered_pidfile_info:
showardf85a0b72009-10-07 20:48:45 +0000806 logging.info('forgetting pidfile %s', pidfile_id)
showardd1195652009-12-08 22:21:02 +0000807 del self._registered_pidfile_info[pidfile_id]
808
809
810 def declare_process_count(self, pidfile_id, num_processes):
811 self._registered_pidfile_info[pidfile_id].num_processes = num_processes
showardf85a0b72009-10-07 20:48:45 +0000812
813
showard170873e2009-01-07 00:22:26 +0000814 def get_pidfile_contents(self, pidfile_id, use_second_read=False):
815 """
816 Retrieve a PidfileContents object for the given pidfile_id. If
817 use_second_read is True, use results that were read after the processes
818 were checked, instead of before.
819 """
showardc6fb6042010-01-25 21:48:20 +0000820 self._reset_pidfile_age(pidfile_id)
showard170873e2009-01-07 00:22:26 +0000821 if use_second_read:
822 pidfile_map = self._pidfiles_second_read
823 else:
824 pidfile_map = self._pidfiles
825 return pidfile_map.get(pidfile_id, PidfileContents())
826
827
828 def is_process_running(self, process):
829 """
830 Check if the given process is in the running process list.
831 """
Alex Millere76e2252013-08-15 09:24:27 -0700832 if process in self._process_set:
833 return True
834
Alex Miller06a5f752013-08-15 11:16:40 -0700835 drone_pid = process.hostname, process.pid
Alex Millere76e2252013-08-15 09:24:27 -0700836 if drone_pid in self._all_processes:
837 logging.error('Process %s found, but not an autoserv process. '
838 'Is %s', process, self._all_processes[drone_pid])
839 return True
840
841 return False
showard170873e2009-01-07 00:22:26 +0000842
843
844 def get_temporary_path(self, base_name):
845 """
846 Get a new temporary path guaranteed to be unique across all drones
847 for this scheduler execution.
848 """
849 self._temporary_path_counter += 1
850 return os.path.join(drone_utility._TEMPORARY_DIRECTORY,
851 '%s.%s' % (base_name, self._temporary_path_counter))
852
853
showard42d44982009-10-12 20:34:03 +0000854 def absolute_path(self, path, on_results_repository=False):
855 if on_results_repository:
856 base_dir = self._results_dir
857 else:
showardc75fded2009-10-14 16:20:02 +0000858 base_dir = os.path.join(drones.AUTOTEST_INSTALL_DIR,
859 _DRONE_RESULTS_DIR_SUFFIX)
showard42d44982009-10-12 20:34:03 +0000860 return os.path.join(base_dir, path)
showard170873e2009-01-07 00:22:26 +0000861
862
showard678df4f2009-02-04 21:36:39 +0000863 def _copy_results_helper(self, process, source_path, destination_path,
864 to_results_repository=False):
Simran Basi882f15b2013-10-29 14:59:34 -0700865 logging.debug('_copy_results_helper. process: %s, source_path: %s, '
866 'destination_path: %s, to_results_repository: %s',
867 process, source_path, destination_path,
868 to_results_repository)
showard678df4f2009-02-04 21:36:39 +0000869 full_source = self.absolute_path(source_path)
showard42d44982009-10-12 20:34:03 +0000870 full_destination = self.absolute_path(
871 destination_path, on_results_repository=to_results_repository)
showard678df4f2009-02-04 21:36:39 +0000872 source_drone = self._get_drone_for_process(process)
873 if to_results_repository:
874 source_drone.send_file_to(self._results_drone, full_source,
875 full_destination, can_fail=True)
876 else:
877 source_drone.queue_call('copy_file_or_directory', full_source,
878 full_destination)
879
880
showard170873e2009-01-07 00:22:26 +0000881 def copy_to_results_repository(self, process, source_path,
882 destination_path=None):
883 """
884 Copy results from the given process at source_path to destination_path
885 in the results repository.
Allen Li036b3a62017-02-06 15:19:43 -0800886
887 This will only copy the results back for Special Agent Tasks (Cleanup,
888 Verify, Repair) that reside in the hosts/ subdirectory of results if
889 the copy_task_results_back flag has been set to True inside
890 global_config.ini
891
892 It will also only copy .parse.log files back to the scheduler if the
893 copy_parse_log_back flag in global_config.ini has been set to True.
894 """
895 if not ENABLE_ARCHIVING:
896 return
897 copy_task_results_back = global_config.global_config.get_config_value(
898 scheduler_config.CONFIG_SECTION, 'copy_task_results_back',
899 type=bool)
900 copy_parse_log_back = global_config.global_config.get_config_value(
901 scheduler_config.CONFIG_SECTION, 'copy_parse_log_back',
902 type=bool)
903 special_task = source_path.startswith(HOSTS_JOB_SUBDIR)
904 parse_log = source_path.endswith(PARSE_LOG)
905 if (copy_task_results_back or not special_task) and (
906 copy_parse_log_back or not parse_log):
907 if destination_path is None:
908 destination_path = source_path
909 self._copy_results_helper(process, source_path, destination_path,
910 to_results_repository=True)
911
912 def _copy_to_results_repository(self, process, source_path,
913 destination_path=None):
914 """
915 Copy results from the given process at source_path to destination_path
916 in the results repository, without special task handling.
showard170873e2009-01-07 00:22:26 +0000917 """
918 if destination_path is None:
919 destination_path = source_path
showard678df4f2009-02-04 21:36:39 +0000920 self._copy_results_helper(process, source_path, destination_path,
921 to_results_repository=True)
922
923
924 def copy_results_on_drone(self, process, source_path, destination_path):
925 """
926 Copy a results directory from one place to another on the drone.
927 """
928 self._copy_results_helper(process, source_path, destination_path)
showard170873e2009-01-07 00:22:26 +0000929
930
showarddb502762009-09-09 15:31:20 +0000931 def _write_attached_files(self, results_dir, drone):
932 attached_files = self._attached_files.pop(results_dir, {})
showard73ec0442009-02-07 02:05:20 +0000933 for file_path, contents in attached_files.iteritems():
showard170873e2009-01-07 00:22:26 +0000934 drone.queue_call('write_to_file', self.absolute_path(file_path),
935 contents)
936
937
showarddb502762009-09-09 15:31:20 +0000938 def attach_file_to_execution(self, results_dir, file_contents,
showard170873e2009-01-07 00:22:26 +0000939 file_path=None):
940 """
showarddb502762009-09-09 15:31:20 +0000941 When the process for the results directory is executed, the given file
942 contents will be placed in a file on the drone. Returns the path at
943 which the file will be placed.
showard170873e2009-01-07 00:22:26 +0000944 """
945 if not file_path:
946 file_path = self.get_temporary_path('attach')
showarddb502762009-09-09 15:31:20 +0000947 files_for_execution = self._attached_files.setdefault(results_dir, {})
showard73ec0442009-02-07 02:05:20 +0000948 assert file_path not in files_for_execution
949 files_for_execution[file_path] = file_contents
showard170873e2009-01-07 00:22:26 +0000950 return file_path
951
952
showard35162b02009-03-03 02:17:30 +0000953 def write_lines_to_file(self, file_path, lines, paired_with_process=None):
showard170873e2009-01-07 00:22:26 +0000954 """
955 Write the given lines (as a list of strings) to a file. If
showard35162b02009-03-03 02:17:30 +0000956 paired_with_process is given, the file will be written on the drone
957 running the given Process. Otherwise, the file will be written to the
showard170873e2009-01-07 00:22:26 +0000958 results repository.
959 """
showard170873e2009-01-07 00:22:26 +0000960 file_contents = '\n'.join(lines) + '\n'
showard35162b02009-03-03 02:17:30 +0000961 if paired_with_process:
962 drone = self._get_drone_for_process(paired_with_process)
showard42d44982009-10-12 20:34:03 +0000963 on_results_repository = False
showard170873e2009-01-07 00:22:26 +0000964 else:
965 drone = self._results_drone
showard42d44982009-10-12 20:34:03 +0000966 on_results_repository = True
967 full_path = self.absolute_path(
968 file_path, on_results_repository=on_results_repository)
showard170873e2009-01-07 00:22:26 +0000969 drone.queue_call('write_to_file', full_path, file_contents)
jamesrenc44ae992010-02-19 00:12:54 +0000970
971
972_the_instance = None
973
974def instance():
975 if _the_instance is None:
976 _set_instance(DroneManager())
977 return _the_instance
978
979
980def _set_instance(instance): # usable for testing
981 global _the_instance
982 _the_instance = instance