showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 1 | import os, re, shutil, signal, subprocess, errno, time, heapq, traceback |
showard | b18134f | 2009-03-20 20:52:18 +0000 | [diff] [blame] | 2 | import common, logging |
Simran Basi | cced309 | 2012-08-02 15:09:23 -0700 | [diff] [blame] | 3 | from autotest_lib.client.common_lib import error, global_config, utils |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 4 | from autotest_lib.scheduler import email_manager, drone_utility, drones |
showard | 324bf81 | 2009-01-20 23:23:38 +0000 | [diff] [blame] | 5 | from autotest_lib.scheduler import scheduler_config |
Alex Miller | 6cbd758 | 2014-05-14 19:06:52 -0700 | [diff] [blame] | 6 | from autotest_lib.site_utils.graphite import stats |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 7 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 8 | |
showard | c75fded | 2009-10-14 16:20:02 +0000 | [diff] [blame] | 9 | # results on drones will be placed under the drone_installation_directory in a |
| 10 | # directory with this name |
| 11 | _DRONE_RESULTS_DIR_SUFFIX = 'results' |
| 12 | |
showard | ed2afea | 2009-07-07 20:54:07 +0000 | [diff] [blame] | 13 | WORKING_DIRECTORY = object() # see execute_command() |
| 14 | |
showard | 8d3dbca | 2009-09-25 20:29:38 +0000 | [diff] [blame] | 15 | |
jamesren | c44ae99 | 2010-02-19 00:12:54 +0000 | [diff] [blame] | 16 | AUTOSERV_PID_FILE = '.autoserv_execute' |
| 17 | CRASHINFO_PID_FILE = '.collect_crashinfo_execute' |
| 18 | PARSER_PID_FILE = '.parser_execute' |
| 19 | ARCHIVER_PID_FILE = '.archiver_execute' |
| 20 | |
| 21 | ALL_PIDFILE_NAMES = (AUTOSERV_PID_FILE, CRASHINFO_PID_FILE, PARSER_PID_FILE, |
| 22 | ARCHIVER_PID_FILE) |
| 23 | |
| 24 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 25 | class DroneManagerError(Exception): |
| 26 | pass |
| 27 | |
| 28 | |
| 29 | class CustomEquals(object): |
| 30 | def _id(self): |
| 31 | raise NotImplementedError |
| 32 | |
| 33 | |
| 34 | def __eq__(self, other): |
| 35 | if not isinstance(other, type(self)): |
| 36 | return NotImplemented |
| 37 | return self._id() == other._id() |
| 38 | |
| 39 | |
| 40 | def __ne__(self, other): |
| 41 | return not self == other |
| 42 | |
| 43 | |
| 44 | def __hash__(self): |
| 45 | return hash(self._id()) |
| 46 | |
| 47 | |
| 48 | class Process(CustomEquals): |
| 49 | def __init__(self, hostname, pid, ppid=None): |
| 50 | self.hostname = hostname |
| 51 | self.pid = pid |
| 52 | self.ppid = ppid |
| 53 | |
| 54 | def _id(self): |
| 55 | return (self.hostname, self.pid) |
| 56 | |
| 57 | |
| 58 | def __str__(self): |
| 59 | return '%s/%s' % (self.hostname, self.pid) |
| 60 | |
| 61 | |
| 62 | def __repr__(self): |
| 63 | return super(Process, self).__repr__() + '<%s>' % self |
| 64 | |
| 65 | |
| 66 | class PidfileId(CustomEquals): |
| 67 | def __init__(self, path): |
| 68 | self.path = path |
| 69 | |
| 70 | |
| 71 | def _id(self): |
| 72 | return self.path |
| 73 | |
| 74 | |
| 75 | def __str__(self): |
| 76 | return str(self.path) |
| 77 | |
| 78 | |
showard | d119565 | 2009-12-08 22:21:02 +0000 | [diff] [blame] | 79 | class _PidfileInfo(object): |
| 80 | age = 0 |
| 81 | num_processes = None |
| 82 | |
| 83 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 84 | class PidfileContents(object): |
| 85 | process = None |
| 86 | exit_status = None |
| 87 | num_tests_failed = None |
| 88 | |
| 89 | def is_invalid(self): |
| 90 | return False |
| 91 | |
| 92 | |
showard | d119565 | 2009-12-08 22:21:02 +0000 | [diff] [blame] | 93 | def is_running(self): |
| 94 | return self.process and not self.exit_status |
| 95 | |
| 96 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 97 | class InvalidPidfile(object): |
Simran Basi | 899f9fe | 2013-02-27 11:58:49 -0800 | [diff] [blame] | 98 | process = None |
| 99 | exit_status = None |
| 100 | num_tests_failed = None |
Simran Basi | 4d7bca2 | 2013-02-27 10:57:04 -0800 | [diff] [blame] | 101 | |
| 102 | |
Simran Basi | 899f9fe | 2013-02-27 11:58:49 -0800 | [diff] [blame] | 103 | def __init__(self, error): |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 104 | self.error = error |
| 105 | |
| 106 | |
| 107 | def is_invalid(self): |
| 108 | return True |
| 109 | |
| 110 | |
showard | d119565 | 2009-12-08 22:21:02 +0000 | [diff] [blame] | 111 | def is_running(self): |
| 112 | return False |
| 113 | |
| 114 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 115 | def __str__(self): |
| 116 | return self.error |
| 117 | |
| 118 | |
showard | 418785b | 2009-11-23 20:19:59 +0000 | [diff] [blame] | 119 | class _DroneHeapWrapper(object): |
| 120 | """Wrapper to compare drones based on used_capacity(). |
| 121 | |
| 122 | These objects can be used to keep a heap of drones by capacity. |
| 123 | """ |
| 124 | def __init__(self, drone): |
| 125 | self.drone = drone |
| 126 | |
| 127 | |
| 128 | def __cmp__(self, other): |
| 129 | assert isinstance(other, _DroneHeapWrapper) |
| 130 | return cmp(self.drone.used_capacity(), other.drone.used_capacity()) |
| 131 | |
| 132 | |
Simran Basi | cced309 | 2012-08-02 15:09:23 -0700 | [diff] [blame] | 133 | class BaseDroneManager(object): |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 134 | """ |
| 135 | This class acts as an interface from the scheduler to drones, whether it be |
| 136 | only a single "drone" for localhost or multiple remote drones. |
| 137 | |
| 138 | All paths going into and out of this class are relative to the full results |
| 139 | directory, except for those returns by absolute_path(). |
| 140 | """ |
Fang Deng | 9a0c6c3 | 2013-09-04 15:34:55 -0700 | [diff] [blame] | 141 | |
| 142 | |
| 143 | # Minimum time to wait before next email |
| 144 | # about a drone hitting process limit is sent. |
| 145 | NOTIFY_INTERVAL = 60 * 60 * 24 # one day |
Alex Miller | 6cbd758 | 2014-05-14 19:06:52 -0700 | [diff] [blame] | 146 | _timer = stats.Timer('drone_manager') |
Fang Deng | 9a0c6c3 | 2013-09-04 15:34:55 -0700 | [diff] [blame] | 147 | |
| 148 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 149 | def __init__(self): |
showard | d119565 | 2009-12-08 22:21:02 +0000 | [diff] [blame] | 150 | # absolute path of base results dir |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 151 | self._results_dir = None |
showard | d119565 | 2009-12-08 22:21:02 +0000 | [diff] [blame] | 152 | # holds Process objects |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 153 | self._process_set = set() |
Alex Miller | e76e225 | 2013-08-15 09:24:27 -0700 | [diff] [blame] | 154 | # holds the list of all processes running on all drones |
| 155 | self._all_processes = {} |
showard | d119565 | 2009-12-08 22:21:02 +0000 | [diff] [blame] | 156 | # maps PidfileId to PidfileContents |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 157 | self._pidfiles = {} |
showard | d119565 | 2009-12-08 22:21:02 +0000 | [diff] [blame] | 158 | # same as _pidfiles |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 159 | self._pidfiles_second_read = {} |
showard | d119565 | 2009-12-08 22:21:02 +0000 | [diff] [blame] | 160 | # maps PidfileId to _PidfileInfo |
| 161 | self._registered_pidfile_info = {} |
| 162 | # used to generate unique temporary paths |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 163 | self._temporary_path_counter = 0 |
showard | d119565 | 2009-12-08 22:21:02 +0000 | [diff] [blame] | 164 | # maps hostname to Drone object |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 165 | self._drones = {} |
| 166 | self._results_drone = None |
showard | d119565 | 2009-12-08 22:21:02 +0000 | [diff] [blame] | 167 | # maps results dir to dict mapping file path to contents |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 168 | self._attached_files = {} |
showard | 418785b | 2009-11-23 20:19:59 +0000 | [diff] [blame] | 169 | # heapq of _DroneHeapWrappers |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 170 | self._drone_queue = [] |
Fang Deng | 9a0c6c3 | 2013-09-04 15:34:55 -0700 | [diff] [blame] | 171 | # map drone hostname to time stamp of email that |
| 172 | # has been sent about the drone hitting process limit. |
| 173 | self._notify_record = {} |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 174 | |
| 175 | |
| 176 | def initialize(self, base_results_dir, drone_hostnames, |
| 177 | results_repository_hostname): |
| 178 | self._results_dir = base_results_dir |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 179 | |
| 180 | for hostname in drone_hostnames: |
Eric Li | 861b2d5 | 2011-02-04 14:50:35 -0800 | [diff] [blame] | 181 | self._add_drone(hostname) |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 182 | |
| 183 | if not self._drones: |
| 184 | # all drones failed to initialize |
| 185 | raise DroneManagerError('No valid drones found') |
| 186 | |
showard | 324bf81 | 2009-01-20 23:23:38 +0000 | [diff] [blame] | 187 | self.refresh_drone_configs() |
showard | c5afc46 | 2009-01-13 00:09:39 +0000 | [diff] [blame] | 188 | |
showard | 4460ee8 | 2009-07-07 20:54:29 +0000 | [diff] [blame] | 189 | logging.info('Using results repository on %s', |
showard | b18134f | 2009-03-20 20:52:18 +0000 | [diff] [blame] | 190 | results_repository_hostname) |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 191 | self._results_drone = drones.get_drone(results_repository_hostname) |
showard | ac5b000 | 2009-10-19 18:34:00 +0000 | [diff] [blame] | 192 | results_installation_dir = global_config.global_config.get_config_value( |
| 193 | scheduler_config.CONFIG_SECTION, |
| 194 | 'results_host_installation_directory', default=None) |
| 195 | if results_installation_dir: |
| 196 | self._results_drone.set_autotest_install_dir( |
| 197 | results_installation_dir) |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 198 | # don't initialize() the results drone - we don't want to clear out any |
showard | d119565 | 2009-12-08 22:21:02 +0000 | [diff] [blame] | 199 | # directories and we don't need to kill any processes |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 200 | |
| 201 | |
| 202 | def reinitialize_drones(self): |
| 203 | self._call_all_drones('initialize', self._results_dir) |
| 204 | |
| 205 | |
| 206 | def shutdown(self): |
showard | 324bf81 | 2009-01-20 23:23:38 +0000 | [diff] [blame] | 207 | for drone in self.get_drones(): |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 208 | drone.shutdown() |
| 209 | |
| 210 | |
showard | 8d3dbca | 2009-09-25 20:29:38 +0000 | [diff] [blame] | 211 | def _get_max_pidfile_refreshes(self): |
| 212 | """ |
| 213 | Normally refresh() is called on every monitor_db.Dispatcher.tick(). |
| 214 | |
| 215 | @returns: The number of refresh() calls before we forget a pidfile. |
| 216 | """ |
| 217 | pidfile_timeout = global_config.global_config.get_config_value( |
| 218 | scheduler_config.CONFIG_SECTION, 'max_pidfile_refreshes', |
| 219 | type=int, default=2000) |
| 220 | return pidfile_timeout |
| 221 | |
| 222 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 223 | def _add_drone(self, hostname): |
showard | b18134f | 2009-03-20 20:52:18 +0000 | [diff] [blame] | 224 | logging.info('Adding drone %s' % hostname) |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 225 | drone = drones.get_drone(hostname) |
Eric Li | 861b2d5 | 2011-02-04 14:50:35 -0800 | [diff] [blame] | 226 | if drone: |
| 227 | self._drones[drone.hostname] = drone |
| 228 | drone.call('initialize', self.absolute_path('')) |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 229 | |
| 230 | |
| 231 | def _remove_drone(self, hostname): |
| 232 | self._drones.pop(hostname, None) |
| 233 | |
| 234 | |
showard | 324bf81 | 2009-01-20 23:23:38 +0000 | [diff] [blame] | 235 | def refresh_drone_configs(self): |
showard | c5afc46 | 2009-01-13 00:09:39 +0000 | [diff] [blame] | 236 | """ |
showard | 324bf81 | 2009-01-20 23:23:38 +0000 | [diff] [blame] | 237 | Reread global config options for all drones. |
showard | c5afc46 | 2009-01-13 00:09:39 +0000 | [diff] [blame] | 238 | """ |
showard | 324bf81 | 2009-01-20 23:23:38 +0000 | [diff] [blame] | 239 | config = global_config.global_config |
| 240 | section = scheduler_config.CONFIG_SECTION |
| 241 | config.parse_config_file() |
showard | c5afc46 | 2009-01-13 00:09:39 +0000 | [diff] [blame] | 242 | for hostname, drone in self._drones.iteritems(): |
showard | 324bf81 | 2009-01-20 23:23:38 +0000 | [diff] [blame] | 243 | disabled = config.get_config_value( |
showard | 9bb960b | 2009-11-19 01:02:11 +0000 | [diff] [blame] | 244 | section, '%s_disabled' % hostname, default='') |
showard | c5afc46 | 2009-01-13 00:09:39 +0000 | [diff] [blame] | 245 | drone.enabled = not bool(disabled) |
| 246 | |
showard | 324bf81 | 2009-01-20 23:23:38 +0000 | [diff] [blame] | 247 | drone.max_processes = config.get_config_value( |
showard | 9bb960b | 2009-11-19 01:02:11 +0000 | [diff] [blame] | 248 | section, '%s_max_processes' % hostname, type=int, |
| 249 | default=scheduler_config.config.max_processes_per_drone) |
| 250 | |
| 251 | allowed_users = config.get_config_value( |
| 252 | section, '%s_users' % hostname, default=None) |
| 253 | if allowed_users is not None: |
showard | 1b7142d | 2010-01-15 00:21:37 +0000 | [diff] [blame] | 254 | allowed_users = set(allowed_users.split()) |
| 255 | drone.allowed_users = allowed_users |
showard | c5afc46 | 2009-01-13 00:09:39 +0000 | [diff] [blame] | 256 | |
showard | 418785b | 2009-11-23 20:19:59 +0000 | [diff] [blame] | 257 | self._reorder_drone_queue() # max_processes may have changed |
Fang Deng | 9a0c6c3 | 2013-09-04 15:34:55 -0700 | [diff] [blame] | 258 | # Clear notification record about reaching max_processes limit. |
| 259 | self._notify_record = {} |
showard | 418785b | 2009-11-23 20:19:59 +0000 | [diff] [blame] | 260 | |
showard | c5afc46 | 2009-01-13 00:09:39 +0000 | [diff] [blame] | 261 | |
showard | 324bf81 | 2009-01-20 23:23:38 +0000 | [diff] [blame] | 262 | def get_drones(self): |
| 263 | return self._drones.itervalues() |
showard | c5afc46 | 2009-01-13 00:09:39 +0000 | [diff] [blame] | 264 | |
| 265 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 266 | def _get_drone_for_process(self, process): |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 267 | return self._drones[process.hostname] |
| 268 | |
| 269 | |
| 270 | def _get_drone_for_pidfile_id(self, pidfile_id): |
| 271 | pidfile_contents = self.get_pidfile_contents(pidfile_id) |
| 272 | assert pidfile_contents.process is not None |
| 273 | return self._get_drone_for_process(pidfile_contents.process) |
| 274 | |
| 275 | |
| 276 | def _drop_old_pidfiles(self): |
showard | d349624 | 2009-12-10 21:41:43 +0000 | [diff] [blame] | 277 | # use items() since the dict is modified in unregister_pidfile() |
| 278 | for pidfile_id, info in self._registered_pidfile_info.items(): |
showard | d119565 | 2009-12-08 22:21:02 +0000 | [diff] [blame] | 279 | if info.age > self._get_max_pidfile_refreshes(): |
showard | f85a0b7 | 2009-10-07 20:48:45 +0000 | [diff] [blame] | 280 | logging.warning('dropping leaked pidfile %s', pidfile_id) |
| 281 | self.unregister_pidfile(pidfile_id) |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 282 | else: |
showard | d119565 | 2009-12-08 22:21:02 +0000 | [diff] [blame] | 283 | info.age += 1 |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 284 | |
| 285 | |
| 286 | def _reset(self): |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 287 | self._process_set = set() |
Alex Miller | e76e225 | 2013-08-15 09:24:27 -0700 | [diff] [blame] | 288 | self._all_processes = {} |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 289 | self._pidfiles = {} |
| 290 | self._pidfiles_second_read = {} |
| 291 | self._drone_queue = [] |
| 292 | |
| 293 | |
| 294 | def _call_all_drones(self, method, *args, **kwargs): |
| 295 | all_results = {} |
showard | 324bf81 | 2009-01-20 23:23:38 +0000 | [diff] [blame] | 296 | for drone in self.get_drones(): |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 297 | all_results[drone] = drone.call(method, *args, **kwargs) |
| 298 | return all_results |
| 299 | |
| 300 | |
| 301 | def _parse_pidfile(self, drone, raw_contents): |
| 302 | contents = PidfileContents() |
| 303 | if not raw_contents: |
| 304 | return contents |
| 305 | lines = raw_contents.splitlines() |
| 306 | if len(lines) > 3: |
| 307 | return InvalidPidfile('Corrupt pid file (%d lines):\n%s' % |
| 308 | (len(lines), lines)) |
| 309 | try: |
| 310 | pid = int(lines[0]) |
| 311 | contents.process = Process(drone.hostname, pid) |
| 312 | # if len(lines) == 2, assume we caught Autoserv between writing |
| 313 | # exit_status and num_failed_tests, so just ignore it and wait for |
| 314 | # the next cycle |
| 315 | if len(lines) == 3: |
| 316 | contents.exit_status = int(lines[1]) |
| 317 | contents.num_tests_failed = int(lines[2]) |
| 318 | except ValueError, exc: |
| 319 | return InvalidPidfile('Corrupt pid file: ' + str(exc.args)) |
| 320 | |
| 321 | return contents |
| 322 | |
| 323 | |
| 324 | def _process_pidfiles(self, drone, pidfiles, store_in_dict): |
| 325 | for pidfile_path, contents in pidfiles.iteritems(): |
| 326 | pidfile_id = PidfileId(pidfile_path) |
| 327 | contents = self._parse_pidfile(drone, contents) |
| 328 | store_in_dict[pidfile_id] = contents |
| 329 | |
| 330 | |
showard | 0205a3e | 2009-01-16 03:03:50 +0000 | [diff] [blame] | 331 | def _add_process(self, drone, process_info): |
| 332 | process = Process(drone.hostname, int(process_info['pid']), |
| 333 | int(process_info['ppid'])) |
| 334 | self._process_set.add(process) |
showard | 0205a3e | 2009-01-16 03:03:50 +0000 | [diff] [blame] | 335 | |
| 336 | |
| 337 | def _add_autoserv_process(self, drone, process_info): |
| 338 | assert process_info['comm'] == 'autoserv' |
| 339 | # only root autoserv processes have pgid == pid |
| 340 | if process_info['pgid'] != process_info['pid']: |
| 341 | return |
showard | d119565 | 2009-12-08 22:21:02 +0000 | [diff] [blame] | 342 | self._add_process(drone, process_info) |
showard | 0205a3e | 2009-01-16 03:03:50 +0000 | [diff] [blame] | 343 | |
| 344 | |
showard | 324bf81 | 2009-01-20 23:23:38 +0000 | [diff] [blame] | 345 | def _enqueue_drone(self, drone): |
showard | 418785b | 2009-11-23 20:19:59 +0000 | [diff] [blame] | 346 | heapq.heappush(self._drone_queue, _DroneHeapWrapper(drone)) |
| 347 | |
| 348 | |
| 349 | def _reorder_drone_queue(self): |
| 350 | heapq.heapify(self._drone_queue) |
showard | 324bf81 | 2009-01-20 23:23:38 +0000 | [diff] [blame] | 351 | |
| 352 | |
showard | d119565 | 2009-12-08 22:21:02 +0000 | [diff] [blame] | 353 | def _compute_active_processes(self, drone): |
| 354 | drone.active_processes = 0 |
| 355 | for pidfile_id, contents in self._pidfiles.iteritems(): |
| 356 | is_running = contents.exit_status is None |
| 357 | on_this_drone = (contents.process |
| 358 | and contents.process.hostname == drone.hostname) |
| 359 | if is_running and on_this_drone: |
| 360 | info = self._registered_pidfile_info[pidfile_id] |
| 361 | if info.num_processes is not None: |
| 362 | drone.active_processes += info.num_processes |
| 363 | |
| 364 | |
Fang Deng | 9a0c6c3 | 2013-09-04 15:34:55 -0700 | [diff] [blame] | 365 | def _check_drone_process_limit(self, drone): |
| 366 | """ |
| 367 | Notify if the number of processes on |drone| is approaching limit. |
| 368 | |
| 369 | @param drone: A Drone object. |
| 370 | """ |
Alex Miller | da713d9 | 2013-12-06 10:02:43 -0800 | [diff] [blame] | 371 | try: |
| 372 | percent = float(drone.active_processes) / drone.max_processes |
| 373 | except ZeroDivisionError: |
| 374 | percent = 100 |
Fang Deng | 9a0c6c3 | 2013-09-04 15:34:55 -0700 | [diff] [blame] | 375 | max_percent = scheduler_config.config.max_processes_warning_threshold |
| 376 | if percent >= max_percent: |
| 377 | message = ('Drone %s is hitting %s of process limit.' % |
| 378 | (drone.hostname, format(percent, '.2%'))) |
| 379 | logging.warning(message) |
| 380 | last_notified = self._notify_record.get(drone.hostname, 0) |
| 381 | now = time.time() |
| 382 | if last_notified + BaseDroneManager.NOTIFY_INTERVAL < now: |
| 383 | body = ('Active processes/Process limit: %d/%d (%s)' % |
| 384 | (drone.active_processes, drone.max_processes, |
| 385 | format(percent, '.2%'))) |
| 386 | email_manager.manager.enqueue_notify_email(message, body) |
| 387 | self._notify_record[drone.hostname] = now |
| 388 | |
| 389 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 390 | def refresh(self): |
| 391 | """ |
| 392 | Called at the beginning of a scheduler cycle to refresh all process |
| 393 | information. |
| 394 | """ |
| 395 | self._reset() |
showard | bf9695d | 2009-07-06 20:22:24 +0000 | [diff] [blame] | 396 | self._drop_old_pidfiles() |
showard | d119565 | 2009-12-08 22:21:02 +0000 | [diff] [blame] | 397 | pidfile_paths = [pidfile_id.path |
| 398 | for pidfile_id in self._registered_pidfile_info] |
Alex Miller | 6cbd758 | 2014-05-14 19:06:52 -0700 | [diff] [blame] | 399 | with self._timer.get_client('refresh'): |
| 400 | all_results = self._call_all_drones('refresh', pidfile_paths) |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 401 | |
| 402 | for drone, results_list in all_results.iteritems(): |
| 403 | results = results_list[0] |
showard | 0205a3e | 2009-01-16 03:03:50 +0000 | [diff] [blame] | 404 | |
Alex Miller | 6cbd758 | 2014-05-14 19:06:52 -0700 | [diff] [blame] | 405 | with self._timer.get_client('%s.results' % drone): |
| 406 | for process_info in results['all_processes']: |
| 407 | if process_info['comm'] == 'autoserv': |
| 408 | self._add_autoserv_process(drone, process_info) |
| 409 | drone_pid = drone.hostname, int(process_info['pid']) |
| 410 | self._all_processes[drone_pid] = process_info |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 411 | |
Alex Miller | 6cbd758 | 2014-05-14 19:06:52 -0700 | [diff] [blame] | 412 | for process_info in results['parse_processes']: |
| 413 | self._add_process(drone, process_info) |
| 414 | |
| 415 | with self._timer.get_client('%s.pidfiles' % drone): |
| 416 | self._process_pidfiles(drone, results['pidfiles'], |
| 417 | self._pidfiles) |
| 418 | with self._timer.get_client('%s.pidfiles_second' % drone): |
| 419 | self._process_pidfiles(drone, results['pidfiles_second_read'], |
| 420 | self._pidfiles_second_read) |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 421 | |
showard | d119565 | 2009-12-08 22:21:02 +0000 | [diff] [blame] | 422 | self._compute_active_processes(drone) |
| 423 | if drone.enabled: |
| 424 | self._enqueue_drone(drone) |
Fang Deng | 9a0c6c3 | 2013-09-04 15:34:55 -0700 | [diff] [blame] | 425 | self._check_drone_process_limit(drone) |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 426 | |
| 427 | |
| 428 | def execute_actions(self): |
| 429 | """ |
| 430 | Called at the end of a scheduler cycle to execute all queued actions |
| 431 | on drones. |
| 432 | """ |
| 433 | for drone in self._drones.values(): |
| 434 | drone.execute_queued_calls() |
| 435 | |
| 436 | try: |
mbligh | 1ef218d | 2009-08-03 16:57:56 +0000 | [diff] [blame] | 437 | self._results_drone.execute_queued_calls() |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 438 | except error.AutoservError: |
| 439 | warning = ('Results repository failed to execute calls:\n' + |
| 440 | traceback.format_exc()) |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 441 | email_manager.manager.enqueue_notify_email( |
| 442 | 'Results repository error', warning) |
| 443 | self._results_drone.clear_call_queue() |
| 444 | |
| 445 | |
| 446 | def get_orphaned_autoserv_processes(self): |
| 447 | """ |
showard | d3dc199 | 2009-04-22 21:01:40 +0000 | [diff] [blame] | 448 | Returns a set of Process objects for orphaned processes only. |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 449 | """ |
showard | d3dc199 | 2009-04-22 21:01:40 +0000 | [diff] [blame] | 450 | return set(process for process in self._process_set |
| 451 | if process.ppid == 1) |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 452 | |
| 453 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 454 | def kill_process(self, process): |
| 455 | """ |
| 456 | Kill the given process. |
| 457 | """ |
showard | d3dc199 | 2009-04-22 21:01:40 +0000 | [diff] [blame] | 458 | logging.info('killing %s', process) |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 459 | drone = self._get_drone_for_process(process) |
| 460 | drone.queue_call('kill_process', process) |
| 461 | |
| 462 | |
| 463 | def _ensure_directory_exists(self, path): |
| 464 | if not os.path.exists(path): |
| 465 | os.makedirs(path) |
| 466 | |
| 467 | |
showard | 324bf81 | 2009-01-20 23:23:38 +0000 | [diff] [blame] | 468 | def total_running_processes(self): |
| 469 | return sum(drone.active_processes for drone in self.get_drones()) |
| 470 | |
| 471 | |
jamesren | 76fcf19 | 2010-04-21 20:39:50 +0000 | [diff] [blame] | 472 | def max_runnable_processes(self, username, drone_hostnames_allowed): |
showard | 324bf81 | 2009-01-20 23:23:38 +0000 | [diff] [blame] | 473 | """ |
| 474 | Return the maximum number of processes that can be run (in a single |
| 475 | execution) given the current load on drones. |
showard | 9bb960b | 2009-11-19 01:02:11 +0000 | [diff] [blame] | 476 | @param username: login of user to run a process. may be None. |
jamesren | 76fcf19 | 2010-04-21 20:39:50 +0000 | [diff] [blame] | 477 | @param drone_hostnames_allowed: list of drones that can be used. May be |
| 478 | None |
showard | 324bf81 | 2009-01-20 23:23:38 +0000 | [diff] [blame] | 479 | """ |
showard | 1b7142d | 2010-01-15 00:21:37 +0000 | [diff] [blame] | 480 | usable_drone_wrappers = [wrapper for wrapper in self._drone_queue |
jamesren | 76fcf19 | 2010-04-21 20:39:50 +0000 | [diff] [blame] | 481 | if wrapper.drone.usable_by(username) and |
| 482 | (drone_hostnames_allowed is None or |
| 483 | wrapper.drone.hostname in |
| 484 | drone_hostnames_allowed)] |
showard | 1b7142d | 2010-01-15 00:21:37 +0000 | [diff] [blame] | 485 | if not usable_drone_wrappers: |
| 486 | # all drones disabled or inaccessible |
showard | de700d3 | 2009-02-25 00:12:42 +0000 | [diff] [blame] | 487 | return 0 |
jamesren | 37b5045 | 2010-03-25 20:38:56 +0000 | [diff] [blame] | 488 | runnable_processes = [ |
| 489 | wrapper.drone.max_processes - wrapper.drone.active_processes |
| 490 | for wrapper in usable_drone_wrappers] |
| 491 | return max([0] + runnable_processes) |
showard | 324bf81 | 2009-01-20 23:23:38 +0000 | [diff] [blame] | 492 | |
| 493 | |
showard | e39ebe9 | 2009-06-18 23:14:48 +0000 | [diff] [blame] | 494 | def _least_loaded_drone(self, drones): |
| 495 | drone_to_use = drones[0] |
| 496 | for drone in drones[1:]: |
| 497 | if drone.used_capacity() < drone_to_use.used_capacity(): |
| 498 | drone_to_use = drone |
| 499 | return drone_to_use |
| 500 | |
| 501 | |
jamesren | 76fcf19 | 2010-04-21 20:39:50 +0000 | [diff] [blame] | 502 | def _choose_drone_for_execution(self, num_processes, username, |
| 503 | drone_hostnames_allowed): |
showard | 324bf81 | 2009-01-20 23:23:38 +0000 | [diff] [blame] | 504 | # cycle through drones is order of increasing used capacity until |
| 505 | # we find one that can handle these processes |
| 506 | checked_drones = [] |
jamesren | 37b5045 | 2010-03-25 20:38:56 +0000 | [diff] [blame] | 507 | usable_drones = [] |
showard | 324bf81 | 2009-01-20 23:23:38 +0000 | [diff] [blame] | 508 | drone_to_use = None |
| 509 | while self._drone_queue: |
showard | 418785b | 2009-11-23 20:19:59 +0000 | [diff] [blame] | 510 | drone = heapq.heappop(self._drone_queue).drone |
showard | 324bf81 | 2009-01-20 23:23:38 +0000 | [diff] [blame] | 511 | checked_drones.append(drone) |
Eric Li | e0493a4 | 2010-11-15 13:05:43 -0800 | [diff] [blame] | 512 | logging.info('Checking drone %s', drone.hostname) |
showard | 9bb960b | 2009-11-19 01:02:11 +0000 | [diff] [blame] | 513 | if not drone.usable_by(username): |
| 514 | continue |
jamesren | 76fcf19 | 2010-04-21 20:39:50 +0000 | [diff] [blame] | 515 | |
| 516 | drone_allowed = (drone_hostnames_allowed is None |
| 517 | or drone.hostname in drone_hostnames_allowed) |
| 518 | if not drone_allowed: |
Eric Li | e0493a4 | 2010-11-15 13:05:43 -0800 | [diff] [blame] | 519 | logging.debug('Drone %s not allowed: ', drone.hostname) |
jamesren | 76fcf19 | 2010-04-21 20:39:50 +0000 | [diff] [blame] | 520 | continue |
| 521 | |
jamesren | 37b5045 | 2010-03-25 20:38:56 +0000 | [diff] [blame] | 522 | usable_drones.append(drone) |
jamesren | 76fcf19 | 2010-04-21 20:39:50 +0000 | [diff] [blame] | 523 | |
showard | 324bf81 | 2009-01-20 23:23:38 +0000 | [diff] [blame] | 524 | if drone.active_processes + num_processes <= drone.max_processes: |
| 525 | drone_to_use = drone |
| 526 | break |
Eric Li | e0493a4 | 2010-11-15 13:05:43 -0800 | [diff] [blame] | 527 | logging.info('Drone %s has %d active + %s requested > %s max', |
| 528 | drone.hostname, drone.active_processes, num_processes, |
| 529 | drone.max_processes) |
showard | 324bf81 | 2009-01-20 23:23:38 +0000 | [diff] [blame] | 530 | |
jamesren | 76fcf19 | 2010-04-21 20:39:50 +0000 | [diff] [blame] | 531 | if not drone_to_use and usable_drones: |
showard | 324bf81 | 2009-01-20 23:23:38 +0000 | [diff] [blame] | 532 | drone_summary = ','.join('%s %s/%s' % (drone.hostname, |
| 533 | drone.active_processes, |
| 534 | drone.max_processes) |
jamesren | 37b5045 | 2010-03-25 20:38:56 +0000 | [diff] [blame] | 535 | for drone in usable_drones) |
| 536 | logging.error('No drone has capacity to handle %d processes (%s) ' |
| 537 | 'for user %s', num_processes, drone_summary, username) |
| 538 | drone_to_use = self._least_loaded_drone(usable_drones) |
showard | e39ebe9 | 2009-06-18 23:14:48 +0000 | [diff] [blame] | 539 | |
showard | 324bf81 | 2009-01-20 23:23:38 +0000 | [diff] [blame] | 540 | # refill _drone_queue |
| 541 | for drone in checked_drones: |
| 542 | self._enqueue_drone(drone) |
| 543 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 544 | return drone_to_use |
| 545 | |
| 546 | |
showard | ed2afea | 2009-07-07 20:54:07 +0000 | [diff] [blame] | 547 | def _substitute_working_directory_into_command(self, command, |
| 548 | working_directory): |
| 549 | for i, item in enumerate(command): |
| 550 | if item is WORKING_DIRECTORY: |
| 551 | command[i] = working_directory |
| 552 | |
| 553 | |
showard | d3dc199 | 2009-04-22 21:01:40 +0000 | [diff] [blame] | 554 | def execute_command(self, command, working_directory, pidfile_name, |
showard | 418785b | 2009-11-23 20:19:59 +0000 | [diff] [blame] | 555 | num_processes, log_file=None, paired_with_pidfile=None, |
jamesren | 76fcf19 | 2010-04-21 20:39:50 +0000 | [diff] [blame] | 556 | username=None, drone_hostnames_allowed=None): |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 557 | """ |
| 558 | Execute the given command, taken as an argv list. |
| 559 | |
showard | ed2afea | 2009-07-07 20:54:07 +0000 | [diff] [blame] | 560 | @param command: command to execute as a list. if any item is |
| 561 | WORKING_DIRECTORY, the absolute path to the working directory |
| 562 | will be substituted for it. |
| 563 | @param working_directory: directory in which the pidfile will be written |
| 564 | @param pidfile_name: name of the pidfile this process will write |
showard | d119565 | 2009-12-08 22:21:02 +0000 | [diff] [blame] | 565 | @param num_processes: number of processes to account for from this |
| 566 | execution |
showard | ed2afea | 2009-07-07 20:54:07 +0000 | [diff] [blame] | 567 | @param log_file (optional): path (in the results repository) to hold |
| 568 | command output. |
| 569 | @param paired_with_pidfile (optional): a PidfileId for an |
| 570 | already-executed process; the new process will execute on the |
| 571 | same drone as the previous process. |
showard | 9bb960b | 2009-11-19 01:02:11 +0000 | [diff] [blame] | 572 | @param username (optional): login of the user responsible for this |
| 573 | process. |
jamesren | 76fcf19 | 2010-04-21 20:39:50 +0000 | [diff] [blame] | 574 | @param drone_hostnames_allowed (optional): hostnames of the drones that |
| 575 | this command is allowed to |
| 576 | execute on |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 577 | """ |
showard | db50276 | 2009-09-09 15:31:20 +0000 | [diff] [blame] | 578 | abs_working_directory = self.absolute_path(working_directory) |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 579 | if not log_file: |
| 580 | log_file = self.get_temporary_path('execute') |
| 581 | log_file = self.absolute_path(log_file) |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 582 | |
showard | ed2afea | 2009-07-07 20:54:07 +0000 | [diff] [blame] | 583 | self._substitute_working_directory_into_command(command, |
showard | db50276 | 2009-09-09 15:31:20 +0000 | [diff] [blame] | 584 | abs_working_directory) |
showard | ed2afea | 2009-07-07 20:54:07 +0000 | [diff] [blame] | 585 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 586 | if paired_with_pidfile: |
| 587 | drone = self._get_drone_for_pidfile_id(paired_with_pidfile) |
| 588 | else: |
jamesren | 76fcf19 | 2010-04-21 20:39:50 +0000 | [diff] [blame] | 589 | drone = self._choose_drone_for_execution(num_processes, username, |
| 590 | drone_hostnames_allowed) |
| 591 | |
| 592 | if not drone: |
| 593 | raise DroneManagerError('command failed; no drones available: %s' |
| 594 | % command) |
| 595 | |
showard | b18134f | 2009-03-20 20:52:18 +0000 | [diff] [blame] | 596 | logging.info("command = %s" % command) |
| 597 | logging.info('log file = %s:%s' % (drone.hostname, log_file)) |
showard | db50276 | 2009-09-09 15:31:20 +0000 | [diff] [blame] | 598 | self._write_attached_files(working_directory, drone) |
| 599 | drone.queue_call('execute_command', command, abs_working_directory, |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 600 | log_file, pidfile_name) |
showard | 418785b | 2009-11-23 20:19:59 +0000 | [diff] [blame] | 601 | drone.active_processes += num_processes |
| 602 | self._reorder_drone_queue() |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 603 | |
showard | 42d4498 | 2009-10-12 20:34:03 +0000 | [diff] [blame] | 604 | pidfile_path = os.path.join(abs_working_directory, pidfile_name) |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 605 | pidfile_id = PidfileId(pidfile_path) |
| 606 | self.register_pidfile(pidfile_id) |
showard | d119565 | 2009-12-08 22:21:02 +0000 | [diff] [blame] | 607 | self._registered_pidfile_info[pidfile_id].num_processes = num_processes |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 608 | return pidfile_id |
| 609 | |
| 610 | |
showard | d3dc199 | 2009-04-22 21:01:40 +0000 | [diff] [blame] | 611 | def get_pidfile_id_from(self, execution_tag, pidfile_name): |
| 612 | path = os.path.join(self.absolute_path(execution_tag), pidfile_name) |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 613 | return PidfileId(path) |
| 614 | |
| 615 | |
| 616 | def register_pidfile(self, pidfile_id): |
| 617 | """ |
| 618 | Indicate that the DroneManager should look for the given pidfile when |
| 619 | refreshing. |
| 620 | """ |
showard | d119565 | 2009-12-08 22:21:02 +0000 | [diff] [blame] | 621 | if pidfile_id not in self._registered_pidfile_info: |
showard | 3739978 | 2009-08-20 23:32:20 +0000 | [diff] [blame] | 622 | logging.info('monitoring pidfile %s', pidfile_id) |
showard | d119565 | 2009-12-08 22:21:02 +0000 | [diff] [blame] | 623 | self._registered_pidfile_info[pidfile_id] = _PidfileInfo() |
showard | c6fb604 | 2010-01-25 21:48:20 +0000 | [diff] [blame] | 624 | self._reset_pidfile_age(pidfile_id) |
| 625 | |
| 626 | |
| 627 | def _reset_pidfile_age(self, pidfile_id): |
| 628 | if pidfile_id in self._registered_pidfile_info: |
| 629 | self._registered_pidfile_info[pidfile_id].age = 0 |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 630 | |
| 631 | |
showard | f85a0b7 | 2009-10-07 20:48:45 +0000 | [diff] [blame] | 632 | def unregister_pidfile(self, pidfile_id): |
showard | d119565 | 2009-12-08 22:21:02 +0000 | [diff] [blame] | 633 | if pidfile_id in self._registered_pidfile_info: |
showard | f85a0b7 | 2009-10-07 20:48:45 +0000 | [diff] [blame] | 634 | logging.info('forgetting pidfile %s', pidfile_id) |
showard | d119565 | 2009-12-08 22:21:02 +0000 | [diff] [blame] | 635 | del self._registered_pidfile_info[pidfile_id] |
| 636 | |
| 637 | |
| 638 | def declare_process_count(self, pidfile_id, num_processes): |
| 639 | self._registered_pidfile_info[pidfile_id].num_processes = num_processes |
showard | f85a0b7 | 2009-10-07 20:48:45 +0000 | [diff] [blame] | 640 | |
| 641 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 642 | def get_pidfile_contents(self, pidfile_id, use_second_read=False): |
| 643 | """ |
| 644 | Retrieve a PidfileContents object for the given pidfile_id. If |
| 645 | use_second_read is True, use results that were read after the processes |
| 646 | were checked, instead of before. |
| 647 | """ |
showard | c6fb604 | 2010-01-25 21:48:20 +0000 | [diff] [blame] | 648 | self._reset_pidfile_age(pidfile_id) |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 649 | if use_second_read: |
| 650 | pidfile_map = self._pidfiles_second_read |
| 651 | else: |
| 652 | pidfile_map = self._pidfiles |
| 653 | return pidfile_map.get(pidfile_id, PidfileContents()) |
| 654 | |
| 655 | |
| 656 | def is_process_running(self, process): |
| 657 | """ |
| 658 | Check if the given process is in the running process list. |
| 659 | """ |
Alex Miller | e76e225 | 2013-08-15 09:24:27 -0700 | [diff] [blame] | 660 | if process in self._process_set: |
| 661 | return True |
| 662 | |
Alex Miller | 06a5f75 | 2013-08-15 11:16:40 -0700 | [diff] [blame] | 663 | drone_pid = process.hostname, process.pid |
Alex Miller | e76e225 | 2013-08-15 09:24:27 -0700 | [diff] [blame] | 664 | if drone_pid in self._all_processes: |
| 665 | logging.error('Process %s found, but not an autoserv process. ' |
| 666 | 'Is %s', process, self._all_processes[drone_pid]) |
| 667 | return True |
| 668 | |
| 669 | return False |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 670 | |
| 671 | |
| 672 | def get_temporary_path(self, base_name): |
| 673 | """ |
| 674 | Get a new temporary path guaranteed to be unique across all drones |
| 675 | for this scheduler execution. |
| 676 | """ |
| 677 | self._temporary_path_counter += 1 |
| 678 | return os.path.join(drone_utility._TEMPORARY_DIRECTORY, |
| 679 | '%s.%s' % (base_name, self._temporary_path_counter)) |
| 680 | |
| 681 | |
showard | 42d4498 | 2009-10-12 20:34:03 +0000 | [diff] [blame] | 682 | def absolute_path(self, path, on_results_repository=False): |
| 683 | if on_results_repository: |
| 684 | base_dir = self._results_dir |
| 685 | else: |
showard | c75fded | 2009-10-14 16:20:02 +0000 | [diff] [blame] | 686 | base_dir = os.path.join(drones.AUTOTEST_INSTALL_DIR, |
| 687 | _DRONE_RESULTS_DIR_SUFFIX) |
showard | 42d4498 | 2009-10-12 20:34:03 +0000 | [diff] [blame] | 688 | return os.path.join(base_dir, path) |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 689 | |
| 690 | |
showard | 678df4f | 2009-02-04 21:36:39 +0000 | [diff] [blame] | 691 | def _copy_results_helper(self, process, source_path, destination_path, |
| 692 | to_results_repository=False): |
Simran Basi | 882f15b | 2013-10-29 14:59:34 -0700 | [diff] [blame] | 693 | logging.debug('_copy_results_helper. process: %s, source_path: %s, ' |
| 694 | 'destination_path: %s, to_results_repository: %s', |
| 695 | process, source_path, destination_path, |
| 696 | to_results_repository) |
showard | 678df4f | 2009-02-04 21:36:39 +0000 | [diff] [blame] | 697 | full_source = self.absolute_path(source_path) |
showard | 42d4498 | 2009-10-12 20:34:03 +0000 | [diff] [blame] | 698 | full_destination = self.absolute_path( |
| 699 | destination_path, on_results_repository=to_results_repository) |
showard | 678df4f | 2009-02-04 21:36:39 +0000 | [diff] [blame] | 700 | source_drone = self._get_drone_for_process(process) |
| 701 | if to_results_repository: |
| 702 | source_drone.send_file_to(self._results_drone, full_source, |
| 703 | full_destination, can_fail=True) |
| 704 | else: |
| 705 | source_drone.queue_call('copy_file_or_directory', full_source, |
| 706 | full_destination) |
| 707 | |
| 708 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 709 | def copy_to_results_repository(self, process, source_path, |
| 710 | destination_path=None): |
| 711 | """ |
| 712 | Copy results from the given process at source_path to destination_path |
| 713 | in the results repository. |
| 714 | """ |
| 715 | if destination_path is None: |
| 716 | destination_path = source_path |
showard | 678df4f | 2009-02-04 21:36:39 +0000 | [diff] [blame] | 717 | self._copy_results_helper(process, source_path, destination_path, |
| 718 | to_results_repository=True) |
| 719 | |
| 720 | |
| 721 | def copy_results_on_drone(self, process, source_path, destination_path): |
| 722 | """ |
| 723 | Copy a results directory from one place to another on the drone. |
| 724 | """ |
| 725 | self._copy_results_helper(process, source_path, destination_path) |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 726 | |
| 727 | |
showard | db50276 | 2009-09-09 15:31:20 +0000 | [diff] [blame] | 728 | def _write_attached_files(self, results_dir, drone): |
| 729 | attached_files = self._attached_files.pop(results_dir, {}) |
showard | 73ec044 | 2009-02-07 02:05:20 +0000 | [diff] [blame] | 730 | for file_path, contents in attached_files.iteritems(): |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 731 | drone.queue_call('write_to_file', self.absolute_path(file_path), |
| 732 | contents) |
| 733 | |
| 734 | |
showard | db50276 | 2009-09-09 15:31:20 +0000 | [diff] [blame] | 735 | def attach_file_to_execution(self, results_dir, file_contents, |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 736 | file_path=None): |
| 737 | """ |
showard | db50276 | 2009-09-09 15:31:20 +0000 | [diff] [blame] | 738 | When the process for the results directory is executed, the given file |
| 739 | contents will be placed in a file on the drone. Returns the path at |
| 740 | which the file will be placed. |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 741 | """ |
| 742 | if not file_path: |
| 743 | file_path = self.get_temporary_path('attach') |
showard | db50276 | 2009-09-09 15:31:20 +0000 | [diff] [blame] | 744 | files_for_execution = self._attached_files.setdefault(results_dir, {}) |
showard | 73ec044 | 2009-02-07 02:05:20 +0000 | [diff] [blame] | 745 | assert file_path not in files_for_execution |
| 746 | files_for_execution[file_path] = file_contents |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 747 | return file_path |
| 748 | |
| 749 | |
showard | 35162b0 | 2009-03-03 02:17:30 +0000 | [diff] [blame] | 750 | def write_lines_to_file(self, file_path, lines, paired_with_process=None): |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 751 | """ |
| 752 | Write the given lines (as a list of strings) to a file. If |
showard | 35162b0 | 2009-03-03 02:17:30 +0000 | [diff] [blame] | 753 | paired_with_process is given, the file will be written on the drone |
| 754 | running the given Process. Otherwise, the file will be written to the |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 755 | results repository. |
| 756 | """ |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 757 | file_contents = '\n'.join(lines) + '\n' |
showard | 35162b0 | 2009-03-03 02:17:30 +0000 | [diff] [blame] | 758 | if paired_with_process: |
| 759 | drone = self._get_drone_for_process(paired_with_process) |
showard | 42d4498 | 2009-10-12 20:34:03 +0000 | [diff] [blame] | 760 | on_results_repository = False |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 761 | else: |
| 762 | drone = self._results_drone |
showard | 42d4498 | 2009-10-12 20:34:03 +0000 | [diff] [blame] | 763 | on_results_repository = True |
| 764 | full_path = self.absolute_path( |
| 765 | file_path, on_results_repository=on_results_repository) |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 766 | drone.queue_call('write_to_file', full_path, file_contents) |
jamesren | c44ae99 | 2010-02-19 00:12:54 +0000 | [diff] [blame] | 767 | |
| 768 | |
Simran Basi | cced309 | 2012-08-02 15:09:23 -0700 | [diff] [blame] | 769 | SiteDroneManager = utils.import_site_class( |
| 770 | __file__, 'autotest_lib.scheduler.site_drone_manager', |
| 771 | 'SiteDroneManager', BaseDroneManager) |
| 772 | |
| 773 | |
| 774 | class DroneManager(SiteDroneManager): |
| 775 | pass |
| 776 | |
| 777 | |
jamesren | c44ae99 | 2010-02-19 00:12:54 +0000 | [diff] [blame] | 778 | _the_instance = None |
| 779 | |
| 780 | def instance(): |
| 781 | if _the_instance is None: |
| 782 | _set_instance(DroneManager()) |
| 783 | return _the_instance |
| 784 | |
| 785 | |
| 786 | def _set_instance(instance): # usable for testing |
| 787 | global _the_instance |
| 788 | _the_instance = instance |