J. Richard Barnette | fa27f5b | 2015-03-23 17:11:54 -0700 | [diff] [blame] | 1 | # Copyright 2015 The Chromium OS Authors. All rights reserved. |
| 2 | # Use of this source code is governed by a BSD-style license that can be |
| 3 | # found in the LICENSE file. |
| 4 | |
J. Richard Barnette | 8abbfd6 | 2015-06-23 12:46:54 -0700 | [diff] [blame] | 5 | """Services relating to DUT status and job history. |
| 6 | |
| 7 | The central abstraction of this module is the `HostJobHistory` |
| 8 | class. This class provides two related pieces of information |
| 9 | regarding a single DUT: |
| 10 | * A history of tests and special tasks that have run on |
| 11 | the DUT in a given time range. |
| 12 | * Whether the DUT was "working" or "broken" at a given |
| 13 | time. |
| 14 | |
| 15 | The "working" or "broken" status of a DUT is determined by |
| 16 | the DUT's special task history. At the end of any job or |
| 17 | task, the status is indicated as follows: |
| 18 | * After any successful special task, the DUT is considered |
| 19 | "working". |
| 20 | * After any failed Repair task, the DUT is considered "broken". |
| 21 | * After any other special task or after any regular test job, the |
| 22 | DUT's status is considered unchanged. |
| 23 | |
| 24 | Definitions for terms used in the code below: |
| 25 | * status task - Any special task that determines the DUT's |
| 26 | status; that is, any successful task, or any failed Repair. |
| 27 | * diagnosis interval - A time interval during which DUT status |
| 28 | changed either from "working" to "broken", or vice versa. The |
| 29 | interval starts with the last status task with the old status, |
| 30 | and ends after the first status task with the new status. |
| 31 | |
| 32 | Diagnosis intervals are interesting because they normally contain |
| 33 | the logs explaining a failure or repair event. |
| 34 | |
| 35 | """ |
| 36 | |
J. Richard Barnette | fa27f5b | 2015-03-23 17:11:54 -0700 | [diff] [blame] | 37 | import common |
J. Richard Barnette | 39255fa | 2015-04-14 17:23:41 -0700 | [diff] [blame] | 38 | from autotest_lib.frontend import setup_django_environment |
| 39 | from django.db import models as django_models |
| 40 | |
J. Richard Barnette | fa27f5b | 2015-03-23 17:11:54 -0700 | [diff] [blame] | 41 | from autotest_lib.client.common_lib import global_config |
| 42 | from autotest_lib.client.common_lib import time_utils |
J. Richard Barnette | 39255fa | 2015-04-14 17:23:41 -0700 | [diff] [blame] | 43 | from autotest_lib.frontend.afe import models as afe_models |
J. Richard Barnette | fa27f5b | 2015-03-23 17:11:54 -0700 | [diff] [blame] | 44 | from autotest_lib.site_utils.suite_scheduler import constants |
| 45 | |
| 46 | |
| 47 | # Values used to describe the diagnosis of a DUT. These values are |
| 48 | # used to indicate both DUT status after a job or task, and also |
| 49 | # diagnosis of whether the DUT was working at the end of a given |
| 50 | # time interval. |
| 51 | # |
| 52 | # UNUSED: Used when there are no events recorded in a given |
| 53 | # time interval. |
| 54 | # UNKNOWN: For an individual event, indicates that the DUT status |
| 55 | # is unchanged from the previous event. For a time interval, |
| 56 | # indicates that the DUT's status can't be determined from the |
| 57 | # DUT's history. |
| 58 | # WORKING: Indicates that the DUT was working normally after the |
| 59 | # event, or at the end of the time interval. |
| 60 | # BROKEN: Indicates that the DUT needed manual repair after the |
| 61 | # event, or at the end of the time interval. |
| 62 | # |
| 63 | UNUSED = 0 |
| 64 | UNKNOWN = 1 |
| 65 | WORKING = 2 |
| 66 | BROKEN = 3 |
| 67 | |
| 68 | |
| 69 | def parse_time(time_string): |
| 70 | """Parse time according to a canonical form. |
| 71 | |
| 72 | The "canonical" form is the form in which date/time |
| 73 | values are stored in the database. |
| 74 | |
| 75 | @param time_string Time to be parsed. |
| 76 | """ |
| 77 | return int(time_utils.to_epoch_time(time_string)) |
| 78 | |
| 79 | |
| 80 | class _JobEvent(object): |
| 81 | """Information about an event in host history. |
| 82 | |
| 83 | This remembers the relevant data from a single event in host |
| 84 | history. An event is any change in DUT state caused by a job |
| 85 | or special task. The data captured are the start and end times |
| 86 | of the event, the URL of logs to the job or task causing the |
| 87 | event, and a diagnosis of whether the DUT was working or failed |
| 88 | afterwards. |
| 89 | |
| 90 | This class is an adapter around the database model objects |
| 91 | describing jobs and special tasks. This is an abstract |
| 92 | superclass, with concrete subclasses for `HostQueueEntry` and |
| 93 | `SpecialTask` objects. |
| 94 | |
| 95 | @property start_time Time the job or task began execution. |
| 96 | @property end_time Time the job or task finished execution. |
J. Richard Barnette | af1e826 | 2016-03-04 12:55:11 -0800 | [diff] [blame] | 97 | @property id id of the event in the AFE database. |
| 98 | @property name Name of the event, derived from the AFE database. |
| 99 | @property job_status Short string describing the event's final status. |
J. Richard Barnette | fa27f5b | 2015-03-23 17:11:54 -0700 | [diff] [blame] | 100 | @property job_url URL to the logs for the event's job. |
| 101 | @property diagnosis Working status of the DUT after the event. |
| 102 | |
| 103 | """ |
| 104 | |
| 105 | get_config_value = global_config.global_config.get_config_value |
| 106 | _LOG_URL_PATTERN = get_config_value('CROS', 'log_url_pattern') |
| 107 | |
| 108 | @classmethod |
| 109 | def get_log_url(cls, afe_hostname, logdir): |
| 110 | """Return a URL to job results. |
| 111 | |
| 112 | The URL is constructed from a base URL determined by the |
| 113 | global config, plus the relative path of the job's log |
| 114 | directory. |
| 115 | |
| 116 | @param afe_hostname Hostname for autotest frontend |
| 117 | @param logdir Relative path of the results log directory. |
| 118 | |
| 119 | @return A URL to the requested results log. |
| 120 | |
| 121 | """ |
| 122 | return cls._LOG_URL_PATTERN % (afe_hostname, logdir) |
| 123 | |
| 124 | |
| 125 | def __init__(self, start_time, end_time): |
| 126 | self.start_time = parse_time(start_time) |
J. Richard Barnette | 2c9b113 | 2015-07-10 10:47:28 -0700 | [diff] [blame] | 127 | self.end_time = parse_time(end_time) |
J. Richard Barnette | fa27f5b | 2015-03-23 17:11:54 -0700 | [diff] [blame] | 128 | |
| 129 | |
| 130 | def __cmp__(self, other): |
| 131 | """Compare two jobs by their start time. |
| 132 | |
| 133 | This is a standard Python `__cmp__` method to allow sorting |
| 134 | `_JobEvent` objects by their times. |
| 135 | |
| 136 | @param other The `_JobEvent` object to compare to `self`. |
| 137 | |
| 138 | """ |
| 139 | return self.start_time - other.start_time |
| 140 | |
| 141 | |
| 142 | @property |
J. Richard Barnette | af1e826 | 2016-03-04 12:55:11 -0800 | [diff] [blame] | 143 | def id(self): |
| 144 | """Return the id of the event in the AFE database.""" |
| 145 | raise NotImplemented() |
| 146 | |
| 147 | |
| 148 | @property |
| 149 | def name(self): |
| 150 | """Return the name of the event.""" |
| 151 | raise NotImplemented() |
| 152 | |
| 153 | |
| 154 | @property |
| 155 | def job_status(self): |
| 156 | """Return a short string describing the event's final status.""" |
| 157 | raise NotImplemented() |
| 158 | |
| 159 | |
| 160 | @property |
J. Richard Barnette | fa27f5b | 2015-03-23 17:11:54 -0700 | [diff] [blame] | 161 | def job_url(self): |
| 162 | """Return the URL for this event's job logs.""" |
| 163 | raise NotImplemented() |
| 164 | |
| 165 | |
| 166 | @property |
| 167 | def diagnosis(self): |
| 168 | """Return the status of the DUT after this event. |
| 169 | |
| 170 | The diagnosis is interpreted as follows: |
| 171 | UNKNOWN - The DUT status was the same before and after |
| 172 | the event. |
| 173 | WORKING - The DUT appeared to be working after the event. |
| 174 | BROKEN - The DUT likely required manual intervention |
| 175 | after the event. |
| 176 | |
| 177 | @return A valid diagnosis value. |
| 178 | |
| 179 | """ |
| 180 | raise NotImplemented() |
| 181 | |
| 182 | |
| 183 | class _SpecialTaskEvent(_JobEvent): |
| 184 | """`_JobEvent` adapter for special tasks. |
| 185 | |
| 186 | This class wraps the standard `_JobEvent` interface around a row |
| 187 | in the `afe_special_tasks` table. |
| 188 | |
| 189 | """ |
| 190 | |
| 191 | @classmethod |
| 192 | def get_tasks(cls, afe, host_id, start_time, end_time): |
| 193 | """Return special tasks for a host in a given time range. |
| 194 | |
| 195 | Return a list of `_SpecialTaskEvent` objects representing all |
| 196 | special tasks that ran on the given host in the given time |
| 197 | range. The list is ordered as it was returned by the query |
| 198 | (i.e. unordered). |
| 199 | |
| 200 | @param afe Autotest frontend |
| 201 | @param host_id Database host id of the desired host. |
| 202 | @param start_time Start time of the range of interest. |
| 203 | @param end_time End time of the range of interest. |
| 204 | |
| 205 | @return A list of `_SpecialTaskEvent` objects. |
| 206 | |
| 207 | """ |
J. Richard Barnette | 2c9b113 | 2015-07-10 10:47:28 -0700 | [diff] [blame] | 208 | query_start = time_utils.epoch_time_to_date_string(start_time) |
| 209 | query_end = time_utils.epoch_time_to_date_string(end_time) |
J. Richard Barnette | 9f10c9f | 2015-04-13 16:44:50 -0700 | [diff] [blame] | 210 | tasks = afe.get_host_special_tasks( |
| 211 | host_id, |
J. Richard Barnette | 2c9b113 | 2015-07-10 10:47:28 -0700 | [diff] [blame] | 212 | time_started__gte=query_start, |
| 213 | time_finished__lte=query_end, |
J. Richard Barnette | fa27f5b | 2015-03-23 17:11:54 -0700 | [diff] [blame] | 214 | is_complete=1) |
| 215 | return [cls(afe.server, t) for t in tasks] |
| 216 | |
| 217 | |
J. Richard Barnette | bc9a795 | 2015-04-16 17:43:27 -0700 | [diff] [blame] | 218 | @classmethod |
| 219 | def get_status_task(cls, afe, host_id, end_time): |
| 220 | """Return the task indicating a host's status at a given time. |
| 221 | |
| 222 | The task returned determines the status of the DUT; the |
| 223 | diagnosis on the task indicates the diagnosis for the DUT at |
| 224 | the given `end_time`. |
| 225 | |
| 226 | @param afe Autotest frontend |
| 227 | @param host_id Database host id of the desired host. |
| 228 | @param end_time Find status as of this time. |
| 229 | |
| 230 | @return A `_SpecialTaskEvent` object for the requested task, |
| 231 | or `None` if no task was found. |
| 232 | |
| 233 | """ |
J. Richard Barnette | 2c9b113 | 2015-07-10 10:47:28 -0700 | [diff] [blame] | 234 | query_end = time_utils.epoch_time_to_date_string(end_time) |
| 235 | task = afe.get_host_status_task(host_id, query_end) |
J. Richard Barnette | bc9a795 | 2015-04-16 17:43:27 -0700 | [diff] [blame] | 236 | return cls(afe.server, task) if task else None |
| 237 | |
| 238 | |
J. Richard Barnette | fa27f5b | 2015-03-23 17:11:54 -0700 | [diff] [blame] | 239 | def __init__(self, afe_hostname, afetask): |
| 240 | self._afe_hostname = afe_hostname |
| 241 | self._afetask = afetask |
| 242 | super(_SpecialTaskEvent, self).__init__( |
| 243 | afetask.time_started, afetask.time_finished) |
| 244 | |
| 245 | |
| 246 | @property |
J. Richard Barnette | af1e826 | 2016-03-04 12:55:11 -0800 | [diff] [blame] | 247 | def id(self): |
| 248 | return self._afetask.id |
| 249 | |
| 250 | |
| 251 | @property |
| 252 | def name(self): |
| 253 | return self._afetask.task |
| 254 | |
| 255 | |
| 256 | @property |
| 257 | def job_status(self): |
| 258 | if self._afetask.is_aborted: |
| 259 | return 'ABORTED' |
| 260 | elif self._afetask.success: |
| 261 | return 'PASS' |
| 262 | else: |
| 263 | return 'FAIL' |
| 264 | |
| 265 | |
| 266 | @property |
J. Richard Barnette | fa27f5b | 2015-03-23 17:11:54 -0700 | [diff] [blame] | 267 | def job_url(self): |
| 268 | logdir = ('hosts/%s/%s-%s' % |
| 269 | (self._afetask.host.hostname, self._afetask.id, |
| 270 | self._afetask.task.lower())) |
| 271 | return _SpecialTaskEvent.get_log_url(self._afe_hostname, logdir) |
| 272 | |
| 273 | |
| 274 | @property |
| 275 | def diagnosis(self): |
| 276 | if self._afetask.success: |
| 277 | return WORKING |
| 278 | elif self._afetask.task == 'Repair': |
| 279 | return BROKEN |
| 280 | else: |
| 281 | return UNKNOWN |
| 282 | |
| 283 | |
| 284 | class _TestJobEvent(_JobEvent): |
| 285 | """`_JobEvent` adapter for regular test jobs. |
| 286 | |
| 287 | This class wraps the standard `_JobEvent` interface around a row |
| 288 | in the `afe_host_queue_entries` table. |
| 289 | |
| 290 | """ |
| 291 | |
| 292 | @classmethod |
| 293 | def get_hqes(cls, afe, host_id, start_time, end_time): |
| 294 | """Return HQEs for a host in a given time range. |
| 295 | |
| 296 | Return a list of `_TestJobEvent` objects representing all the |
| 297 | HQEs of all the jobs that ran on the given host in the given |
| 298 | time range. The list is ordered as it was returned by the |
| 299 | query (i.e. unordered). |
| 300 | |
| 301 | @param afe Autotest frontend |
| 302 | @param host_id Database host id of the desired host. |
| 303 | @param start_time Start time of the range of interest. |
| 304 | @param end_time End time of the range of interest. |
| 305 | |
| 306 | @return A list of `_TestJobEvent` objects. |
| 307 | |
| 308 | """ |
J. Richard Barnette | 2c9b113 | 2015-07-10 10:47:28 -0700 | [diff] [blame] | 309 | query_start = time_utils.epoch_time_to_date_string(start_time) |
| 310 | query_end = time_utils.epoch_time_to_date_string(end_time) |
J. Richard Barnette | fa27f5b | 2015-03-23 17:11:54 -0700 | [diff] [blame] | 311 | hqelist = afe.get_host_queue_entries( |
| 312 | host_id=host_id, |
J. Richard Barnette | 2c9b113 | 2015-07-10 10:47:28 -0700 | [diff] [blame] | 313 | start_time=query_start, |
| 314 | end_time=query_end, |
J. Richard Barnette | fa27f5b | 2015-03-23 17:11:54 -0700 | [diff] [blame] | 315 | complete=1) |
| 316 | return [cls(afe.server, hqe) for hqe in hqelist] |
| 317 | |
| 318 | |
| 319 | def __init__(self, afe_hostname, hqe): |
| 320 | self._afe_hostname = afe_hostname |
| 321 | self._hqe = hqe |
| 322 | super(_TestJobEvent, self).__init__( |
| 323 | hqe.started_on, hqe.finished_on) |
| 324 | |
| 325 | |
| 326 | @property |
J. Richard Barnette | af1e826 | 2016-03-04 12:55:11 -0800 | [diff] [blame] | 327 | def id(self): |
| 328 | return self._hqe.id |
| 329 | |
| 330 | |
| 331 | @property |
| 332 | def name(self): |
| 333 | return self._hqe.job.name |
| 334 | |
| 335 | |
| 336 | @property |
| 337 | def job_status(self): |
| 338 | return self._hqe.status |
| 339 | |
| 340 | |
| 341 | @property |
J. Richard Barnette | fa27f5b | 2015-03-23 17:11:54 -0700 | [diff] [blame] | 342 | def job_url(self): |
| 343 | logdir = '%s-%s' % (self._hqe.job.id, self._hqe.job.owner) |
| 344 | return _TestJobEvent.get_log_url(self._afe_hostname, logdir) |
| 345 | |
| 346 | |
| 347 | @property |
| 348 | def diagnosis(self): |
J. Richard Barnette | bc9a795 | 2015-04-16 17:43:27 -0700 | [diff] [blame] | 349 | return UNKNOWN |
J. Richard Barnette | fa27f5b | 2015-03-23 17:11:54 -0700 | [diff] [blame] | 350 | |
| 351 | |
| 352 | class HostJobHistory(object): |
J. Richard Barnette | 8abbfd6 | 2015-06-23 12:46:54 -0700 | [diff] [blame] | 353 | """Class to query and remember DUT execution and status history. |
J. Richard Barnette | fa27f5b | 2015-03-23 17:11:54 -0700 | [diff] [blame] | 354 | |
| 355 | This class is responsible for querying the database to determine |
| 356 | the history of a single DUT in a time interval of interest, and |
| 357 | for remembering the query results for reporting. |
| 358 | |
| 359 | @property hostname Host name of the DUT. |
| 360 | @property start_time Start of the requested time interval. |
J. Richard Barnette | 2c9b113 | 2015-07-10 10:47:28 -0700 | [diff] [blame] | 361 | This field may be `None`. |
J. Richard Barnette | fa27f5b | 2015-03-23 17:11:54 -0700 | [diff] [blame] | 362 | @property end_time End of the requested time interval. |
J. Richard Barnette | 8abbfd6 | 2015-06-23 12:46:54 -0700 | [diff] [blame] | 363 | @property _afe Autotest frontend for queries. |
| 364 | @property _host Database host object for the DUT. |
| 365 | @property _history A list of jobs and special tasks that |
J. Richard Barnette | fa27f5b | 2015-03-23 17:11:54 -0700 | [diff] [blame] | 366 | ran on the DUT in the requested time |
| 367 | interval, ordered in reverse, from latest |
| 368 | to earliest. |
| 369 | |
J. Richard Barnette | 8abbfd6 | 2015-06-23 12:46:54 -0700 | [diff] [blame] | 370 | @property _status_interval A list of all the jobs and special |
| 371 | tasks that ran on the DUT in the |
| 372 | last diagnosis interval prior to |
| 373 | `end_time`, ordered from latest to |
| 374 | earliest. |
| 375 | @property _status_diagnosis The DUT's status as of `end_time`. |
| 376 | @property _status_task The DUT's last status task as of |
| 377 | `end_time`. |
| 378 | |
J. Richard Barnette | fa27f5b | 2015-03-23 17:11:54 -0700 | [diff] [blame] | 379 | """ |
| 380 | |
| 381 | @classmethod |
| 382 | def get_host_history(cls, afe, hostname, start_time, end_time): |
J. Richard Barnette | 8abbfd6 | 2015-06-23 12:46:54 -0700 | [diff] [blame] | 383 | """Create a `HostJobHistory` instance for a single host. |
J. Richard Barnette | fa27f5b | 2015-03-23 17:11:54 -0700 | [diff] [blame] | 384 | |
| 385 | Simple factory method to construct host history from a |
| 386 | hostname. Simply looks up the host in the AFE database, and |
| 387 | passes it to the class constructor. |
| 388 | |
| 389 | @param afe Autotest frontend |
| 390 | @param hostname Name of the host. |
| 391 | @param start_time Start time for the history's time |
| 392 | interval. |
| 393 | @param end_time End time for the history's time interval. |
| 394 | |
J. Richard Barnette | 8abbfd6 | 2015-06-23 12:46:54 -0700 | [diff] [blame] | 395 | @return A new `HostJobHistory` instance. |
J. Richard Barnette | fa27f5b | 2015-03-23 17:11:54 -0700 | [diff] [blame] | 396 | |
| 397 | """ |
| 398 | afehost = afe.get_hosts(hostname=hostname)[0] |
| 399 | return cls(afe, afehost, start_time, end_time) |
| 400 | |
| 401 | |
| 402 | @classmethod |
| 403 | def get_multiple_histories(cls, afe, start_time, end_time, |
| 404 | board=None, pool=None): |
J. Richard Barnette | 8abbfd6 | 2015-06-23 12:46:54 -0700 | [diff] [blame] | 405 | """Create `HostJobHistory` instances for a set of hosts. |
J. Richard Barnette | fa27f5b | 2015-03-23 17:11:54 -0700 | [diff] [blame] | 406 | |
| 407 | The set of hosts can be specified as "all hosts of a given |
| 408 | board type", "all hosts in a given pool", or "all hosts |
| 409 | of a given board and pool". |
| 410 | |
| 411 | @param afe Autotest frontend |
| 412 | @param start_time Start time for the history's time |
| 413 | interval. |
| 414 | @param end_time End time for the history's time interval. |
| 415 | @param board All hosts must have this board type; if |
| 416 | `None`, all boards are allowed. |
| 417 | @param pool All hosts must be in this pool; if |
| 418 | `None`, all pools are allowed. |
| 419 | |
J. Richard Barnette | 8abbfd6 | 2015-06-23 12:46:54 -0700 | [diff] [blame] | 420 | @return A list of new `HostJobHistory` instances. |
J. Richard Barnette | fa27f5b | 2015-03-23 17:11:54 -0700 | [diff] [blame] | 421 | |
| 422 | """ |
| 423 | # If `board` or `pool` are both `None`, we could search the |
| 424 | # entire database, which is more expensive than we want. |
| 425 | # Our caller currently won't (can't) do this, but assert to |
| 426 | # be safe. |
| 427 | assert board is not None or pool is not None |
| 428 | labels = [] |
| 429 | if board is not None: |
| 430 | labels.append(constants.Labels.BOARD_PREFIX + board) |
| 431 | if pool is not None: |
| 432 | labels.append(constants.Labels.POOL_PREFIX + pool) |
| 433 | kwargs = {'multiple_labels': labels} |
| 434 | hosts = afe.get_hosts(**kwargs) |
| 435 | return [cls(afe, h, start_time, end_time) for h in hosts] |
| 436 | |
| 437 | |
| 438 | def __init__(self, afe, afehost, start_time, end_time): |
| 439 | self._afe = afe |
| 440 | self.hostname = afehost.hostname |
J. Richard Barnette | 2c9b113 | 2015-07-10 10:47:28 -0700 | [diff] [blame] | 441 | self.end_time = end_time |
| 442 | self.start_time = start_time |
J. Richard Barnette | fa27f5b | 2015-03-23 17:11:54 -0700 | [diff] [blame] | 443 | self._host = afehost |
J. Richard Barnette | bc9a795 | 2015-04-16 17:43:27 -0700 | [diff] [blame] | 444 | # Don't spend time on queries until they're needed. |
J. Richard Barnette | fa27f5b | 2015-03-23 17:11:54 -0700 | [diff] [blame] | 445 | self._history = None |
J. Richard Barnette | 8abbfd6 | 2015-06-23 12:46:54 -0700 | [diff] [blame] | 446 | self._status_interval = None |
J. Richard Barnette | bc9a795 | 2015-04-16 17:43:27 -0700 | [diff] [blame] | 447 | self._status_diagnosis = None |
| 448 | self._status_task = None |
| 449 | |
| 450 | |
J. Richard Barnette | 8abbfd6 | 2015-06-23 12:46:54 -0700 | [diff] [blame] | 451 | def _get_history(self, start_time, end_time): |
| 452 | """Get the list of events for the given interval.""" |
J. Richard Barnette | bc9a795 | 2015-04-16 17:43:27 -0700 | [diff] [blame] | 453 | newtasks = _SpecialTaskEvent.get_tasks( |
| 454 | self._afe, self._host.id, start_time, end_time) |
| 455 | newhqes = _TestJobEvent.get_hqes( |
| 456 | self._afe, self._host.id, start_time, end_time) |
| 457 | newhistory = newtasks + newhqes |
| 458 | newhistory.sort(reverse=True) |
J. Richard Barnette | 8abbfd6 | 2015-06-23 12:46:54 -0700 | [diff] [blame] | 459 | return newhistory |
J. Richard Barnette | fa27f5b | 2015-03-23 17:11:54 -0700 | [diff] [blame] | 460 | |
| 461 | |
| 462 | def __iter__(self): |
J. Richard Barnette | 8abbfd6 | 2015-06-23 12:46:54 -0700 | [diff] [blame] | 463 | if self._history is None: |
| 464 | self._history = self._get_history(self.start_time, |
| 465 | self.end_time) |
J. Richard Barnette | fa27f5b | 2015-03-23 17:11:54 -0700 | [diff] [blame] | 466 | return self._history.__iter__() |
| 467 | |
| 468 | |
J. Richard Barnette | 96db349 | 2015-03-27 17:23:52 -0700 | [diff] [blame] | 469 | def _extract_prefixed_label(self, prefix): |
J. Richard Barnette | 6948ed3 | 2015-05-06 08:57:10 -0700 | [diff] [blame] | 470 | labels = [l for l in self._host.labels |
| 471 | if l.startswith(prefix)] |
| 472 | return labels[0][len(prefix) : ] if labels else None |
J. Richard Barnette | 96db349 | 2015-03-27 17:23:52 -0700 | [diff] [blame] | 473 | |
| 474 | |
J. Richard Barnette | 3d0590a | 2015-04-29 12:56:12 -0700 | [diff] [blame] | 475 | @property |
| 476 | def host(self): |
| 477 | """Return the AFE host object for this history.""" |
| 478 | return self._host |
| 479 | |
| 480 | |
| 481 | @property |
| 482 | def host_board(self): |
J. Richard Barnette | 96db349 | 2015-03-27 17:23:52 -0700 | [diff] [blame] | 483 | """Return the board name for this history's DUT.""" |
| 484 | prefix = constants.Labels.BOARD_PREFIX |
| 485 | return self._extract_prefixed_label(prefix) |
| 486 | |
| 487 | |
J. Richard Barnette | 3d0590a | 2015-04-29 12:56:12 -0700 | [diff] [blame] | 488 | @property |
| 489 | def host_pool(self): |
J. Richard Barnette | 96db349 | 2015-03-27 17:23:52 -0700 | [diff] [blame] | 490 | """Return the pool name for this history's DUT.""" |
| 491 | prefix = constants.Labels.POOL_PREFIX |
| 492 | return self._extract_prefixed_label(prefix) |
| 493 | |
| 494 | |
J. Richard Barnette | 8abbfd6 | 2015-06-23 12:46:54 -0700 | [diff] [blame] | 495 | def _init_status_task(self): |
J. Richard Barnette | bc9a795 | 2015-04-16 17:43:27 -0700 | [diff] [blame] | 496 | """Fill in `self._status_diagnosis` and `_status_task`.""" |
| 497 | if self._status_diagnosis is not None: |
| 498 | return |
J. Richard Barnette | bc9a795 | 2015-04-16 17:43:27 -0700 | [diff] [blame] | 499 | self._status_task = _SpecialTaskEvent.get_status_task( |
J. Richard Barnette | 8abbfd6 | 2015-06-23 12:46:54 -0700 | [diff] [blame] | 500 | self._afe, self._host.id, self.end_time) |
J. Richard Barnette | bc9a795 | 2015-04-16 17:43:27 -0700 | [diff] [blame] | 501 | if self._status_task is not None: |
| 502 | self._status_diagnosis = self._status_task.diagnosis |
| 503 | else: |
| 504 | self._status_diagnosis = UNKNOWN |
| 505 | |
| 506 | |
J. Richard Barnette | 8abbfd6 | 2015-06-23 12:46:54 -0700 | [diff] [blame] | 507 | def _init_status_interval(self): |
| 508 | """Fill in `self._status_interval`.""" |
| 509 | if self._status_interval is not None: |
| 510 | return |
| 511 | self._init_status_task() |
| 512 | self._status_interval = [] |
| 513 | if self._status_task is None: |
| 514 | return |
J. Richard Barnette | 2c9b113 | 2015-07-10 10:47:28 -0700 | [diff] [blame] | 515 | query_end = time_utils.epoch_time_to_date_string(self.end_time) |
J. Richard Barnette | 8abbfd6 | 2015-06-23 12:46:54 -0700 | [diff] [blame] | 516 | interval = self._afe.get_host_diagnosis_interval( |
J. Richard Barnette | 2c9b113 | 2015-07-10 10:47:28 -0700 | [diff] [blame] | 517 | self._host.id, query_end, |
J. Richard Barnette | 8abbfd6 | 2015-06-23 12:46:54 -0700 | [diff] [blame] | 518 | self._status_diagnosis != WORKING) |
| 519 | if not interval: |
| 520 | return |
J. Richard Barnette | 2c9b113 | 2015-07-10 10:47:28 -0700 | [diff] [blame] | 521 | self._status_interval = self._get_history( |
| 522 | parse_time(interval[0]), |
| 523 | parse_time(interval[1])) |
J. Richard Barnette | 8abbfd6 | 2015-06-23 12:46:54 -0700 | [diff] [blame] | 524 | |
| 525 | |
| 526 | def diagnosis_interval(self): |
| 527 | """Find this history's most recent diagnosis interval. |
| 528 | |
| 529 | Returns a list of `_JobEvent` instances corresponding to the |
| 530 | most recent diagnosis interval occurring before this |
| 531 | history's end time. |
| 532 | |
| 533 | The list is returned as with `self._history`, ordered from |
| 534 | most to least recent. |
| 535 | |
| 536 | @return The list of the `_JobEvent`s in the diagnosis |
| 537 | interval. |
| 538 | |
| 539 | """ |
| 540 | self._init_status_interval() |
| 541 | return self._status_interval |
| 542 | |
| 543 | |
J. Richard Barnette | fa27f5b | 2015-03-23 17:11:54 -0700 | [diff] [blame] | 544 | def last_diagnosis(self): |
| 545 | """Return the diagnosis of whether the DUT is working. |
| 546 | |
J. Richard Barnette | 8abbfd6 | 2015-06-23 12:46:54 -0700 | [diff] [blame] | 547 | This searches the DUT's job history, looking for the most |
| 548 | recent status task for the DUT. Return a tuple of |
| 549 | `(diagnosis, task)`. |
J. Richard Barnette | fa27f5b | 2015-03-23 17:11:54 -0700 | [diff] [blame] | 550 | |
| 551 | The `diagnosis` entry in the tuple is one of these values: |
J. Richard Barnette | 2c9b113 | 2015-07-10 10:47:28 -0700 | [diff] [blame] | 552 | * UNUSED - The host's last status task is older than |
| 553 | `self.start_time`. |
J. Richard Barnette | fa27f5b | 2015-03-23 17:11:54 -0700 | [diff] [blame] | 554 | * WORKING - The DUT is working. |
| 555 | * BROKEN - The DUT likely requires manual intervention. |
J. Richard Barnette | bc9a795 | 2015-04-16 17:43:27 -0700 | [diff] [blame] | 556 | * UNKNOWN - No task could be found indicating status for |
| 557 | the DUT. |
J. Richard Barnette | fa27f5b | 2015-03-23 17:11:54 -0700 | [diff] [blame] | 558 | |
J. Richard Barnette | 2c9b113 | 2015-07-10 10:47:28 -0700 | [diff] [blame] | 559 | If the DUT was working at last check, but hasn't been used |
| 560 | inside this history's time interval, the status `UNUSED` is |
| 561 | returned with the last status task, instead of `WORKING`. |
| 562 | |
J. Richard Barnette | 8abbfd6 | 2015-06-23 12:46:54 -0700 | [diff] [blame] | 563 | The `task` entry in the tuple is the status task that led to |
| 564 | the diagnosis. The task will be `None` if the diagnosis is |
J. Richard Barnette | bc9a795 | 2015-04-16 17:43:27 -0700 | [diff] [blame] | 565 | `UNKNOWN`. |
J. Richard Barnette | fa27f5b | 2015-03-23 17:11:54 -0700 | [diff] [blame] | 566 | |
J. Richard Barnette | bc9a795 | 2015-04-16 17:43:27 -0700 | [diff] [blame] | 567 | @return A tuple with the DUT's diagnosis and the task that |
J. Richard Barnette | fa27f5b | 2015-03-23 17:11:54 -0700 | [diff] [blame] | 568 | determined it. |
| 569 | |
| 570 | """ |
J. Richard Barnette | 8abbfd6 | 2015-06-23 12:46:54 -0700 | [diff] [blame] | 571 | self._init_status_task() |
J. Richard Barnette | 2c9b113 | 2015-07-10 10:47:28 -0700 | [diff] [blame] | 572 | diagnosis = self._status_diagnosis |
| 573 | if (self.start_time is not None and |
| 574 | self._status_task is not None and |
| 575 | self._status_task.end_time < self.start_time and |
| 576 | diagnosis == WORKING): |
| 577 | diagnosis = UNUSED |
| 578 | return diagnosis, self._status_task |
J. Richard Barnette | 39255fa | 2015-04-14 17:23:41 -0700 | [diff] [blame] | 579 | |
| 580 | |
J. Richard Barnette | 8abbfd6 | 2015-06-23 12:46:54 -0700 | [diff] [blame] | 581 | def get_diagnosis_interval(host_id, end_time, success): |
| 582 | """Return the last diagnosis interval for a given host and time. |
| 583 | |
| 584 | This routine queries the database for the special tasks on a |
| 585 | given host before a given time. From those tasks it selects the |
| 586 | last status task before a change in status, and the first status |
| 587 | task after the change. When `success` is true, the change must |
| 588 | be from "working" to "broken". When false, the search is for a |
| 589 | change in the opposite direction. |
| 590 | |
| 591 | A "successful status task" is any successful special task. A |
| 592 | "failed status task" is a failed Repair task. These criteria |
| 593 | are based on the definition of "status task" in the module-level |
| 594 | docstring, above. |
| 595 | |
| 596 | This is the RPC endpoint for `AFE.get_host_diagnosis_interval()`. |
| 597 | |
| 598 | @param host_id Database host id of the desired host. |
| 599 | @param end_time Find the last eligible interval before this time. |
| 600 | @param success Whether the eligible interval should start with a |
| 601 | success or a failure. |
| 602 | |
| 603 | @return A list containing the start time of the earliest job |
| 604 | selected, and the end time of the latest job. |
| 605 | |
| 606 | """ |
| 607 | base_query = afe_models.SpecialTask.objects.filter( |
| 608 | host_id=host_id, is_complete=True) |
| 609 | success_query = base_query.filter(success=True) |
| 610 | failure_query = base_query.filter(success=False, task='Repair') |
| 611 | if success: |
| 612 | query0 = success_query |
| 613 | query1 = failure_query |
| 614 | else: |
| 615 | query0 = failure_query |
| 616 | query1 = success_query |
| 617 | query0 = query0.filter(time_finished__lte=end_time) |
| 618 | query0 = query0.order_by('time_started').reverse() |
| 619 | if not query0: |
| 620 | return [] |
| 621 | task0 = query0[0] |
| 622 | query1 = query1.filter(time_finished__gt=task0.time_finished) |
| 623 | task1 = query1.order_by('time_started')[0] |
| 624 | return [task0.time_started.strftime(time_utils.TIME_FMT), |
| 625 | task1.time_finished.strftime(time_utils.TIME_FMT)] |
| 626 | |
| 627 | |
J. Richard Barnette | 39255fa | 2015-04-14 17:23:41 -0700 | [diff] [blame] | 628 | def get_status_task(host_id, end_time): |
J. Richard Barnette | 8abbfd6 | 2015-06-23 12:46:54 -0700 | [diff] [blame] | 629 | """Get the last status task for a host before a given time. |
| 630 | |
| 631 | This routine returns a Django query for the AFE database to find |
| 632 | the last task that finished on the given host before the given |
| 633 | time that was either a successful task, or a Repair task. The |
| 634 | query criteria are based on the definition of "status task" in |
| 635 | the module-level docstring, above. |
J. Richard Barnette | 39255fa | 2015-04-14 17:23:41 -0700 | [diff] [blame] | 636 | |
| 637 | This is the RPC endpoint for `_SpecialTaskEvent.get_status_task()`. |
J. Richard Barnette | 39255fa | 2015-04-14 17:23:41 -0700 | [diff] [blame] | 638 | |
| 639 | @param host_id Database host id of the desired host. |
| 640 | @param end_time End time of the range of interest. |
| 641 | |
| 642 | @return A Django query-set selecting the single special task of |
| 643 | interest. |
| 644 | |
| 645 | """ |
J. Richard Barnette | 8abbfd6 | 2015-06-23 12:46:54 -0700 | [diff] [blame] | 646 | # Selects status tasks: any Repair task, or any successful task. |
| 647 | status_tasks = (django_models.Q(task='Repair') | |
| 648 | django_models.Q(success=True)) |
J. Richard Barnette | 39255fa | 2015-04-14 17:23:41 -0700 | [diff] [blame] | 649 | # Our caller needs a Django query set in order to serialize the |
| 650 | # result, so we don't resolve the query here; we just return a |
| 651 | # slice with at most one element. |
| 652 | return afe_models.SpecialTask.objects.filter( |
J. Richard Barnette | 8abbfd6 | 2015-06-23 12:46:54 -0700 | [diff] [blame] | 653 | status_tasks, |
J. Richard Barnette | 39255fa | 2015-04-14 17:23:41 -0700 | [diff] [blame] | 654 | host_id=host_id, |
| 655 | time_finished__lte=end_time, |
| 656 | is_complete=True).order_by('time_started').reverse()[0:1] |