blob: 5a9e7d13c0018e35e53c1f16dca175120071aba4 [file] [log] [blame]
Dan Shi193905e2014-07-25 23:33:09 -07001#! /usr/bin/python
2# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6# This module provides functions for caller to retrieve a job's history,
7# including special tasks executed before and after the job, and each steps
8# start/end time.
9
10import argparse
11import datetime as datetime_base
12
13import common
14from autotest_lib.client.common_lib import global_config
15from autotest_lib.frontend import setup_django_environment
16from autotest_lib.frontend.afe import models
17from autotest_lib.frontend.tko import models as tko_models
18
19CONFIG = global_config.global_config
20AUTOTEST_SERVER = CONFIG.get_config_value('SERVER', 'hostname', type=str)
21
22LOG_BASE_URL = 'http://%s/tko/retrieve_logs.cgi?job=/results/' % AUTOTEST_SERVER
23JOB_URL = LOG_BASE_URL + '%(job_id)s-%(owner)s/%(hostname)s'
Dan Shidfea3682014-08-10 23:38:40 -070024LOG_PATH_FMT = 'hosts/%(hostname)s/%(task_id)d-%(task_name)s'
Dan Shi193905e2014-07-25 23:33:09 -070025TASK_URL = LOG_BASE_URL + LOG_PATH_FMT
26AUTOSERV_DEBUG_LOG = 'debug/autoserv.DEBUG'
27
28# Add some buffer before and after job start/end time when searching for special
29# tasks. This is to guarantee to include reset before the job starts and repair
30# and cleanup after the job finishes.
31TIME_BUFFER = datetime_base.timedelta(hours=2)
32
33
34class JobHistoryObject(object):
35 """A common interface to call get_history to return a dictionary of the
36 object's history record, e.g., start/end time.
37 """
38
39 def build_history_entry(self):
40 """Build a history entry.
41
42 This function expect the object has required attributes. Any missing
43 attributes will lead to failure.
44
45 @return: A dictionary as the history entry of given job/task.
46 """
47 return {'id': self.id,
48 'name': self.name,
49 'hostname': self.hostname,
50 'status': self.status,
51 'log_url': self.log_url,
52 'autoserv_log_url': self.autoserv_log_url,
53 'start_time': self.start_time,
54 'end_time': self.end_time,
55 'time_used': self.time_used,
56 }
57
58
59 def get_history(self):
60 """Return a list of dictionaries of select job/task's history.
61 """
62 raise NotImplementedError('You must override this method in child '
63 'class.')
64
65
66class SpecialTaskInfo(JobHistoryObject):
67 """Information of a special task.
68
69 Its properties include:
70 id: Special task ID.
71 task: An AFE models.SpecialTask object.
72 hostname: hostname of the DUT that runs the special task.
73 log_url: Url to debug log.
74 autoserv_log_url: Url to the autoserv log.
75 """
76
77 def __init__(self, task):
78 """Constructor
79
80 @param task: An AFE models.SpecialTask object, which has the information
81 of the special task from database.
82 """
83 # Special task ID
84 self.id = task.id
85 # AFE special_task model
86 self.task = task
87 self.name = task.task
88 self.hostname = task.host.hostname
89 self.status = task.status
90
91 # Link to log
Dan Shidfea3682014-08-10 23:38:40 -070092 task_info = {'task_id': task.id, 'task_name': task.task.lower(),
Dan Shi193905e2014-07-25 23:33:09 -070093 'hostname': self.hostname}
94 self.log_url = TASK_URL % task_info
95 self.autoserv_log_url = '%s/%s' % (self.log_url, AUTOSERV_DEBUG_LOG)
96
97 self.start_time = self.task.time_started
98 self.end_time = self.task.time_finished
99 if self.start_time and self.end_time:
100 self.time_used = (self.end_time - self.start_time).total_seconds()
101 else:
102 self.time_used = None
103
104
105 def __str__(self):
106 """Get a formatted string of the details of the task info.
107 """
108 return ('Task %d: %s from %s to %s, for %s seconds.\n' %
109 (self.id, self.task.task, self.start_time, self.end_time,
110 self.time_used))
111
112
113 def get_history(self):
114 """Return a dictionary of selected object properties.
115 """
116 return [self.build_history_entry()]
117
118
119class TaskCacheCollection(dict):
120 """A cache to hold tasks for multiple hosts.
121
122 It's a dictionary of host_id: TaskCache.
123 """
124
125 def try_get(self, host_id, job_id, start_time, end_time):
126 """Try to get tasks from cache.
127
128 @param host_id: ID of the host.
129 @param job_id: ID of the test job that's related to the special task.
130 @param start_time: Start time to search for special task.
131 @param end_time: End time to search for special task.
132 @return: The list of special tasks that are related to given host and
133 Job id. Note that, None means the cache is not available.
134 However, [] means no special tasks found in cache.
135 """
136 if not host_id in self:
137 return None
138 return self[host_id].try_get(job_id, start_time, end_time)
139
140
141 def update(self, host_id, start_time, end_time):
142 """Update the cache of the given host by searching database.
143
144 @param host_id: ID of the host.
145 @param start_time: Start time to search for special task.
146 @param end_time: End time to search for special task.
147 """
148 search_start_time = start_time - TIME_BUFFER
149 search_end_time = end_time + TIME_BUFFER
150 tasks = models.SpecialTask.objects.filter(
151 host_id=host_id,
152 time_started__gte=search_start_time,
153 time_started__lte=search_end_time)
154 self[host_id] = TaskCache(tasks, search_start_time, search_end_time)
155
156
157class TaskCache(object):
158 """A cache that hold tasks for a host.
159 """
160
161 def __init__(self, tasks=[], start_time=None, end_time=None):
162 """Constructor
163 """
164 self.tasks = tasks
165 self.start_time = start_time
166 self.end_time = end_time
167
168 def try_get(self, job_id, start_time, end_time):
169 """Try to get tasks from cache.
170
171 @param job_id: ID of the test job that's related to the special task.
172 @param start_time: Start time to search for special task.
173 @param end_time: End time to search for special task.
174 @return: The list of special tasks that are related to the job id.
175 Note that, None means the cache is not available.
176 However, [] means no special tasks found in cache.
177 """
178 if start_time < self.start_time or end_time > self.end_time:
179 return None
180 return [task for task in self.tasks if task.queue_entry and
181 task.queue_entry.job.id == job_id]
182
183
184class TestJobInfo(JobHistoryObject):
185 """Information of a test job
186 """
187
188 def __init__(self, hqe, task_caches=None, suite_start_time=None,
189 suite_end_time=None):
190 """Constructor
191
192 @param hqe: HostQueueEntry of the job.
193 @param task_caches: Special tasks that's from a previous query.
194 @param suite_start_time: Start time of the suite job, default is
195 None. Used to build special task search cache.
196 @param suite_end_time: End time of the suite job, default is
197 None. Used to build special task search cache.
198 """
199 # AFE job ID
200 self.id = hqe.job.id
201 # AFE job model
202 self.job = hqe.job
203 # Name of the job, strip all build and suite info.
204 self.name = hqe.job.name.split('/')[-1]
205 self.status = hqe.status if hqe else None
206
207 try:
208 self.tko_job = tko_models.Job.objects.filter(afe_job_id=self.id)[0]
209 self.host = models.Host.objects.filter(
210 hostname=self.tko_job.machine.hostname)[0]
211 self.hostname = self.tko_job.machine.hostname
212 self.start_time = self.tko_job.started_time
213 self.end_time = self.tko_job.finished_time
214 except IndexError:
215 # The test job was never started.
216 self.tko_job = None
217 self.host = None
218 self.hostname = None
219 self.start_time = None
220 self.end_time = None
221
222 if self.end_time and self.start_time:
223 self.time_used = (self.end_time - self.start_time).total_seconds()
224 else:
225 self.time_used = None
226
227 # Link to log
228 self.log_url = JOB_URL % {'job_id': hqe.job.id, 'owner': hqe.job.owner,
229 'hostname': self.hostname}
230 self.autoserv_log_url = '%s/%s' % (self.log_url, AUTOSERV_DEBUG_LOG)
231
232 self._get_special_tasks(hqe, task_caches, suite_start_time,
233 suite_end_time)
234
235
236 def _get_special_tasks(self, hqe, task_caches=None, suite_start_time=None,
237 suite_end_time=None):
238 """Get special tasks ran before and after the test job.
239
240 @param hqe: HostQueueEntry of the job.
241 @param task_caches: Special tasks that's from a previous query.
242 @param suite_start_time: Start time of the suite job, default is
243 None. Used to build special task search cache.
244 @param suite_end_time: End time of the suite job, default is
245 None. Used to build special task search cache.
246 """
247 # Special tasks run before job starts.
248 self.tasks_before = []
249 # Special tasks run after job finished.
250 self.tasks_after = []
251
252 # Skip locating special tasks if hqe is None, or not started yet, as
253 # that indicates the test job might not be started.
254 if not hqe or not hqe.started_on:
255 return
256
257 # Assume special tasks for the test job all start within 2 hours
258 # before the test job starts or 2 hours after the test finishes. In most
259 # cases, special task won't take longer than 2 hours to start before
260 # test job starts and after test job finishes.
261 search_start_time = hqe.started_on - TIME_BUFFER
262 search_end_time = (hqe.finished_on + TIME_BUFFER if hqe.finished_on else
263 hqe.started_on + TIME_BUFFER)
264
265 if task_caches is not None and suite_start_time and suite_end_time:
266 tasks = task_caches.try_get(self.host.id, self.id,
267 suite_start_time, suite_end_time)
268 if tasks is None:
269 task_caches.update(self.host.id, search_start_time,
270 search_end_time)
271 tasks = task_caches.try_get(self.host.id, self.id,
272 suite_start_time, suite_end_time)
273 else:
274 tasks = models.SpecialTask.objects.filter(
275 host_id=self.host.id,
276 time_started__gte=search_start_time,
277 time_started__lte=search_end_time)
278 tasks = [task for task in tasks if task.queue_entry and
279 task.queue_entry.job.id == self.id]
280
281 for task in tasks:
282 task_info = SpecialTaskInfo(task)
283 if task.time_started < self.start_time:
284 self.tasks_before.append(task_info)
285 else:
286 self.tasks_after.append(task_info)
287
288
289 def get_history(self):
290 """Get the history of a test job.
291
292 @return: A list of special tasks and test job information.
293 """
294 history = []
295 history.extend([task.build_history_entry() for task in
296 self.tasks_before])
297 history.append(self.build_history_entry())
298 history.extend([task.build_history_entry() for task in
299 self.tasks_after])
300 return history
301
302
303 def __str__(self):
304 """Get a formatted string of the details of the job info.
305 """
306 result = '%d: %s\n' % (self.id, self.name)
307 for task in self.tasks_before:
308 result += str(task)
309
310 result += ('Test from %s to %s, for %s seconds.\n' %
311 (self.start_time, self.end_time, self.time_used))
312
313 for task in self.tasks_after:
314 result += str(task)
315
316 return result
317
318
319class SuiteJobInfo(JobHistoryObject):
320 """Information of a suite job
321 """
322
323 def __init__(self, hqe):
324 """Constructor
325
326 @param hqe: HostQueueEntry of the job.
327 """
328 # AFE job ID
329 self.id = hqe.job.id
330 # AFE job model
331 self.job = hqe.job
332 # Name of the job, strip all build and suite info.
333 self.name = hqe.job.name.split('/')[-1]
334 self.status = hqe.status if hqe else None
335
336 self.log_url = JOB_URL % {'job_id': hqe.job.id, 'owner': hqe.job.owner,
337 'hostname': 'hostless'}
338
339 hqe = models.HostQueueEntry.objects.filter(job_id=hqe.job.id)[0]
340 self.start_time = hqe.started_on
341 self.end_time = hqe.finished_on
342 if self.start_time and self.end_time:
343 self.time_used = (self.end_time - self.start_time).total_seconds()
344 else:
345 self.time_used = None
346
347 # Cache of special tasks, hostname: ((start_time, end_time), [tasks])
348 task_caches = TaskCacheCollection()
349 self.test_jobs = []
350 for job in models.Job.objects.filter(parent_job_id=self.id):
351 try:
352 job_hqe = models.HostQueueEntry.objects.filter(job_id=job.id)[0]
353 except IndexError:
354 continue
355 self.test_jobs.append(TestJobInfo(job_hqe, task_caches,
356 self.start_time, self.end_time))
357
358
359 def get_history(self):
360 """Get the history of a suite job.
361
362 @return: A list of special tasks and test job information that has
363 suite job as the parent job.
364 """
365 history = []
366 for job in sorted(self.test_jobs,
367 key=lambda j: (j.hostname, j.start_time)):
368 history.extend(job.get_history())
369 return history
370
371
372 def __str__(self):
373 """Get a formatted string of the details of the job info.
374 """
375 result = '%d: %s\n' % (self.id, self.name)
376 for job in self.test_jobs:
377 result += str(job)
378 result += '-' * 80 + '\n'
379 return result
380
381
382def get_job_info(job_id):
383 """Get the history of a job.
384
385 @param job_id: ID of the job.
386 @return: A TestJobInfo object that contains the test job and its special
387 tasks' start/end time, if the job is a test job. Otherwise, return
388 a SuiteJobInfo object if the job is a suite job.
389 @raise Exception: if the test job can't be found in database.
390 """
391 try:
392 hqe = models.HostQueueEntry.objects.filter(job_id=job_id)[0]
393 except IndexError:
394 raise Exception('No HQE found for job ID %d' % job_id)
395
396 if hqe and hqe.execution_subdir != 'hostless':
397 return TestJobInfo(hqe)
398 else:
399 return SuiteJobInfo(hqe)
400
401
402def main():
403 """Main script.
404
405 The script accepts a job ID and print out the test job and its special
406 tasks' start/end time.
407 """
408 parser = argparse.ArgumentParser()
409 parser.add_argument('--job_id', type=int, dest='job_id', required=True)
410 options = parser.parse_args()
411
412 job_info = get_job_info(options.job_id)
413
414 print job_info
415
416
417if __name__ == '__main__':
418 main()