blob: beafd2308f1934704e28b83a6a2105f9d2eb1177 [file] [log] [blame]
mbligh67647152008-11-19 00:18:14 +00001# Copyright Martin J. Bligh, Google Inc 2008
2# Released under the GPL v2
3
4"""
5This class allows you to communicate with the frontend to submit jobs etc
6It is designed for writing more sophisiticated server-side control files that
7can recursively add and manage other jobs.
8
9We turn the JSON dictionaries into real objects that are more idiomatic
10
mblighc31e4022008-12-11 19:32:30 +000011For docs, see:
jamesren1a2914a2010-02-12 00:44:31 +000012 http://autotest/afe/server/rpc_doc/
13 http://autotest/new_tko/server/rpc_doc/
mblighc31e4022008-12-11 19:32:30 +000014 http://docs.djangoproject.com/en/dev/ref/models/querysets/#queryset-api
mbligh67647152008-11-19 00:18:14 +000015"""
16
mblighdb59e3c2009-11-21 01:45:18 +000017import getpass, os, time, traceback, re
mbligh67647152008-11-19 00:18:14 +000018import common
19from autotest_lib.frontend.afe import rpc_client_lib
mbligh37eceaa2008-12-15 22:56:37 +000020from autotest_lib.client.common_lib import global_config
mbligh67647152008-11-19 00:18:14 +000021from autotest_lib.client.common_lib import utils
mbligh4e576612008-12-22 14:56:36 +000022try:
23 from autotest_lib.server.site_common import site_utils as server_utils
24except:
25 from autotest_lib.server import utils as server_utils
26form_ntuples_from_machines = server_utils.form_ntuples_from_machines
mbligh67647152008-11-19 00:18:14 +000027
mbligh37eceaa2008-12-15 22:56:37 +000028GLOBAL_CONFIG = global_config.global_config
29DEFAULT_SERVER = 'autotest'
30
mbligh67647152008-11-19 00:18:14 +000031def dump_object(header, obj):
32 """
33 Standard way to print out the frontend objects (eg job, host, acl, label)
34 in a human-readable fashion for debugging
35 """
36 result = header + '\n'
37 for key in obj.hash:
38 if key == 'afe' or key == 'hash':
39 continue
40 result += '%20s: %s\n' % (key, obj.hash[key])
41 return result
42
43
mbligh5280e3b2008-12-22 14:39:28 +000044class RpcClient(object):
mbligh67647152008-11-19 00:18:14 +000045 """
mbligh451ede12009-02-12 21:54:03 +000046 Abstract RPC class for communicating with the autotest frontend
47 Inherited for both TKO and AFE uses.
mbligh67647152008-11-19 00:18:14 +000048
mbligh1ef218d2009-08-03 16:57:56 +000049 All the constructors go in the afe / tko class.
mbligh451ede12009-02-12 21:54:03 +000050 Manipulating methods go in the object classes themselves
mbligh67647152008-11-19 00:18:14 +000051 """
mbligh99b24f42009-06-08 16:45:55 +000052 def __init__(self, path, user, server, print_log, debug, reply_debug):
mbligh67647152008-11-19 00:18:14 +000053 """
mbligh451ede12009-02-12 21:54:03 +000054 Create a cached instance of a connection to the frontend
mbligh67647152008-11-19 00:18:14 +000055
56 user: username to connect as
mbligh451ede12009-02-12 21:54:03 +000057 server: frontend server to connect to
mbligh67647152008-11-19 00:18:14 +000058 print_log: pring a logging message to stdout on every operation
59 debug: print out all RPC traffic
60 """
mblighc31e4022008-12-11 19:32:30 +000061 if not user:
mblighdb59e3c2009-11-21 01:45:18 +000062 user = getpass.getuser()
mbligh451ede12009-02-12 21:54:03 +000063 if not server:
mbligh475f7762009-01-30 00:34:04 +000064 if 'AUTOTEST_WEB' in os.environ:
mbligh451ede12009-02-12 21:54:03 +000065 server = os.environ['AUTOTEST_WEB']
mbligh475f7762009-01-30 00:34:04 +000066 else:
mbligh451ede12009-02-12 21:54:03 +000067 server = GLOBAL_CONFIG.get_config_value('SERVER', 'hostname',
68 default=DEFAULT_SERVER)
69 self.server = server
mbligh67647152008-11-19 00:18:14 +000070 self.user = user
71 self.print_log = print_log
72 self.debug = debug
mbligh99b24f42009-06-08 16:45:55 +000073 self.reply_debug = reply_debug
jamesren1a2914a2010-02-12 00:44:31 +000074 http_server = 'http://' + server
75 headers = rpc_client_lib.authorization_headers(user, http_server)
76 rpc_server = http_server + path
mbligh1354c9d2008-12-22 14:56:13 +000077 if debug:
78 print 'SERVER: %s' % rpc_server
79 print 'HEADERS: %s' % headers
mbligh67647152008-11-19 00:18:14 +000080 self.proxy = rpc_client_lib.get_proxy(rpc_server, headers=headers)
81
82
83 def run(self, call, **dargs):
84 """
85 Make a RPC call to the AFE server
86 """
87 rpc_call = getattr(self.proxy, call)
88 if self.debug:
89 print 'DEBUG: %s %s' % (call, dargs)
mbligh451ede12009-02-12 21:54:03 +000090 try:
mbligh99b24f42009-06-08 16:45:55 +000091 result = utils.strip_unicode(rpc_call(**dargs))
92 if self.reply_debug:
93 print result
94 return result
mbligh451ede12009-02-12 21:54:03 +000095 except Exception:
96 print 'FAILED RPC CALL: %s %s' % (call, dargs)
97 raise
mbligh67647152008-11-19 00:18:14 +000098
99
100 def log(self, message):
101 if self.print_log:
102 print message
103
104
mbligh5280e3b2008-12-22 14:39:28 +0000105class TKO(RpcClient):
mbligh99b24f42009-06-08 16:45:55 +0000106 def __init__(self, user=None, server=None, print_log=True, debug=False,
107 reply_debug=False):
jamesren1a2914a2010-02-12 00:44:31 +0000108 super(TKO, self).__init__(path='/new_tko/server/rpc/',
mbligh99b24f42009-06-08 16:45:55 +0000109 user=user,
110 server=server,
111 print_log=print_log,
112 debug=debug,
113 reply_debug=reply_debug)
mblighc31e4022008-12-11 19:32:30 +0000114
115
116 def get_status_counts(self, job, **data):
117 entries = self.run('get_status_counts',
mbligh1ef218d2009-08-03 16:57:56 +0000118 group_by=['hostname', 'test_name', 'reason'],
mblighc31e4022008-12-11 19:32:30 +0000119 job_tag__startswith='%s-' % job, **data)
mbligh5280e3b2008-12-22 14:39:28 +0000120 return [TestStatus(self, e) for e in entries['groups']]
mblighc31e4022008-12-11 19:32:30 +0000121
122
mbligh5280e3b2008-12-22 14:39:28 +0000123class AFE(RpcClient):
mbligh17c75e62009-06-08 16:18:21 +0000124 def __init__(self, user=None, server=None, print_log=True, debug=False,
mbligh99b24f42009-06-08 16:45:55 +0000125 reply_debug=False, job=None):
mbligh17c75e62009-06-08 16:18:21 +0000126 self.job = job
jamesren1a2914a2010-02-12 00:44:31 +0000127 super(AFE, self).__init__(path='/afe/server/rpc/',
mbligh99b24f42009-06-08 16:45:55 +0000128 user=user,
129 server=server,
130 print_log=print_log,
131 debug=debug,
132 reply_debug=reply_debug)
mblighc31e4022008-12-11 19:32:30 +0000133
mbligh1ef218d2009-08-03 16:57:56 +0000134
mbligh67647152008-11-19 00:18:14 +0000135 def host_statuses(self, live=None):
mblighc2847b72009-03-25 19:32:20 +0000136 dead_statuses = ['Dead', 'Repair Failed', 'Repairing']
mbligh67647152008-11-19 00:18:14 +0000137 statuses = self.run('get_static_data')['host_statuses']
138 if live == True:
mblighc2847b72009-03-25 19:32:20 +0000139 return list(set(statuses) - set(dead_statuses))
mbligh67647152008-11-19 00:18:14 +0000140 if live == False:
141 return dead_statuses
142 else:
143 return statuses
144
145
mbligh71094012009-12-19 05:35:21 +0000146 @staticmethod
147 def _dict_for_host_query(hostnames=(), status=None, label=None):
148 query_args = {}
mbligh4e545a52009-12-19 05:30:39 +0000149 if hostnames:
150 query_args['hostname__in'] = hostnames
151 if status:
152 query_args['status'] = status
153 if label:
154 query_args['labels__name'] = label
mbligh71094012009-12-19 05:35:21 +0000155 return query_args
156
157
158 def get_hosts(self, hostnames=(), status=None, label=None, **dargs):
159 query_args = dict(dargs)
160 query_args.update(self._dict_for_host_query(hostnames=hostnames,
161 status=status,
162 label=label))
163 hosts = self.run('get_hosts', **query_args)
164 return [Host(self, h) for h in hosts]
165
166
167 def get_hostnames(self, status=None, label=None, **dargs):
168 """Like get_hosts() but returns hostnames instead of Host objects."""
169 # This implementation can be replaced with a more efficient one
170 # that does not query for entire host objects in the future.
171 return [host_obj.hostname for host_obj in
172 self.get_hosts(status=status, label=label, **dargs)]
173
174
175 def reverify_hosts(self, hostnames=(), status=None, label=None):
176 query_args = dict(locked=False,
177 aclgroup__users__login=self.user)
178 query_args.update(self._dict_for_host_query(hostnames=hostnames,
179 status=status,
180 label=label))
mbligh4e545a52009-12-19 05:30:39 +0000181 return self.run('reverify_hosts', **query_args)
182
183
mbligh67647152008-11-19 00:18:14 +0000184 def create_host(self, hostname, **dargs):
mbligh54459c72009-01-21 19:26:44 +0000185 id = self.run('add_host', hostname=hostname, **dargs)
mbligh67647152008-11-19 00:18:14 +0000186 return self.get_hosts(id=id)[0]
187
188
189 def get_labels(self, **dargs):
190 labels = self.run('get_labels', **dargs)
mbligh5280e3b2008-12-22 14:39:28 +0000191 return [Label(self, l) for l in labels]
mbligh67647152008-11-19 00:18:14 +0000192
193
194 def create_label(self, name, **dargs):
mbligh54459c72009-01-21 19:26:44 +0000195 id = self.run('add_label', name=name, **dargs)
mbligh67647152008-11-19 00:18:14 +0000196 return self.get_labels(id=id)[0]
197
198
199 def get_acls(self, **dargs):
200 acls = self.run('get_acl_groups', **dargs)
mbligh5280e3b2008-12-22 14:39:28 +0000201 return [Acl(self, a) for a in acls]
mbligh67647152008-11-19 00:18:14 +0000202
203
204 def create_acl(self, name, **dargs):
mbligh54459c72009-01-21 19:26:44 +0000205 id = self.run('add_acl_group', name=name, **dargs)
mbligh67647152008-11-19 00:18:14 +0000206 return self.get_acls(id=id)[0]
207
208
mbligh54459c72009-01-21 19:26:44 +0000209 def get_users(self, **dargs):
210 users = self.run('get_users', **dargs)
211 return [User(self, u) for u in users]
212
213
mbligh1354c9d2008-12-22 14:56:13 +0000214 def generate_control_file(self, tests, **dargs):
215 ret = self.run('generate_control_file', tests=tests, **dargs)
216 return ControlFile(self, ret)
217
218
mbligh67647152008-11-19 00:18:14 +0000219 def get_jobs(self, summary=False, **dargs):
220 if summary:
221 jobs_data = self.run('get_jobs_summary', **dargs)
222 else:
223 jobs_data = self.run('get_jobs', **dargs)
mblighafbba0c2009-06-08 16:44:45 +0000224 jobs = []
225 for j in jobs_data:
226 job = Job(self, j)
227 # Set up some extra information defaults
228 job.testname = re.sub('\s.*', '', job.name) # arbitrary default
229 job.platform_results = {}
230 job.platform_reasons = {}
231 jobs.append(job)
232 return jobs
mbligh67647152008-11-19 00:18:14 +0000233
234
235 def get_host_queue_entries(self, **data):
236 entries = self.run('get_host_queue_entries', **data)
mblighf9e35862009-02-26 01:03:11 +0000237 job_statuses = [JobStatus(self, e) for e in entries]
mbligh99b24f42009-06-08 16:45:55 +0000238
239 # Sadly, get_host_queue_entries doesn't return platforms, we have
240 # to get those back from an explicit get_hosts queury, then patch
241 # the new host objects back into the host list.
242 hostnames = [s.host.hostname for s in job_statuses if s.host]
243 host_hash = {}
244 for host in self.get_hosts(hostname__in=hostnames):
245 host_hash[host.hostname] = host
246 for status in job_statuses:
247 if status.host:
248 status.host = host_hash[status.host.hostname]
mblighf9e35862009-02-26 01:03:11 +0000249 # filter job statuses that have either host or meta_host
250 return [status for status in job_statuses if (status.host or
251 status.meta_host)]
mbligh67647152008-11-19 00:18:14 +0000252
253
mblighb9db5162009-04-17 22:21:41 +0000254 def create_job_by_test(self, tests, kernel=None, use_container=False,
mbligh1354c9d2008-12-22 14:56:13 +0000255 **dargs):
mbligh67647152008-11-19 00:18:14 +0000256 """
257 Given a test name, fetch the appropriate control file from the server
mbligh4e576612008-12-22 14:56:36 +0000258 and submit it.
259
260 Returns a list of job objects
mbligh67647152008-11-19 00:18:14 +0000261 """
mblighb9db5162009-04-17 22:21:41 +0000262 assert ('hosts' in dargs or
263 'atomic_group_name' in dargs and 'synch_count' in dargs)
showarda2cd72b2009-10-01 18:43:53 +0000264 if kernel:
265 kernel_list = re.split('[\s,]+', kernel.strip())
266 kernel_info = [{'version': version} for version in kernel_list]
267 else:
268 kernel_info = None
269 control_file = self.generate_control_file(
270 tests=tests, kernel=kernel_info, use_container=use_container,
271 do_push_packages=True)
mbligh1354c9d2008-12-22 14:56:13 +0000272 if control_file.is_server:
mbligh67647152008-11-19 00:18:14 +0000273 dargs['control_type'] = 'Server'
274 else:
275 dargs['control_type'] = 'Client'
276 dargs['dependencies'] = dargs.get('dependencies', []) + \
mbligh1354c9d2008-12-22 14:56:13 +0000277 control_file.dependencies
278 dargs['control_file'] = control_file.control_file
mbligh672666c2009-07-28 23:22:13 +0000279 if not dargs.get('synch_count', None):
mblighc99fccf2009-07-11 00:59:33 +0000280 dargs['synch_count'] = control_file.synch_count
mblighb9db5162009-04-17 22:21:41 +0000281 if 'hosts' in dargs and len(dargs['hosts']) < dargs['synch_count']:
282 # will not be able to satisfy this request
mbligh38b09152009-04-28 18:34:25 +0000283 return None
284 return self.create_job(**dargs)
mbligh67647152008-11-19 00:18:14 +0000285
286
287 def create_job(self, control_file, name=' ', priority='Medium',
288 control_type='Client', **dargs):
289 id = self.run('create_job', name=name, priority=priority,
290 control_file=control_file, control_type=control_type, **dargs)
291 return self.get_jobs(id=id)[0]
292
293
mbligh282ce892010-01-06 18:40:17 +0000294 def run_test_suites(self, pairings, kernel, kernel_label=None,
295 priority='Medium', wait=True, poll_interval=10,
296 email_from=None, email_to=None, timeout=168):
mbligh5b618382008-12-03 15:24:01 +0000297 """
298 Run a list of test suites on a particular kernel.
mbligh1ef218d2009-08-03 16:57:56 +0000299
mbligh5b618382008-12-03 15:24:01 +0000300 Poll for them to complete, and return whether they worked or not.
mbligh1ef218d2009-08-03 16:57:56 +0000301
mbligh282ce892010-01-06 18:40:17 +0000302 @param pairings: List of MachineTestPairing objects to invoke.
303 @param kernel: Name of the kernel to run.
304 @param kernel_label: Label (string) of the kernel to run such as
305 '<kernel-version> : <config> : <date>'
306 If any pairing object has its job_label attribute set it
307 will override this value for that particular job.
308 @param wait: boolean - Wait for the results to come back?
309 @param poll_interval: Interval between polling for job results (in mins)
310 @param email_from: Send notification email upon completion from here.
311 @param email_from: Send notification email upon completion to here.
mbligh5b618382008-12-03 15:24:01 +0000312 """
313 jobs = []
314 for pairing in pairings:
mbligh0c4f8d72009-05-12 20:52:18 +0000315 try:
316 new_job = self.invoke_test(pairing, kernel, kernel_label,
317 priority, timeout=timeout)
318 if not new_job:
319 continue
mbligh0c4f8d72009-05-12 20:52:18 +0000320 jobs.append(new_job)
321 except Exception, e:
322 traceback.print_exc()
mblighb9db5162009-04-17 22:21:41 +0000323 if not wait or not jobs:
mbligh5b618382008-12-03 15:24:01 +0000324 return
mbligh5280e3b2008-12-22 14:39:28 +0000325 tko = TKO()
mbligh5b618382008-12-03 15:24:01 +0000326 while True:
327 time.sleep(60 * poll_interval)
mbligh5280e3b2008-12-22 14:39:28 +0000328 result = self.poll_all_jobs(tko, jobs, email_from, email_to)
mbligh5b618382008-12-03 15:24:01 +0000329 if result is not None:
330 return result
331
332
mbligh45ffc432008-12-09 23:35:17 +0000333 def result_notify(self, job, email_from, email_to):
mbligh5b618382008-12-03 15:24:01 +0000334 """
mbligh45ffc432008-12-09 23:35:17 +0000335 Notify about the result of a job. Will always print, if email data
336 is provided, will send email for it as well.
337
338 job: job object to notify about
339 email_from: send notification email upon completion from here
340 email_from: send notification email upon completion to here
341 """
342 if job.result == True:
343 subject = 'Testing PASSED: '
344 else:
345 subject = 'Testing FAILED: '
346 subject += '%s : %s\n' % (job.name, job.id)
347 text = []
348 for platform in job.results_platform_map:
349 for status in job.results_platform_map[platform]:
350 if status == 'Total':
351 continue
mbligh451ede12009-02-12 21:54:03 +0000352 for host in job.results_platform_map[platform][status]:
353 text.append('%20s %10s %10s' % (platform, status, host))
354 if status == 'Failed':
355 for test_status in job.test_status[host].fail:
356 text.append('(%s, %s) : %s' % \
357 (host, test_status.test_name,
358 test_status.reason))
359 text.append('')
mbligh37eceaa2008-12-15 22:56:37 +0000360
mbligh451ede12009-02-12 21:54:03 +0000361 base_url = 'http://' + self.server
mbligh37eceaa2008-12-15 22:56:37 +0000362
363 params = ('columns=test',
364 'rows=machine_group',
365 "condition=tag~'%s-%%25'" % job.id,
366 'title=Report')
367 query_string = '&'.join(params)
mbligh451ede12009-02-12 21:54:03 +0000368 url = '%s/tko/compose_query.cgi?%s' % (base_url, query_string)
369 text.append(url + '\n')
370 url = '%s/afe/#tab_id=view_job&object_id=%s' % (base_url, job.id)
371 text.append(url + '\n')
mbligh37eceaa2008-12-15 22:56:37 +0000372
373 body = '\n'.join(text)
374 print '---------------------------------------------------'
375 print 'Subject: ', subject
mbligh45ffc432008-12-09 23:35:17 +0000376 print body
mbligh37eceaa2008-12-15 22:56:37 +0000377 print '---------------------------------------------------'
mbligh45ffc432008-12-09 23:35:17 +0000378 if email_from and email_to:
mbligh37eceaa2008-12-15 22:56:37 +0000379 print 'Sending email ...'
mbligh45ffc432008-12-09 23:35:17 +0000380 utils.send_email(email_from, email_to, subject, body)
381 print
mbligh37eceaa2008-12-15 22:56:37 +0000382
mbligh45ffc432008-12-09 23:35:17 +0000383
mbligh1354c9d2008-12-22 14:56:13 +0000384 def print_job_result(self, job):
385 """
386 Print the result of a single job.
387 job: a job object
388 """
389 if job.result is None:
390 print 'PENDING',
391 elif job.result == True:
392 print 'PASSED',
393 elif job.result == False:
394 print 'FAILED',
mbligh912c3f32009-03-25 19:31:30 +0000395 elif job.result == "Abort":
396 print 'ABORT',
mbligh1354c9d2008-12-22 14:56:13 +0000397 print ' %s : %s' % (job.id, job.name)
398
399
mbligh451ede12009-02-12 21:54:03 +0000400 def poll_all_jobs(self, tko, jobs, email_from=None, email_to=None):
mbligh45ffc432008-12-09 23:35:17 +0000401 """
402 Poll all jobs in a list.
403 jobs: list of job objects to poll
404 email_from: send notification email upon completion from here
405 email_from: send notification email upon completion to here
406
407 Returns:
mbligh5b618382008-12-03 15:24:01 +0000408 a) All complete successfully (return True)
409 b) One or more has failed (return False)
410 c) Cannot tell yet (return None)
411 """
mbligh45ffc432008-12-09 23:35:17 +0000412 results = []
mbligh5b618382008-12-03 15:24:01 +0000413 for job in jobs:
mbligh676dcbe2009-06-15 21:57:27 +0000414 if getattr(job, 'result', None) is None:
415 job.result = self.poll_job_results(tko, job)
416 if job.result is not None:
417 self.result_notify(job, email_from, email_to)
mbligh45ffc432008-12-09 23:35:17 +0000418
mbligh676dcbe2009-06-15 21:57:27 +0000419 results.append(job.result)
mbligh1354c9d2008-12-22 14:56:13 +0000420 self.print_job_result(job)
mbligh45ffc432008-12-09 23:35:17 +0000421
422 if None in results:
423 return None
mbligh912c3f32009-03-25 19:31:30 +0000424 elif False in results or "Abort" in results:
mbligh45ffc432008-12-09 23:35:17 +0000425 return False
426 else:
427 return True
mbligh5b618382008-12-03 15:24:01 +0000428
429
mbligh1f23f362008-12-22 14:46:12 +0000430 def _included_platform(self, host, platforms):
431 """
432 See if host's platforms matches any of the patterns in the included
433 platforms list.
434 """
435 if not platforms:
436 return True # No filtering of platforms
437 for platform in platforms:
438 if re.search(platform, host.platform):
439 return True
440 return False
441
442
mbligh7b312282009-01-07 16:45:43 +0000443 def invoke_test(self, pairing, kernel, kernel_label, priority='Medium',
444 **dargs):
mbligh5b618382008-12-03 15:24:01 +0000445 """
446 Given a pairing of a control file to a machine label, find all machines
447 with that label, and submit that control file to them.
mbligh1ef218d2009-08-03 16:57:56 +0000448
mbligh282ce892010-01-06 18:40:17 +0000449 @param kernel_label: Label (string) of the kernel to run such as
450 '<kernel-version> : <config> : <date>'
451 If any pairing object has its job_label attribute set it
452 will override this value for that particular job.
453
454 @returns A list of job objects.
mbligh5b618382008-12-03 15:24:01 +0000455 """
mbligh282ce892010-01-06 18:40:17 +0000456 # The pairing can override the job label.
457 if pairing.job_label:
458 kernel_label = pairing.job_label
mbligh5b618382008-12-03 15:24:01 +0000459 job_name = '%s : %s' % (pairing.machine_label, kernel_label)
460 hosts = self.get_hosts(multiple_labels=[pairing.machine_label])
mbligh1f23f362008-12-22 14:46:12 +0000461 platforms = pairing.platforms
462 hosts = [h for h in hosts if self._included_platform(h, platforms)]
mblighc2847b72009-03-25 19:32:20 +0000463 dead_statuses = self.host_statuses(live=False)
464 host_list = [h.hostname for h in hosts if h.status not in dead_statuses]
mbligh1f23f362008-12-22 14:46:12 +0000465 print 'HOSTS: %s' % host_list
mblighb9db5162009-04-17 22:21:41 +0000466 if pairing.atomic_group_sched:
mblighc99fccf2009-07-11 00:59:33 +0000467 dargs['synch_count'] = pairing.synch_count
mblighb9db5162009-04-17 22:21:41 +0000468 dargs['atomic_group_name'] = pairing.machine_label
469 else:
470 dargs['hosts'] = host_list
mbligh38b09152009-04-28 18:34:25 +0000471 new_job = self.create_job_by_test(name=job_name,
mbligh17c75e62009-06-08 16:18:21 +0000472 dependencies=[pairing.machine_label],
473 tests=[pairing.control_file],
474 priority=priority,
475 kernel=kernel,
476 use_container=pairing.container,
477 **dargs)
mbligh38b09152009-04-28 18:34:25 +0000478 if new_job:
mbligh17c75e62009-06-08 16:18:21 +0000479 if pairing.testname:
480 new_job.testname = pairing.testname
mbligh4e576612008-12-22 14:56:36 +0000481 print 'Invoked test %s : %s' % (new_job.id, job_name)
mbligh38b09152009-04-28 18:34:25 +0000482 return new_job
mbligh5b618382008-12-03 15:24:01 +0000483
484
mblighb9db5162009-04-17 22:21:41 +0000485 def _job_test_results(self, tko, job, debug, tests=[]):
mbligh5b618382008-12-03 15:24:01 +0000486 """
mbligh5280e3b2008-12-22 14:39:28 +0000487 Retrieve test results for a job
mbligh5b618382008-12-03 15:24:01 +0000488 """
mbligh5280e3b2008-12-22 14:39:28 +0000489 job.test_status = {}
490 try:
491 test_statuses = tko.get_status_counts(job=job.id)
492 except Exception:
493 print "Ignoring exception on poll job; RPC interface is flaky"
494 traceback.print_exc()
495 return
496
497 for test_status in test_statuses:
mbligh7479a182009-01-07 16:46:24 +0000498 # SERVER_JOB is buggy, and often gives false failures. Ignore it.
499 if test_status.test_name == 'SERVER_JOB':
500 continue
mblighb9db5162009-04-17 22:21:41 +0000501 # if tests is not empty, restrict list of test_statuses to tests
502 if tests and test_status.test_name not in tests:
503 continue
mbligh451ede12009-02-12 21:54:03 +0000504 if debug:
505 print test_status
mbligh5280e3b2008-12-22 14:39:28 +0000506 hostname = test_status.hostname
507 if hostname not in job.test_status:
508 job.test_status[hostname] = TestResults()
509 job.test_status[hostname].add(test_status)
510
511
mbligh451ede12009-02-12 21:54:03 +0000512 def _job_results_platform_map(self, job, debug):
mblighc9e427e2009-04-28 18:35:06 +0000513 # Figure out which hosts passed / failed / aborted in a job
514 # Creates a 2-dimensional hash, stored as job.results_platform_map
515 # 1st index - platform type (string)
516 # 2nd index - Status (string)
517 # 'Completed' / 'Failed' / 'Aborted'
518 # Data indexed by this hash is a list of hostnames (text strings)
mbligh5280e3b2008-12-22 14:39:28 +0000519 job.results_platform_map = {}
mbligh5b618382008-12-03 15:24:01 +0000520 try:
mbligh45ffc432008-12-09 23:35:17 +0000521 job_statuses = self.get_host_queue_entries(job=job.id)
mbligh5b618382008-12-03 15:24:01 +0000522 except Exception:
523 print "Ignoring exception on poll job; RPC interface is flaky"
524 traceback.print_exc()
525 return None
mbligh5280e3b2008-12-22 14:39:28 +0000526
mbligh5b618382008-12-03 15:24:01 +0000527 platform_map = {}
mbligh5280e3b2008-12-22 14:39:28 +0000528 job.job_status = {}
mbligh451ede12009-02-12 21:54:03 +0000529 job.metahost_index = {}
mbligh5b618382008-12-03 15:24:01 +0000530 for job_status in job_statuses:
mblighc9e427e2009-04-28 18:35:06 +0000531 # This is basically "for each host / metahost in the job"
mbligh451ede12009-02-12 21:54:03 +0000532 if job_status.host:
533 hostname = job_status.host.hostname
534 else: # This is a metahost
535 metahost = job_status.meta_host
536 index = job.metahost_index.get(metahost, 1)
537 job.metahost_index[metahost] = index + 1
538 hostname = '%s.%s' % (metahost, index)
mbligh5280e3b2008-12-22 14:39:28 +0000539 job.job_status[hostname] = job_status.status
mbligh5b618382008-12-03 15:24:01 +0000540 status = job_status.status
mbligh0ecbe632009-05-13 21:34:56 +0000541 # Skip hosts that failed verify or repair:
542 # that's a machine failure, not a job failure
mbligh451ede12009-02-12 21:54:03 +0000543 if hostname in job.test_status:
544 verify_failed = False
545 for failure in job.test_status[hostname].fail:
mbligh0ecbe632009-05-13 21:34:56 +0000546 if (failure.test_name == 'verify' or
547 failure.test_name == 'repair'):
mbligh451ede12009-02-12 21:54:03 +0000548 verify_failed = True
549 break
550 if verify_failed:
551 continue
mblighc9e427e2009-04-28 18:35:06 +0000552 if hostname in job.test_status and job.test_status[hostname].fail:
553 # If the any tests failed in the job, we want to mark the
554 # job result as failed, overriding the default job status.
555 if status != "Aborted": # except if it's an aborted job
556 status = 'Failed'
mbligh451ede12009-02-12 21:54:03 +0000557 if job_status.host:
558 platform = job_status.host.platform
559 else: # This is a metahost
560 platform = job_status.meta_host
mbligh5b618382008-12-03 15:24:01 +0000561 if platform not in platform_map:
562 platform_map[platform] = {'Total' : [hostname]}
563 else:
564 platform_map[platform]['Total'].append(hostname)
565 new_host_list = platform_map[platform].get(status, []) + [hostname]
566 platform_map[platform][status] = new_host_list
mbligh45ffc432008-12-09 23:35:17 +0000567 job.results_platform_map = platform_map
mbligh5280e3b2008-12-22 14:39:28 +0000568
569
mbligh17c75e62009-06-08 16:18:21 +0000570 def set_platform_results(self, test_job, platform, result):
571 """
572 Result must be None, 'FAIL', 'WARN' or 'GOOD'
573 """
574 if test_job.platform_results[platform] is not None:
575 # We're already done, and results recorded. This can't change later.
576 return
577 test_job.platform_results[platform] = result
578 # Note that self.job refers to the metajob we're IN, not the job
579 # that we're excuting from here.
580 testname = '%s.%s' % (test_job.testname, platform)
581 if self.job:
582 self.job.record(result, None, testname, status='')
583
584
mbligh5280e3b2008-12-22 14:39:28 +0000585 def poll_job_results(self, tko, job, debug=False):
586 """
587 Analyse all job results by platform, return:
mbligh1ef218d2009-08-03 16:57:56 +0000588
mbligh5280e3b2008-12-22 14:39:28 +0000589 False: if any platform has more than one failure
590 None: if any platform has more than one machine not yet Good.
591 True: if all platforms have at least all-but-one machines Good.
592 """
mbligh451ede12009-02-12 21:54:03 +0000593 self._job_test_results(tko, job, debug)
mblighe7fcf562009-05-21 01:43:17 +0000594 if job.test_status == {}:
595 return None
mbligh451ede12009-02-12 21:54:03 +0000596 self._job_results_platform_map(job, debug)
mbligh5280e3b2008-12-22 14:39:28 +0000597
mbligh5b618382008-12-03 15:24:01 +0000598 good_platforms = []
mbligh912c3f32009-03-25 19:31:30 +0000599 failed_platforms = []
600 aborted_platforms = []
mbligh5b618382008-12-03 15:24:01 +0000601 unknown_platforms = []
mbligh5280e3b2008-12-22 14:39:28 +0000602 platform_map = job.results_platform_map
mbligh5b618382008-12-03 15:24:01 +0000603 for platform in platform_map:
mbligh17c75e62009-06-08 16:18:21 +0000604 if not job.platform_results.has_key(platform):
605 # record test start, but there's no way to do this right now
606 job.platform_results[platform] = None
mbligh5b618382008-12-03 15:24:01 +0000607 total = len(platform_map[platform]['Total'])
608 completed = len(platform_map[platform].get('Completed', []))
mbligh912c3f32009-03-25 19:31:30 +0000609 failed = len(platform_map[platform].get('Failed', []))
610 aborted = len(platform_map[platform].get('Aborted', []))
mbligh17c75e62009-06-08 16:18:21 +0000611
mbligh1ef218d2009-08-03 16:57:56 +0000612 # We set up what we want to record here, but don't actually do
mbligh17c75e62009-06-08 16:18:21 +0000613 # it yet, until we have a decisive answer for this platform
614 if aborted or failed:
615 bad = aborted + failed
616 if (bad > 1) or (bad * 2 >= total):
617 platform_test_result = 'FAIL'
618 else:
619 platform_test_result = 'WARN'
620
mbligh912c3f32009-03-25 19:31:30 +0000621 if aborted > 1:
622 aborted_platforms.append(platform)
mbligh17c75e62009-06-08 16:18:21 +0000623 self.set_platform_results(job, platform, platform_test_result)
mbligh912c3f32009-03-25 19:31:30 +0000624 elif (failed * 2 >= total) or (failed > 1):
625 failed_platforms.append(platform)
mbligh17c75e62009-06-08 16:18:21 +0000626 self.set_platform_results(job, platform, platform_test_result)
mbligh451ede12009-02-12 21:54:03 +0000627 elif (completed >= 1) and (completed + 1 >= total):
mbligh5b618382008-12-03 15:24:01 +0000628 # if all or all but one are good, call the job good.
629 good_platforms.append(platform)
mbligh17c75e62009-06-08 16:18:21 +0000630 self.set_platform_results(job, platform, 'GOOD')
mbligh5b618382008-12-03 15:24:01 +0000631 else:
632 unknown_platforms.append(platform)
633 detail = []
634 for status in platform_map[platform]:
635 if status == 'Total':
636 continue
637 detail.append('%s=%s' % (status,platform_map[platform][status]))
638 if debug:
mbligh1ef218d2009-08-03 16:57:56 +0000639 print '%20s %d/%d %s' % (platform, completed, total,
mbligh5b618382008-12-03 15:24:01 +0000640 ' '.join(detail))
641 print
mbligh1ef218d2009-08-03 16:57:56 +0000642
mbligh912c3f32009-03-25 19:31:30 +0000643 if len(aborted_platforms) > 0:
mbligh5b618382008-12-03 15:24:01 +0000644 if debug:
mbligh17c75e62009-06-08 16:18:21 +0000645 print 'Result aborted - platforms: ',
646 print ' '.join(aborted_platforms)
mbligh912c3f32009-03-25 19:31:30 +0000647 return "Abort"
648 if len(failed_platforms) > 0:
649 if debug:
650 print 'Result bad - platforms: ' + ' '.join(failed_platforms)
mbligh5b618382008-12-03 15:24:01 +0000651 return False
652 if len(unknown_platforms) > 0:
653 if debug:
654 platform_list = ' '.join(unknown_platforms)
655 print 'Result unknown - platforms: ', platform_list
656 return None
657 if debug:
658 platform_list = ' '.join(good_platforms)
659 print 'Result good - all platforms passed: ', platform_list
660 return True
661
662
mbligh5280e3b2008-12-22 14:39:28 +0000663class TestResults(object):
664 """
665 Container class used to hold the results of the tests for a job
666 """
667 def __init__(self):
668 self.good = []
669 self.fail = []
mbligh451ede12009-02-12 21:54:03 +0000670 self.pending = []
mbligh5280e3b2008-12-22 14:39:28 +0000671
672
673 def add(self, result):
mbligh451ede12009-02-12 21:54:03 +0000674 if result.complete_count > result.pass_count:
675 self.fail.append(result)
676 elif result.incomplete_count > 0:
677 self.pending.append(result)
mbligh5280e3b2008-12-22 14:39:28 +0000678 else:
mbligh451ede12009-02-12 21:54:03 +0000679 self.good.append(result)
mbligh5280e3b2008-12-22 14:39:28 +0000680
681
682class RpcObject(object):
mbligh67647152008-11-19 00:18:14 +0000683 """
684 Generic object used to construct python objects from rpc calls
685 """
686 def __init__(self, afe, hash):
687 self.afe = afe
688 self.hash = hash
689 self.__dict__.update(hash)
690
691
692 def __str__(self):
693 return dump_object(self.__repr__(), self)
694
695
mbligh1354c9d2008-12-22 14:56:13 +0000696class ControlFile(RpcObject):
697 """
698 AFE control file object
699
700 Fields: synch_count, dependencies, control_file, is_server
701 """
702 def __repr__(self):
703 return 'CONTROL FILE: %s' % self.control_file
704
705
mbligh5280e3b2008-12-22 14:39:28 +0000706class Label(RpcObject):
mbligh67647152008-11-19 00:18:14 +0000707 """
708 AFE label object
709
710 Fields:
711 name, invalid, platform, kernel_config, id, only_if_needed
712 """
713 def __repr__(self):
714 return 'LABEL: %s' % self.name
715
716
717 def add_hosts(self, hosts):
718 return self.afe.run('label_add_hosts', self.id, hosts)
719
720
721 def remove_hosts(self, hosts):
722 return self.afe.run('label_remove_hosts', self.id, hosts)
723
724
mbligh5280e3b2008-12-22 14:39:28 +0000725class Acl(RpcObject):
mbligh67647152008-11-19 00:18:14 +0000726 """
727 AFE acl object
728
729 Fields:
730 users, hosts, description, name, id
731 """
732 def __repr__(self):
733 return 'ACL: %s' % self.name
734
735
736 def add_hosts(self, hosts):
737 self.afe.log('Adding hosts %s to ACL %s' % (hosts, self.name))
738 return self.afe.run('acl_group_add_hosts', self.id, hosts)
739
740
741 def remove_hosts(self, hosts):
742 self.afe.log('Removing hosts %s from ACL %s' % (hosts, self.name))
743 return self.afe.run('acl_group_remove_hosts', self.id, hosts)
744
745
mbligh54459c72009-01-21 19:26:44 +0000746 def add_users(self, users):
747 self.afe.log('Adding users %s to ACL %s' % (users, self.name))
748 return self.afe.run('acl_group_add_users', id=self.name, users=users)
749
750
mbligh5280e3b2008-12-22 14:39:28 +0000751class Job(RpcObject):
mbligh67647152008-11-19 00:18:14 +0000752 """
753 AFE job object
754
755 Fields:
756 name, control_file, control_type, synch_count, reboot_before,
757 run_verify, priority, email_list, created_on, dependencies,
758 timeout, owner, reboot_after, id
759 """
760 def __repr__(self):
761 return 'JOB: %s' % self.id
762
763
mbligh5280e3b2008-12-22 14:39:28 +0000764class JobStatus(RpcObject):
mbligh67647152008-11-19 00:18:14 +0000765 """
766 AFE job_status object
767
768 Fields:
769 status, complete, deleted, meta_host, host, active, execution_subdir, id
770 """
771 def __init__(self, afe, hash):
772 # This should call super
773 self.afe = afe
774 self.hash = hash
775 self.__dict__.update(hash)
mbligh5280e3b2008-12-22 14:39:28 +0000776 self.job = Job(afe, self.job)
mbligh67647152008-11-19 00:18:14 +0000777 if self.host:
mbligh99b24f42009-06-08 16:45:55 +0000778 self.host = Host(afe, self.host)
mbligh67647152008-11-19 00:18:14 +0000779
780
781 def __repr__(self):
mbligh451ede12009-02-12 21:54:03 +0000782 if self.host and self.host.hostname:
783 hostname = self.host.hostname
784 else:
785 hostname = 'None'
786 return 'JOB STATUS: %s-%s' % (self.job.id, hostname)
mbligh67647152008-11-19 00:18:14 +0000787
788
mbligh5280e3b2008-12-22 14:39:28 +0000789class Host(RpcObject):
mbligh67647152008-11-19 00:18:14 +0000790 """
791 AFE host object
792
793 Fields:
794 status, lock_time, locked_by, locked, hostname, invalid,
795 synch_id, labels, platform, protection, dirty, id
796 """
797 def __repr__(self):
798 return 'HOST OBJECT: %s' % self.hostname
799
800
801 def show(self):
802 labels = list(set(self.labels) - set([self.platform]))
803 print '%-6s %-7s %-7s %-16s %s' % (self.hostname, self.status,
804 self.locked, self.platform,
805 ', '.join(labels))
806
807
mbligh54459c72009-01-21 19:26:44 +0000808 def delete(self):
809 return self.afe.run('delete_host', id=self.id)
810
811
mbligh6463c4b2009-01-30 00:33:37 +0000812 def modify(self, **dargs):
813 return self.afe.run('modify_host', id=self.id, **dargs)
814
815
mbligh67647152008-11-19 00:18:14 +0000816 def get_acls(self):
817 return self.afe.get_acls(hosts__hostname=self.hostname)
818
819
820 def add_acl(self, acl_name):
821 self.afe.log('Adding ACL %s to host %s' % (acl_name, self.hostname))
822 return self.afe.run('acl_group_add_hosts', id=acl_name,
823 hosts=[self.hostname])
824
825
826 def remove_acl(self, acl_name):
827 self.afe.log('Removing ACL %s from host %s' % (acl_name, self.hostname))
828 return self.afe.run('acl_group_remove_hosts', id=acl_name,
829 hosts=[self.hostname])
830
831
832 def get_labels(self):
833 return self.afe.get_labels(host__hostname__in=[self.hostname])
834
835
836 def add_labels(self, labels):
837 self.afe.log('Adding labels %s to host %s' % (labels, self.hostname))
838 return self.afe.run('host_add_labels', id=self.id, labels=labels)
839
840
841 def remove_labels(self, labels):
842 self.afe.log('Removing labels %s from host %s' % (labels,self.hostname))
843 return self.afe.run('host_remove_labels', id=self.id, labels=labels)
mbligh5b618382008-12-03 15:24:01 +0000844
845
mbligh54459c72009-01-21 19:26:44 +0000846class User(RpcObject):
847 def __repr__(self):
848 return 'USER: %s' % self.login
849
850
mbligh5280e3b2008-12-22 14:39:28 +0000851class TestStatus(RpcObject):
mblighc31e4022008-12-11 19:32:30 +0000852 """
853 TKO test status object
854
855 Fields:
856 test_idx, hostname, testname, id
857 complete_count, incomplete_count, group_count, pass_count
858 """
859 def __repr__(self):
860 return 'TEST STATUS: %s' % self.id
861
862
mbligh5b618382008-12-03 15:24:01 +0000863class MachineTestPairing(object):
864 """
865 Object representing the pairing of a machine label with a control file
mbligh1f23f362008-12-22 14:46:12 +0000866
867 machine_label: use machines from this label
868 control_file: use this control file (by name in the frontend)
869 platforms: list of rexeps to filter platforms by. [] => no filtering
mbligh282ce892010-01-06 18:40:17 +0000870 job_label: The label (name) to give to the autotest job launched
871 to run this pairing. '<kernel-version> : <config> : <date>'
mbligh5b618382008-12-03 15:24:01 +0000872 """
mbligh1354c9d2008-12-22 14:56:13 +0000873 def __init__(self, machine_label, control_file, platforms=[],
mbligh17c75e62009-06-08 16:18:21 +0000874 container=False, atomic_group_sched=False, synch_count=0,
mbligh282ce892010-01-06 18:40:17 +0000875 testname=None, job_label=None):
mbligh5b618382008-12-03 15:24:01 +0000876 self.machine_label = machine_label
877 self.control_file = control_file
mbligh1f23f362008-12-22 14:46:12 +0000878 self.platforms = platforms
mbligh1354c9d2008-12-22 14:56:13 +0000879 self.container = container
mblighb9db5162009-04-17 22:21:41 +0000880 self.atomic_group_sched = atomic_group_sched
881 self.synch_count = synch_count
mbligh17c75e62009-06-08 16:18:21 +0000882 self.testname = testname
mbligh282ce892010-01-06 18:40:17 +0000883 self.job_label = job_label
mbligh1354c9d2008-12-22 14:56:13 +0000884
885
886 def __repr__(self):
887 return '%s %s %s %s' % (self.machine_label, self.control_file,
888 self.platforms, self.container)