blob: 15b4258be9154254bd2c4b1a02d6994c001aae6c [file] [log] [blame]
mblighe8819cd2008-02-15 16:48:40 +00001"""\
2Functions to expose over the RPC interface.
3
4For all modify* and delete* functions that ask for an 'id' parameter to
5identify the object to operate on, the id may be either
6 * the database row ID
7 * the name of the object (label name, hostname, user login, etc.)
8 * a dictionary containing uniquely identifying field (this option should seldom
9 be used)
10
11When specifying foreign key fields (i.e. adding hosts to a label, or adding
12users to an ACL group), the given value may be either the database row ID or the
13name of the object.
14
15All get* functions return lists of dictionaries. Each dictionary represents one
16object and maps field names to values.
17
18Some examples:
19modify_host(2, hostname='myhost') # modify hostname of host with database ID 2
20modify_host('ipaj2', hostname='myhost') # modify hostname of host 'ipaj2'
21modify_test('sleeptest', test_type='Client', params=', seconds=60')
22delete_acl_group(1) # delete by ID
23delete_acl_group('Everyone') # delete by name
24acl_group_add_users('Everyone', ['mbligh', 'showard'])
25get_jobs(owner='showard', status='Queued')
26
27See doctests/rpc_test.txt for (lots) more examples.
28"""
29
30__author__ = 'showard@google.com (Steve Howard)'
31
showardff901382008-07-07 23:22:16 +000032from frontend import thread_local
showard09096d82008-07-07 23:20:49 +000033from frontend.afe import models, model_logic, control_file, rpc_utils
34from frontend.afe import readonly_connection
showard3bb499f2008-07-03 19:42:20 +000035from autotest_lib.client.common_lib import global_config
36
mblighe8819cd2008-02-15 16:48:40 +000037
38# labels
39
40def add_label(name, kernel_config=None, platform=None):
jadmanski0afbb632008-06-06 21:10:57 +000041 return models.Label.add_object(name=name, kernel_config=kernel_config,
42 platform=platform).id
mblighe8819cd2008-02-15 16:48:40 +000043
44
45def modify_label(id, **data):
jadmanski0afbb632008-06-06 21:10:57 +000046 models.Label.smart_get(id).update_object(data)
mblighe8819cd2008-02-15 16:48:40 +000047
48
49def delete_label(id):
jadmanski0afbb632008-06-06 21:10:57 +000050 models.Label.smart_get(id).delete()
mblighe8819cd2008-02-15 16:48:40 +000051
52
showardbbabf502008-06-06 00:02:02 +000053def label_add_hosts(id, hosts):
jadmanski0afbb632008-06-06 21:10:57 +000054 host_objs = [models.Host.smart_get(host) for host in hosts]
55 models.Label.smart_get(id).host_set.add(*host_objs)
showardbbabf502008-06-06 00:02:02 +000056
57
58def label_remove_hosts(id, hosts):
jadmanski0afbb632008-06-06 21:10:57 +000059 host_objs = [models.Host.smart_get(host) for host in hosts]
60 models.Label.smart_get(id).host_set.remove(*host_objs)
showardbbabf502008-06-06 00:02:02 +000061
62
mblighe8819cd2008-02-15 16:48:40 +000063def get_labels(**filter_data):
jadmanski0afbb632008-06-06 21:10:57 +000064 return rpc_utils.prepare_for_serialization(
65 models.Label.list_objects(filter_data))
mblighe8819cd2008-02-15 16:48:40 +000066
67
68# hosts
69
showarddf062562008-07-03 19:56:37 +000070def add_host(hostname, status=None, locked=None, protection=None):
jadmanski0afbb632008-06-06 21:10:57 +000071 return models.Host.add_object(hostname=hostname, status=status,
showarddf062562008-07-03 19:56:37 +000072 locked=locked, protection=protection).id
mblighe8819cd2008-02-15 16:48:40 +000073
74
75def modify_host(id, **data):
jadmanski0afbb632008-06-06 21:10:57 +000076 models.Host.smart_get(id).update_object(data)
mblighe8819cd2008-02-15 16:48:40 +000077
78
79def host_add_labels(id, labels):
jadmanski0afbb632008-06-06 21:10:57 +000080 labels = [models.Label.smart_get(label) for label in labels]
81 models.Host.smart_get(id).labels.add(*labels)
mblighe8819cd2008-02-15 16:48:40 +000082
83
84def host_remove_labels(id, labels):
jadmanski0afbb632008-06-06 21:10:57 +000085 labels = [models.Label.smart_get(label) for label in labels]
86 models.Host.smart_get(id).labels.remove(*labels)
mblighe8819cd2008-02-15 16:48:40 +000087
88
89def delete_host(id):
jadmanski0afbb632008-06-06 21:10:57 +000090 models.Host.smart_get(id).delete()
mblighe8819cd2008-02-15 16:48:40 +000091
92
showard8e3aa5e2008-04-08 19:42:32 +000093def get_hosts(multiple_labels=[], **filter_data):
jadmanski0afbb632008-06-06 21:10:57 +000094 """\
95 multiple_labels: match hosts in all of the labels given. Should be a
96 list of label names.
97 """
98 filter_data['extra_args'] = (
99 rpc_utils.extra_host_filters(multiple_labels))
100 hosts = models.Host.list_objects(filter_data)
101 for host in hosts:
102 host_obj = models.Host.objects.get(id=host['id'])
103 host['labels'] = [label.name
104 for label in host_obj.labels.all()]
105 platform = host_obj.platform()
106 host['platform'] = platform and platform.name or None
107 return rpc_utils.prepare_for_serialization(hosts)
mblighe8819cd2008-02-15 16:48:40 +0000108
109
showard8e3aa5e2008-04-08 19:42:32 +0000110def get_num_hosts(multiple_labels=[], **filter_data):
jadmanski0afbb632008-06-06 21:10:57 +0000111 filter_data['extra_args'] = (
112 rpc_utils.extra_host_filters(multiple_labels))
113 return models.Host.query_count(filter_data)
showard1385b162008-03-13 15:59:40 +0000114
mblighe8819cd2008-02-15 16:48:40 +0000115
116# tests
117
118def add_test(name, test_type, path, test_class=None, description=None):
jadmanski0afbb632008-06-06 21:10:57 +0000119 return models.Test.add_object(name=name, test_type=test_type, path=path,
120 test_class=test_class,
121 description=description).id
mblighe8819cd2008-02-15 16:48:40 +0000122
123
124def modify_test(id, **data):
jadmanski0afbb632008-06-06 21:10:57 +0000125 models.Test.smart_get(id).update_object(data)
mblighe8819cd2008-02-15 16:48:40 +0000126
127
128def delete_test(id):
jadmanski0afbb632008-06-06 21:10:57 +0000129 models.Test.smart_get(id).delete()
mblighe8819cd2008-02-15 16:48:40 +0000130
131
132def get_tests(**filter_data):
jadmanski0afbb632008-06-06 21:10:57 +0000133 return rpc_utils.prepare_for_serialization(
134 models.Test.list_objects(filter_data))
mblighe8819cd2008-02-15 16:48:40 +0000135
136
showard2b9a88b2008-06-13 20:55:03 +0000137# profilers
138
139def add_profiler(name, description=None):
140 return models.Profiler.add_object(name=name, description=description).id
141
142
143def modify_profiler(id, **data):
144 models.Profiler.smart_get(id).update_object(data)
145
146
147def delete_profiler(id):
148 models.Profiler.smart_get(id).delete()
149
150
151def get_profilers(**filter_data):
152 return rpc_utils.prepare_for_serialization(
153 models.Profiler.list_objects(filter_data))
154
155
mblighe8819cd2008-02-15 16:48:40 +0000156# users
157
158def add_user(login, access_level=None):
jadmanski0afbb632008-06-06 21:10:57 +0000159 return models.User.add_object(login=login, access_level=access_level).id
mblighe8819cd2008-02-15 16:48:40 +0000160
161
162def modify_user(id, **data):
jadmanski0afbb632008-06-06 21:10:57 +0000163 models.User.smart_get(id).update_object(data)
mblighe8819cd2008-02-15 16:48:40 +0000164
165
166def delete_user(id):
jadmanski0afbb632008-06-06 21:10:57 +0000167 models.User.smart_get(id).delete()
mblighe8819cd2008-02-15 16:48:40 +0000168
169
170def get_users(**filter_data):
jadmanski0afbb632008-06-06 21:10:57 +0000171 return rpc_utils.prepare_for_serialization(
172 models.User.list_objects(filter_data))
mblighe8819cd2008-02-15 16:48:40 +0000173
174
175# acl groups
176
177def add_acl_group(name, description=None):
jadmanski0afbb632008-06-06 21:10:57 +0000178 return models.AclGroup.add_object(name=name, description=description).id
mblighe8819cd2008-02-15 16:48:40 +0000179
180
181def modify_acl_group(id, **data):
jadmanski0afbb632008-06-06 21:10:57 +0000182 models.AclGroup.smart_get(id).update_object(data)
mblighe8819cd2008-02-15 16:48:40 +0000183
184
185def acl_group_add_users(id, users):
jadmanski0afbb632008-06-06 21:10:57 +0000186 users = [models.User.smart_get(user) for user in users]
187 group = models.AclGroup.smart_get(id)
188 group.users.add(*users)
mblighe8819cd2008-02-15 16:48:40 +0000189
190
191def acl_group_remove_users(id, users):
jadmanski0afbb632008-06-06 21:10:57 +0000192 users = [models.User.smart_get(user) for user in users]
193 group = models.AclGroup.smart_get(id)
194 group.users.remove(*users)
mblighe8819cd2008-02-15 16:48:40 +0000195
196
197def acl_group_add_hosts(id, hosts):
jadmanski0afbb632008-06-06 21:10:57 +0000198 hosts = [models.Host.smart_get(host) for host in hosts]
199 group = models.AclGroup.smart_get(id)
200 group.hosts.add(*hosts)
showard08f981b2008-06-24 21:59:03 +0000201 group.on_host_membership_change()
mblighe8819cd2008-02-15 16:48:40 +0000202
203
204def acl_group_remove_hosts(id, hosts):
jadmanski0afbb632008-06-06 21:10:57 +0000205 hosts = [models.Host.smart_get(host) for host in hosts]
206 group = models.AclGroup.smart_get(id)
207 group.hosts.remove(*hosts)
showard08f981b2008-06-24 21:59:03 +0000208 group.on_host_membership_change()
mblighe8819cd2008-02-15 16:48:40 +0000209
210
211def delete_acl_group(id):
jadmanski0afbb632008-06-06 21:10:57 +0000212 models.AclGroup.smart_get(id).delete()
mblighe8819cd2008-02-15 16:48:40 +0000213
214
215def get_acl_groups(**filter_data):
jadmanski0afbb632008-06-06 21:10:57 +0000216 acl_groups = models.AclGroup.list_objects(filter_data)
217 for acl_group in acl_groups:
218 acl_group_obj = models.AclGroup.objects.get(id=acl_group['id'])
219 acl_group['users'] = [user.login
220 for user in acl_group_obj.users.all()]
221 acl_group['hosts'] = [host.hostname
222 for host in acl_group_obj.hosts.all()]
223 return rpc_utils.prepare_for_serialization(acl_groups)
mblighe8819cd2008-02-15 16:48:40 +0000224
225
226# jobs
227
showard2b9a88b2008-06-13 20:55:03 +0000228def generate_control_file(tests, kernel=None, label=None, profilers=[]):
jadmanski0afbb632008-06-06 21:10:57 +0000229 """\
230 Generates a client-side control file to load a kernel and run a set of
231 tests. Returns a tuple (control_file, is_server, is_synchronous):
232 control_file - the control file text
233 is_server - is the control file a server-side control file?
234 is_synchronous - should the control file be run synchronously?
mblighe8819cd2008-02-15 16:48:40 +0000235
jadmanski0afbb632008-06-06 21:10:57 +0000236 tests: list of tests to run
237 kernel: kernel to install in generated control file
238 label: name of label to grab kernel config from
showard2b9a88b2008-06-13 20:55:03 +0000239 profilers: list of profilers to activate during the job
jadmanski0afbb632008-06-06 21:10:57 +0000240 """
241 if not tests:
242 return '', False, False
mblighe8819cd2008-02-15 16:48:40 +0000243
showard2b9a88b2008-06-13 20:55:03 +0000244 is_server, is_synchronous, test_objects, profiler_objects, label = (
245 rpc_utils.prepare_generate_control_file(tests, kernel, label,
246 profilers))
247 cf_text = control_file.generate_control(tests=test_objects, kernel=kernel,
248 platform=label,
249 profilers=profiler_objects,
250 is_server=is_server)
jadmanski0afbb632008-06-06 21:10:57 +0000251 return cf_text, is_server, is_synchronous
mblighe8819cd2008-02-15 16:48:40 +0000252
253
showard3bb499f2008-07-03 19:42:20 +0000254def create_job(name, priority, control_file, control_type, timeout=None,
showardb8471e32008-07-03 19:51:08 +0000255 is_synchronous=None, hosts=None, meta_hosts=None,
256 one_time_hosts=None):
jadmanski0afbb632008-06-06 21:10:57 +0000257 """\
258 Create and enqueue a job.
mblighe8819cd2008-02-15 16:48:40 +0000259
jadmanski0afbb632008-06-06 21:10:57 +0000260 priority: Low, Medium, High, Urgent
261 control_file: contents of control file
262 control_type: type of control file, Client or Server
263 is_synchronous: boolean indicating if a job is synchronous
264 hosts: list of hosts to run job on
265 meta_hosts: list where each entry is a label name, and for each entry
266 one host will be chosen from that label to run the job
267 on.
showard3bb499f2008-07-03 19:42:20 +0000268 timeout: hours until job times out
jadmanski0afbb632008-06-06 21:10:57 +0000269 """
showard3bb499f2008-07-03 19:42:20 +0000270
271 if timeout is None:
272 timeout=global_config.global_config.get_config_value(
273 'AUTOTEST_WEB', 'job_timeout_default')
274
showardff901382008-07-07 23:22:16 +0000275 owner = thread_local.get_user().login
jadmanski0afbb632008-06-06 21:10:57 +0000276 # input validation
showardb8471e32008-07-03 19:51:08 +0000277 if not hosts and not meta_hosts and not one_time_hosts:
mblighec5546d2008-06-16 16:51:28 +0000278 raise model_logic.ValidationError({
showardb8471e32008-07-03 19:51:08 +0000279 'arguments' : "You must pass at least one of 'hosts', "
280 "'meta_hosts', or 'one_time_hosts'"
jadmanski0afbb632008-06-06 21:10:57 +0000281 })
mblighe8819cd2008-02-15 16:48:40 +0000282
showardba872902008-06-28 00:51:08 +0000283 requested_host_counts = {}
284
jadmanski0afbb632008-06-06 21:10:57 +0000285 # convert hostnames & meta hosts to host/label objects
286 host_objects = []
287 for host in hosts or []:
288 this_host = models.Host.smart_get(host)
289 host_objects.append(this_host)
290 for label in meta_hosts or []:
291 this_label = models.Label.smart_get(label)
292 host_objects.append(this_label)
showardba872902008-06-28 00:51:08 +0000293 requested_host_counts.setdefault(this_label.name, 0)
294 requested_host_counts[this_label.name] += 1
showardb8471e32008-07-03 19:51:08 +0000295 for host in one_time_hosts or []:
296 this_host = models.Host.create_one_time_host(host)
297 host_objects.append(this_host)
showardba872902008-06-28 00:51:08 +0000298
299 # check that each metahost request has enough hosts under the label
300 if meta_hosts:
showard09096d82008-07-07 23:20:49 +0000301 labels = models.Label.objects.filter(
302 name__in=requested_host_counts.keys())
showardba872902008-06-28 00:51:08 +0000303 for label in labels:
304 count = label.host_set.count()
305 if requested_host_counts[label.name] > count:
306 error = ("You have requested %d %s's, but there are only %d."
307 % (requested_host_counts[label.name],
308 label.name, count))
309 raise model_logic.ValidationError({'arguments' : error})
mblighe8819cd2008-02-15 16:48:40 +0000310
jadmanski0afbb632008-06-06 21:10:57 +0000311 # default is_synchronous to some appropriate value
312 ControlType = models.Job.ControlType
313 control_type = ControlType.get_value(control_type)
314 if is_synchronous is None:
315 is_synchronous = (control_type == ControlType.SERVER)
316 # convert the synch flag to an actual type
317 if is_synchronous:
318 synch_type = models.Test.SynchType.SYNCHRONOUS
319 else:
320 synch_type = models.Test.SynchType.ASYNCHRONOUS
mblighe8819cd2008-02-15 16:48:40 +0000321
jadmanski0afbb632008-06-06 21:10:57 +0000322 job = models.Job.create(owner=owner, name=name, priority=priority,
323 control_file=control_file,
324 control_type=control_type,
325 synch_type=synch_type,
showard3bb499f2008-07-03 19:42:20 +0000326 hosts=host_objects,
327 timeout=timeout)
jadmanski0afbb632008-06-06 21:10:57 +0000328 job.queue(host_objects)
329 return job.id
mblighe8819cd2008-02-15 16:48:40 +0000330
331
mbligh3cab4a72008-03-05 23:19:09 +0000332def requeue_job(id):
jadmanski0afbb632008-06-06 21:10:57 +0000333 """\
334 Create and enqueue a copy of the given job.
335 """
336 job = models.Job.objects.get(id=id)
showardff901382008-07-07 23:22:16 +0000337 new_job = job.requeue(thread_local.get_user().login)
jadmanski0afbb632008-06-06 21:10:57 +0000338 return new_job.id
mbligh3cab4a72008-03-05 23:19:09 +0000339
340
mblighe8819cd2008-02-15 16:48:40 +0000341def abort_job(id):
jadmanski0afbb632008-06-06 21:10:57 +0000342 """\
343 Abort the job with the given id number.
344 """
345 job = models.Job.objects.get(id=id)
346 job.abort()
mblighe8819cd2008-02-15 16:48:40 +0000347
348
349def get_jobs(not_yet_run=False, running=False, finished=False, **filter_data):
jadmanski0afbb632008-06-06 21:10:57 +0000350 """\
351 Extra filter args for get_jobs:
352 -not_yet_run: Include only jobs that have not yet started running.
353 -running: Include only jobs that have start running but for which not
354 all hosts have completed.
355 -finished: Include only jobs for which all hosts have completed (or
356 aborted).
357 At most one of these three fields should be specified.
358 """
359 filter_data['extra_args'] = rpc_utils.extra_job_filters(not_yet_run,
360 running,
361 finished)
362 return rpc_utils.prepare_for_serialization(
363 models.Job.list_objects(filter_data))
mblighe8819cd2008-02-15 16:48:40 +0000364
365
366def get_num_jobs(not_yet_run=False, running=False, finished=False,
jadmanski0afbb632008-06-06 21:10:57 +0000367 **filter_data):
368 """\
369 See get_jobs() for documentation of extra filter parameters.
370 """
371 filter_data['extra_args'] = rpc_utils.extra_job_filters(not_yet_run,
372 running,
373 finished)
374 return models.Job.query_count(filter_data)
mblighe8819cd2008-02-15 16:48:40 +0000375
376
mblighe8819cd2008-02-15 16:48:40 +0000377def get_jobs_summary(**filter_data):
jadmanski0afbb632008-06-06 21:10:57 +0000378 """\
showarda8709c52008-07-03 19:44:54 +0000379 Like get_jobs(), but adds a 'status_counts' field, which is a dictionary
jadmanski0afbb632008-06-06 21:10:57 +0000380 mapping status strings to the number of hosts currently with that
381 status, i.e. {'Queued' : 4, 'Running' : 2}.
382 """
383 jobs = get_jobs(**filter_data)
384 ids = [job['id'] for job in jobs]
385 all_status_counts = models.Job.objects.get_status_counts(ids)
386 for job in jobs:
387 job['status_counts'] = all_status_counts[job['id']]
388 return rpc_utils.prepare_for_serialization(jobs)
mblighe8819cd2008-02-15 16:48:40 +0000389
390
showarda8709c52008-07-03 19:44:54 +0000391def get_info_for_clone(id):
392 """\
393 Retrieves all the information needed to clone a job.
394 """
395 info = {}
396 job = models.Job.objects.get(id=id)
397 query = job.hostqueueentry_set.filter(deleted=False)
398 hosts = [queue_entry.host for queue_entry
399 in query if queue_entry.host]
400 meta_hosts = [queue_entry.meta_host.name for queue_entry
401 in query if queue_entry.meta_host]
402 host_info = get_hosts(hostname__in=[host.hostname for host in hosts])
403
404 for host in host_info:
405 platform = host['platform']
406 if platform is not None and platform in host['labels']:
407 host['labels'].remove(platform)
408 host['other_labels'] = ', '.join(host['labels'])
409
410 meta_host_counts = {}
411 for meta_host in meta_hosts:
412 meta_host_counts.setdefault(meta_host, 0)
413 meta_host_counts[meta_host] += 1
414
415 info['job'] = job.get_object_dict()
416 info['meta_host_counts'] = meta_host_counts
417 info['hosts'] = host_info
418
419 return rpc_utils.prepare_for_serialization(info)
420
421
showard34dc5fa2008-04-24 20:58:40 +0000422# host queue entries
423
424def get_host_queue_entries(**filter_data):
jadmanski0afbb632008-06-06 21:10:57 +0000425 """\
426 TODO
427 """
428 query = models.HostQueueEntry.query_objects(filter_data)
429 all_dicts = []
430 for queue_entry in query.select_related():
431 entry_dict = queue_entry.get_object_dict()
432 if entry_dict['host'] is not None:
433 entry_dict['host'] = queue_entry.host.get_object_dict()
434 entry_dict['job'] = queue_entry.job.get_object_dict()
435 all_dicts.append(entry_dict)
436 return rpc_utils.prepare_for_serialization(all_dicts)
showard34dc5fa2008-04-24 20:58:40 +0000437
438
439def get_num_host_queue_entries(**filter_data):
jadmanski0afbb632008-06-06 21:10:57 +0000440 """\
441 Get the number of host queue entries associated with this job.
442 """
443 return models.HostQueueEntry.query_count(filter_data)
showard34dc5fa2008-04-24 20:58:40 +0000444
445
mblighe8819cd2008-02-15 16:48:40 +0000446# other
447
448def get_static_data():
jadmanski0afbb632008-06-06 21:10:57 +0000449 """\
450 Returns a dictionary containing a bunch of data that shouldn't change
451 often and is otherwise inaccessible. This includes:
452 priorities: list of job priority choices
453 default_priority: default priority value for new jobs
454 users: sorted list of all users
455 labels: sorted list of all labels
456 tests: sorted list of all tests
showard2b9a88b2008-06-13 20:55:03 +0000457 profilers: sorted list of all profilers
jadmanski0afbb632008-06-06 21:10:57 +0000458 user_login: logged-in username
459 host_statuses: sorted list of possible Host statuses
460 job_statuses: sorted list of possible HostQueueEntry statuses
461 """
462 result = {}
463 result['priorities'] = models.Job.Priority.choices()
464 default_priority = models.Job.get_field_dict()['priority'].default
465 default_string = models.Job.Priority.get_string(default_priority)
466 result['default_priority'] = default_string
467 result['users'] = get_users(sort_by=['login'])
468 result['labels'] = get_labels(sort_by=['-platform', 'name'])
469 result['tests'] = get_tests(sort_by=['name'])
showard2b9a88b2008-06-13 20:55:03 +0000470 result['profilers'] = get_profilers(sort_by=['name'])
showardff901382008-07-07 23:22:16 +0000471 result['user_login'] = thread_local.get_user().login
showard2b9a88b2008-06-13 20:55:03 +0000472 result['host_statuses'] = sorted(models.Host.Status.names)
473 result['job_statuses'] = sorted(models.Job.Status.names)
showard3bb499f2008-07-03 19:42:20 +0000474 result['job_timeout_default'] = global_config.global_config.get_config_value(
475 'AUTOTEST_WEB', 'job_timeout_default')
jadmanski0afbb632008-06-06 21:10:57 +0000476 return result