blob: abc9786400a82d494ceaa241ddaa1233a2aa201e [file] [log] [blame]
mblighe8819cd2008-02-15 16:48:40 +00001"""\
2Functions to expose over the RPC interface.
3
4For all modify* and delete* functions that ask for an 'id' parameter to
5identify the object to operate on, the id may be either
6 * the database row ID
7 * the name of the object (label name, hostname, user login, etc.)
8 * a dictionary containing uniquely identifying field (this option should seldom
9 be used)
10
11When specifying foreign key fields (i.e. adding hosts to a label, or adding
12users to an ACL group), the given value may be either the database row ID or the
13name of the object.
14
15All get* functions return lists of dictionaries. Each dictionary represents one
16object and maps field names to values.
17
18Some examples:
19modify_host(2, hostname='myhost') # modify hostname of host with database ID 2
20modify_host('ipaj2', hostname='myhost') # modify hostname of host 'ipaj2'
21modify_test('sleeptest', test_type='Client', params=', seconds=60')
22delete_acl_group(1) # delete by ID
23delete_acl_group('Everyone') # delete by name
24acl_group_add_users('Everyone', ['mbligh', 'showard'])
25get_jobs(owner='showard', status='Queued')
26
mbligh93c80e62009-02-03 17:48:30 +000027See doctests/001_rpc_test.txt for (lots) more examples.
mblighe8819cd2008-02-15 16:48:40 +000028"""
29
30__author__ = 'showard@google.com (Steve Howard)'
31
showardff901382008-07-07 23:22:16 +000032from frontend import thread_local
showard09096d82008-07-07 23:20:49 +000033from frontend.afe import models, model_logic, control_file, rpc_utils
showard3bb499f2008-07-03 19:42:20 +000034from autotest_lib.client.common_lib import global_config
35
mblighe8819cd2008-02-15 16:48:40 +000036
37# labels
38
showard989f25d2008-10-01 11:38:11 +000039def add_label(name, kernel_config=None, platform=None, only_if_needed=None):
showardc92da832009-04-07 18:14:34 +000040 return models.Label.add_object(
41 name=name, kernel_config=kernel_config, platform=platform,
42 only_if_needed=only_if_needed).id
mblighe8819cd2008-02-15 16:48:40 +000043
44
45def modify_label(id, **data):
jadmanski0afbb632008-06-06 21:10:57 +000046 models.Label.smart_get(id).update_object(data)
mblighe8819cd2008-02-15 16:48:40 +000047
48
49def delete_label(id):
jadmanski0afbb632008-06-06 21:10:57 +000050 models.Label.smart_get(id).delete()
mblighe8819cd2008-02-15 16:48:40 +000051
52
showardbbabf502008-06-06 00:02:02 +000053def label_add_hosts(id, hosts):
showardbe3ec042008-11-12 18:16:07 +000054 host_objs = models.Host.smart_get_bulk(hosts)
jadmanski0afbb632008-06-06 21:10:57 +000055 models.Label.smart_get(id).host_set.add(*host_objs)
showardbbabf502008-06-06 00:02:02 +000056
57
58def label_remove_hosts(id, hosts):
showardbe3ec042008-11-12 18:16:07 +000059 host_objs = models.Host.smart_get_bulk(hosts)
jadmanski0afbb632008-06-06 21:10:57 +000060 models.Label.smart_get(id).host_set.remove(*host_objs)
showardbbabf502008-06-06 00:02:02 +000061
62
mblighe8819cd2008-02-15 16:48:40 +000063def get_labels(**filter_data):
showardc92da832009-04-07 18:14:34 +000064 """\
65 @returns A sequence of nested dictionaries of label information.
66 """
67 return rpc_utils.prepare_rows_as_nested_dicts(
68 models.Label.query_objects(filter_data),
69 ('atomic_group',))
70
71
72# atomic groups
73
74def add_atomic_group(name, max_number_of_machines, description=None):
75 return models.AtomicGroup.add_object(
76 name=name, max_number_of_machines=max_number_of_machines,
77 description=description).id
78
79
80def modify_atomic_group(id, **data):
81 models.AtomicGroup.smart_get(id).update_object(data)
82
83
84def delete_atomic_group(id):
85 models.AtomicGroup.smart_get(id).delete()
86
87
88def atomic_group_add_labels(id, labels):
89 label_objs = models.Label.smart_get_bulk(labels)
90 models.AtomicGroup.smart_get(id).label_set.add(*label_objs)
91
92
93def atomic_group_remove_labels(id, labels):
94 label_objs = models.Label.smart_get_bulk(labels)
95 models.AtomicGroup.smart_get(id).label_set.remove(*label_objs)
96
97
98def get_atomic_groups(**filter_data):
jadmanski0afbb632008-06-06 21:10:57 +000099 return rpc_utils.prepare_for_serialization(
showardc92da832009-04-07 18:14:34 +0000100 models.AtomicGroup.list_objects(filter_data))
mblighe8819cd2008-02-15 16:48:40 +0000101
102
103# hosts
104
showarddf062562008-07-03 19:56:37 +0000105def add_host(hostname, status=None, locked=None, protection=None):
jadmanski0afbb632008-06-06 21:10:57 +0000106 return models.Host.add_object(hostname=hostname, status=status,
showarddf062562008-07-03 19:56:37 +0000107 locked=locked, protection=protection).id
mblighe8819cd2008-02-15 16:48:40 +0000108
109
110def modify_host(id, **data):
jadmanski0afbb632008-06-06 21:10:57 +0000111 models.Host.smart_get(id).update_object(data)
mblighe8819cd2008-02-15 16:48:40 +0000112
113
114def host_add_labels(id, labels):
showardbe3ec042008-11-12 18:16:07 +0000115 labels = models.Label.smart_get_bulk(labels)
jadmanski0afbb632008-06-06 21:10:57 +0000116 models.Host.smart_get(id).labels.add(*labels)
mblighe8819cd2008-02-15 16:48:40 +0000117
118
119def host_remove_labels(id, labels):
showardbe3ec042008-11-12 18:16:07 +0000120 labels = models.Label.smart_get_bulk(labels)
jadmanski0afbb632008-06-06 21:10:57 +0000121 models.Host.smart_get(id).labels.remove(*labels)
mblighe8819cd2008-02-15 16:48:40 +0000122
123
124def delete_host(id):
jadmanski0afbb632008-06-06 21:10:57 +0000125 models.Host.smart_get(id).delete()
mblighe8819cd2008-02-15 16:48:40 +0000126
127
showard43a3d262008-11-12 18:17:05 +0000128def get_hosts(multiple_labels=[], exclude_only_if_needed_labels=False,
129 **filter_data):
jadmanski0afbb632008-06-06 21:10:57 +0000130 """\
131 multiple_labels: match hosts in all of the labels given. Should be a
132 list of label names.
showard43a3d262008-11-12 18:17:05 +0000133 exclude_only_if_needed_labels: exclude hosts with at least one
134 "only_if_needed" label applied.
jadmanski0afbb632008-06-06 21:10:57 +0000135 """
showard43a3d262008-11-12 18:17:05 +0000136 hosts = rpc_utils.get_host_query(multiple_labels,
137 exclude_only_if_needed_labels,
138 filter_data)
139 host_dicts = []
140 for host_obj in hosts:
141 host_dict = host_obj.get_object_dict()
142 host_dict['labels'] = [label.name for label in host_obj.labels.all()]
jadmanski0afbb632008-06-06 21:10:57 +0000143 platform = host_obj.platform()
showard43a3d262008-11-12 18:17:05 +0000144 host_dict['platform'] = platform and platform.name or None
145 host_dicts.append(host_dict)
146 return rpc_utils.prepare_for_serialization(host_dicts)
mblighe8819cd2008-02-15 16:48:40 +0000147
148
showard43a3d262008-11-12 18:17:05 +0000149def get_num_hosts(multiple_labels=[], exclude_only_if_needed_labels=False,
150 **filter_data):
151 hosts = rpc_utils.get_host_query(multiple_labels,
152 exclude_only_if_needed_labels,
153 filter_data)
154 return hosts.count()
showard1385b162008-03-13 15:59:40 +0000155
mblighe8819cd2008-02-15 16:48:40 +0000156
157# tests
158
showard909c7a62008-07-15 21:52:38 +0000159def add_test(name, test_type, path, author=None, dependencies=None,
showard3d9899a2008-07-31 02:11:58 +0000160 experimental=True, run_verify=None, test_class=None,
showard909c7a62008-07-15 21:52:38 +0000161 test_time=None, test_category=None, description=None,
162 sync_count=1):
jadmanski0afbb632008-06-06 21:10:57 +0000163 return models.Test.add_object(name=name, test_type=test_type, path=path,
showard909c7a62008-07-15 21:52:38 +0000164 author=author, dependencies=dependencies,
165 experimental=experimental,
166 run_verify=run_verify, test_time=test_time,
167 test_category=test_category,
168 sync_count=sync_count,
jadmanski0afbb632008-06-06 21:10:57 +0000169 test_class=test_class,
170 description=description).id
mblighe8819cd2008-02-15 16:48:40 +0000171
172
173def modify_test(id, **data):
jadmanski0afbb632008-06-06 21:10:57 +0000174 models.Test.smart_get(id).update_object(data)
mblighe8819cd2008-02-15 16:48:40 +0000175
176
177def delete_test(id):
jadmanski0afbb632008-06-06 21:10:57 +0000178 models.Test.smart_get(id).delete()
mblighe8819cd2008-02-15 16:48:40 +0000179
180
181def get_tests(**filter_data):
jadmanski0afbb632008-06-06 21:10:57 +0000182 return rpc_utils.prepare_for_serialization(
183 models.Test.list_objects(filter_data))
mblighe8819cd2008-02-15 16:48:40 +0000184
185
showard2b9a88b2008-06-13 20:55:03 +0000186# profilers
187
188def add_profiler(name, description=None):
189 return models.Profiler.add_object(name=name, description=description).id
190
191
192def modify_profiler(id, **data):
193 models.Profiler.smart_get(id).update_object(data)
194
195
196def delete_profiler(id):
197 models.Profiler.smart_get(id).delete()
198
199
200def get_profilers(**filter_data):
201 return rpc_utils.prepare_for_serialization(
202 models.Profiler.list_objects(filter_data))
203
204
mblighe8819cd2008-02-15 16:48:40 +0000205# users
206
207def add_user(login, access_level=None):
jadmanski0afbb632008-06-06 21:10:57 +0000208 return models.User.add_object(login=login, access_level=access_level).id
mblighe8819cd2008-02-15 16:48:40 +0000209
210
211def modify_user(id, **data):
jadmanski0afbb632008-06-06 21:10:57 +0000212 models.User.smart_get(id).update_object(data)
mblighe8819cd2008-02-15 16:48:40 +0000213
214
215def delete_user(id):
jadmanski0afbb632008-06-06 21:10:57 +0000216 models.User.smart_get(id).delete()
mblighe8819cd2008-02-15 16:48:40 +0000217
218
219def get_users(**filter_data):
jadmanski0afbb632008-06-06 21:10:57 +0000220 return rpc_utils.prepare_for_serialization(
221 models.User.list_objects(filter_data))
mblighe8819cd2008-02-15 16:48:40 +0000222
223
224# acl groups
225
226def add_acl_group(name, description=None):
showard04f2cd82008-07-25 20:53:31 +0000227 group = models.AclGroup.add_object(name=name, description=description)
228 group.users.add(thread_local.get_user())
229 return group.id
mblighe8819cd2008-02-15 16:48:40 +0000230
231
232def modify_acl_group(id, **data):
showard04f2cd82008-07-25 20:53:31 +0000233 group = models.AclGroup.smart_get(id)
234 group.check_for_acl_violation_acl_group()
235 group.update_object(data)
236 group.add_current_user_if_empty()
mblighe8819cd2008-02-15 16:48:40 +0000237
238
239def acl_group_add_users(id, users):
jadmanski0afbb632008-06-06 21:10:57 +0000240 group = models.AclGroup.smart_get(id)
showard04f2cd82008-07-25 20:53:31 +0000241 group.check_for_acl_violation_acl_group()
showardbe3ec042008-11-12 18:16:07 +0000242 users = models.User.smart_get_bulk(users)
jadmanski0afbb632008-06-06 21:10:57 +0000243 group.users.add(*users)
mblighe8819cd2008-02-15 16:48:40 +0000244
245
246def acl_group_remove_users(id, users):
jadmanski0afbb632008-06-06 21:10:57 +0000247 group = models.AclGroup.smart_get(id)
showard04f2cd82008-07-25 20:53:31 +0000248 group.check_for_acl_violation_acl_group()
showardbe3ec042008-11-12 18:16:07 +0000249 users = models.User.smart_get_bulk(users)
jadmanski0afbb632008-06-06 21:10:57 +0000250 group.users.remove(*users)
showard04f2cd82008-07-25 20:53:31 +0000251 group.add_current_user_if_empty()
mblighe8819cd2008-02-15 16:48:40 +0000252
253
254def acl_group_add_hosts(id, hosts):
jadmanski0afbb632008-06-06 21:10:57 +0000255 group = models.AclGroup.smart_get(id)
showard04f2cd82008-07-25 20:53:31 +0000256 group.check_for_acl_violation_acl_group()
showardbe3ec042008-11-12 18:16:07 +0000257 hosts = models.Host.smart_get_bulk(hosts)
jadmanski0afbb632008-06-06 21:10:57 +0000258 group.hosts.add(*hosts)
showard08f981b2008-06-24 21:59:03 +0000259 group.on_host_membership_change()
mblighe8819cd2008-02-15 16:48:40 +0000260
261
262def acl_group_remove_hosts(id, hosts):
jadmanski0afbb632008-06-06 21:10:57 +0000263 group = models.AclGroup.smart_get(id)
showard04f2cd82008-07-25 20:53:31 +0000264 group.check_for_acl_violation_acl_group()
showardbe3ec042008-11-12 18:16:07 +0000265 hosts = models.Host.smart_get_bulk(hosts)
jadmanski0afbb632008-06-06 21:10:57 +0000266 group.hosts.remove(*hosts)
showard08f981b2008-06-24 21:59:03 +0000267 group.on_host_membership_change()
mblighe8819cd2008-02-15 16:48:40 +0000268
269
270def delete_acl_group(id):
jadmanski0afbb632008-06-06 21:10:57 +0000271 models.AclGroup.smart_get(id).delete()
mblighe8819cd2008-02-15 16:48:40 +0000272
273
274def get_acl_groups(**filter_data):
jadmanski0afbb632008-06-06 21:10:57 +0000275 acl_groups = models.AclGroup.list_objects(filter_data)
276 for acl_group in acl_groups:
277 acl_group_obj = models.AclGroup.objects.get(id=acl_group['id'])
278 acl_group['users'] = [user.login
279 for user in acl_group_obj.users.all()]
280 acl_group['hosts'] = [host.hostname
281 for host in acl_group_obj.hosts.all()]
282 return rpc_utils.prepare_for_serialization(acl_groups)
mblighe8819cd2008-02-15 16:48:40 +0000283
284
285# jobs
286
mbligh120351e2009-01-24 01:40:45 +0000287def generate_control_file(tests=(), kernel=None, label=None, profilers=(),
288 client_control_file='', use_container=False):
jadmanski0afbb632008-06-06 21:10:57 +0000289 """
mbligh120351e2009-01-24 01:40:45 +0000290 Generates a client-side control file to load a kernel and run tests.
291
292 @param tests List of tests to run.
293 @param kernel Kernel to install in generated control file.
294 @param label Name of label to grab kernel config from.
295 @param profilers List of profilers to activate during the job.
296 @param client_control_file The contents of a client-side control file to
297 run at the end of all tests. If this is supplied, all tests must be
298 client side.
299 TODO: in the future we should support server control files directly
300 to wrap with a kernel. That'll require changing the parameter
301 name and adding a boolean to indicate if it is a client or server
302 control file.
303 @param use_container unused argument today. TODO: Enable containers
304 on the host during a client side test.
305
306 @returns a dict with the following keys:
307 control_file: str, The control file text.
308 is_server: bool, is the control file a server-side control file?
309 synch_count: How many machines the job uses per autoserv execution.
310 synch_count == 1 means the job is asynchronous.
311 dependencies: A list of the names of labels on which the job depends.
312 """
313 if not tests and not control_file:
showard2bab8f42008-11-12 18:15:22 +0000314 return dict(control_file='', is_server=False, synch_count=1,
showard989f25d2008-10-01 11:38:11 +0000315 dependencies=[])
mblighe8819cd2008-02-15 16:48:40 +0000316
showard989f25d2008-10-01 11:38:11 +0000317 cf_info, test_objects, profiler_objects, label = (
showard2b9a88b2008-06-13 20:55:03 +0000318 rpc_utils.prepare_generate_control_file(tests, kernel, label,
319 profilers))
showard989f25d2008-10-01 11:38:11 +0000320 cf_info['control_file'] = control_file.generate_control(
321 tests=test_objects, kernel=kernel, platform=label,
mbligh120351e2009-01-24 01:40:45 +0000322 profilers=profiler_objects, is_server=cf_info['is_server'],
323 client_control_file=client_control_file)
showard989f25d2008-10-01 11:38:11 +0000324 return cf_info
mblighe8819cd2008-02-15 16:48:40 +0000325
326
showard3bb499f2008-07-03 19:42:20 +0000327def create_job(name, priority, control_file, control_type, timeout=None,
showardc92da832009-04-07 18:14:34 +0000328 synch_count=None, hosts=(), meta_hosts=(),
329 run_verify=True, one_time_hosts=(), email_list='',
330 dependencies=(), reboot_before=None, reboot_after=None,
331 atomic_group_name=None):
jadmanski0afbb632008-06-06 21:10:57 +0000332 """\
333 Create and enqueue a job.
mblighe8819cd2008-02-15 16:48:40 +0000334
jadmanski0afbb632008-06-06 21:10:57 +0000335 priority: Low, Medium, High, Urgent
showardc92da832009-04-07 18:14:34 +0000336 control_file: String contents of the control file.
337 control_type: Type of control file, Client or Server.
338 timeout: Hours after this call returns until the job times out.
339 synch_count: How many machines the job uses per autoserv execution.
340 synch_count == 1 means the job is asynchronous. If an
341 atomic group is given this value is treated as a minimum.
342 hosts: List of hosts to run job on.
343 meta_hosts: List where each entry is a label name, and for each entry
jadmanski0afbb632008-06-06 21:10:57 +0000344 one host will be chosen from that label to run the job
345 on.
showardc92da832009-04-07 18:14:34 +0000346 run_verify: Should the host be verified before running the test?
347 one_time_hosts: List of hosts not in the database to run the job on.
348 email_list: String containing emails to mail when the job is done
349 dependencies: List of label names on which this job depends
mbligh94ae6d62009-02-03 17:47:49 +0000350 reboot_before: Never, If dirty, or Always
351 reboot_after: Never, If all tests passed, or Always
showardc92da832009-04-07 18:14:34 +0000352 atomic_group_name: The name of an atomic group to schedule the job on.
353
354 @returns The created Job id number.
jadmanski0afbb632008-06-06 21:10:57 +0000355 """
showard3bb499f2008-07-03 19:42:20 +0000356
357 if timeout is None:
358 timeout=global_config.global_config.get_config_value(
359 'AUTOTEST_WEB', 'job_timeout_default')
360
showardff901382008-07-07 23:22:16 +0000361 owner = thread_local.get_user().login
jadmanski0afbb632008-06-06 21:10:57 +0000362 # input validation
showardc92da832009-04-07 18:14:34 +0000363 if not (hosts or meta_hosts or one_time_hosts or atomic_group_name):
mblighec5546d2008-06-16 16:51:28 +0000364 raise model_logic.ValidationError({
showardb8471e32008-07-03 19:51:08 +0000365 'arguments' : "You must pass at least one of 'hosts', "
showardc92da832009-04-07 18:14:34 +0000366 "'meta_hosts', 'one_time_hosts', "
367 "or 'atomic_group_name'"
jadmanski0afbb632008-06-06 21:10:57 +0000368 })
mblighe8819cd2008-02-15 16:48:40 +0000369
showardc92da832009-04-07 18:14:34 +0000370 # Create and sanity check an AtomicGroup object if requested.
371 if atomic_group_name:
372 if one_time_hosts:
373 raise model_logic.ValidationError(
374 {'one_time_hosts':
375 'One time hosts cannot be used with an Atomic Group.'})
376 atomic_group = models.AtomicGroup.smart_get(atomic_group_name)
377 if synch_count and synch_count > atomic_group.max_number_of_machines:
378 raise model_logic.ValidationError(
379 {'atomic_group_name' :
380 'You have requested a synch_count (%d) greater than the '
381 'maximum machines in the requested Atomic Group (%d).' %
382 (synch_count, atomic_group.max_number_of_machines)})
383 else:
384 atomic_group = None
385
showard989f25d2008-10-01 11:38:11 +0000386 labels_by_name = dict((label.name, label)
387 for label in models.Label.objects.all())
showardba872902008-06-28 00:51:08 +0000388
jadmanski0afbb632008-06-06 21:10:57 +0000389 # convert hostnames & meta hosts to host/label objects
showardc92da832009-04-07 18:14:34 +0000390 host_objects = models.Host.smart_get_bulk(hosts)
showard989f25d2008-10-01 11:38:11 +0000391 metahost_objects = []
392 metahost_counts = {}
jadmanski0afbb632008-06-06 21:10:57 +0000393 for label in meta_hosts or []:
showard25087ac2008-11-24 22:12:03 +0000394 if label not in labels_by_name:
395 raise model_logic.ValidationError(
396 {'meta_hosts' : 'Label "%s" not found' % label})
showard989f25d2008-10-01 11:38:11 +0000397 this_label = labels_by_name[label]
398 metahost_objects.append(this_label)
399 metahost_counts.setdefault(this_label, 0)
400 metahost_counts[this_label] += 1
showardb8471e32008-07-03 19:51:08 +0000401 for host in one_time_hosts or []:
402 this_host = models.Host.create_one_time_host(host)
403 host_objects.append(this_host)
showardba872902008-06-28 00:51:08 +0000404
showard65c267d2009-01-20 23:24:47 +0000405 all_host_objects = host_objects + metahost_objects
406
showardba872902008-06-28 00:51:08 +0000407 # check that each metahost request has enough hosts under the label
showard989f25d2008-10-01 11:38:11 +0000408 for label, requested_count in metahost_counts.iteritems():
409 available_count = label.host_set.count()
410 if requested_count > available_count:
411 error = ("You have requested %d %s's, but there are only %d."
412 % (requested_count, label.name, available_count))
413 raise model_logic.ValidationError({'meta_hosts' : error})
mblighe8819cd2008-02-15 16:48:40 +0000414
showardc92da832009-04-07 18:14:34 +0000415 if atomic_group:
416 rpc_utils.check_atomic_group_create_job(
417 synch_count, host_objects, metahost_objects,
418 dependencies, atomic_group, labels_by_name)
419 else:
420 if synch_count is not None and synch_count > len(all_host_objects):
421 raise model_logic.ValidationError(
422 {'hosts':
423 'only %d hosts provided for job with synch_count = %d' %
424 (len(all_host_objects), synch_count)})
showard47d9e7d2009-04-20 19:33:56 +0000425 atomic_hosts = models.Host.objects.filter(
426 id__in=[host.id for host in host_objects],
427 labels__atomic_group=True)
428 unusable_host_names = [host.hostname for host in atomic_hosts]
429 if unusable_host_names:
430 raise model_logic.ValidationError(
431 {'hosts':
432 'Host(s) "%s" are atomic group hosts but no '
433 'atomic group was specified for this job.' %
434 (', '.join(unusable_host_names),)})
435
showard65c267d2009-01-20 23:24:47 +0000436
showard989f25d2008-10-01 11:38:11 +0000437 rpc_utils.check_job_dependencies(host_objects, dependencies)
438 dependency_labels = [labels_by_name[label_name]
439 for label_name in dependencies]
440
showard3181c7b2009-04-20 19:34:42 +0000441 for label in metahost_objects + dependency_labels:
442 if label.atomic_group and not atomic_group:
443 raise model_logic.ValidationError(
444 {'atomic_group_name':
445 'Some meta_hosts or dependencies require an atomic group '
446 'but no atomic_group_name was specified for this job.'})
447 elif (label.atomic_group and
448 label.atomic_group.name != atomic_group_name):
449 raise model_logic.ValidationError(
450 {'atomic_group_name':
451 'Some meta_hosts or dependencies require an atomic group '
452 'other than the one requested for this job.'})
453
jadmanski0afbb632008-06-06 21:10:57 +0000454 job = models.Job.create(owner=owner, name=name, priority=priority,
455 control_file=control_file,
456 control_type=control_type,
showard2bab8f42008-11-12 18:15:22 +0000457 synch_count=synch_count,
showard65c267d2009-01-20 23:24:47 +0000458 hosts=all_host_objects,
showard909c7a62008-07-15 21:52:38 +0000459 timeout=timeout,
showard542e8402008-09-19 20:16:18 +0000460 run_verify=run_verify,
showard989f25d2008-10-01 11:38:11 +0000461 email_list=email_list.strip(),
showard21baa452008-10-21 00:08:39 +0000462 dependencies=dependency_labels,
463 reboot_before=reboot_before,
464 reboot_after=reboot_after)
showardc92da832009-04-07 18:14:34 +0000465 job.queue(all_host_objects, atomic_group=atomic_group)
jadmanski0afbb632008-06-06 21:10:57 +0000466 return job.id
mblighe8819cd2008-02-15 16:48:40 +0000467
468
showard9dbdcda2008-10-14 17:34:36 +0000469def abort_host_queue_entries(**filter_data):
jadmanski0afbb632008-06-06 21:10:57 +0000470 """\
showard9dbdcda2008-10-14 17:34:36 +0000471 Abort a set of host queue entries.
jadmanski0afbb632008-06-06 21:10:57 +0000472 """
showard9dbdcda2008-10-14 17:34:36 +0000473 query = models.HostQueueEntry.query_objects(filter_data)
showard0c185192009-01-16 03:07:57 +0000474 query = query.filter(complete=False)
showarddc817512008-11-12 18:16:41 +0000475 models.AclGroup.check_abort_permissions(query)
showard9dbdcda2008-10-14 17:34:36 +0000476 host_queue_entries = list(query.select_related())
showard2bab8f42008-11-12 18:15:22 +0000477 rpc_utils.check_abort_synchronous_jobs(host_queue_entries)
mblighe8819cd2008-02-15 16:48:40 +0000478
showard9dbdcda2008-10-14 17:34:36 +0000479 user = thread_local.get_user()
480 for queue_entry in host_queue_entries:
481 queue_entry.abort(user)
showard9d821ab2008-07-11 16:54:29 +0000482
483
mblighe8819cd2008-02-15 16:48:40 +0000484def get_jobs(not_yet_run=False, running=False, finished=False, **filter_data):
jadmanski0afbb632008-06-06 21:10:57 +0000485 """\
486 Extra filter args for get_jobs:
487 -not_yet_run: Include only jobs that have not yet started running.
488 -running: Include only jobs that have start running but for which not
489 all hosts have completed.
490 -finished: Include only jobs for which all hosts have completed (or
491 aborted).
492 At most one of these three fields should be specified.
493 """
494 filter_data['extra_args'] = rpc_utils.extra_job_filters(not_yet_run,
495 running,
496 finished)
showard989f25d2008-10-01 11:38:11 +0000497 jobs = models.Job.list_objects(filter_data)
498 models.Job.objects.populate_dependencies(jobs)
499 return rpc_utils.prepare_for_serialization(jobs)
mblighe8819cd2008-02-15 16:48:40 +0000500
501
502def get_num_jobs(not_yet_run=False, running=False, finished=False,
jadmanski0afbb632008-06-06 21:10:57 +0000503 **filter_data):
504 """\
505 See get_jobs() for documentation of extra filter parameters.
506 """
507 filter_data['extra_args'] = rpc_utils.extra_job_filters(not_yet_run,
508 running,
509 finished)
510 return models.Job.query_count(filter_data)
mblighe8819cd2008-02-15 16:48:40 +0000511
512
mblighe8819cd2008-02-15 16:48:40 +0000513def get_jobs_summary(**filter_data):
jadmanski0afbb632008-06-06 21:10:57 +0000514 """\
showarda8709c52008-07-03 19:44:54 +0000515 Like get_jobs(), but adds a 'status_counts' field, which is a dictionary
jadmanski0afbb632008-06-06 21:10:57 +0000516 mapping status strings to the number of hosts currently with that
517 status, i.e. {'Queued' : 4, 'Running' : 2}.
518 """
519 jobs = get_jobs(**filter_data)
520 ids = [job['id'] for job in jobs]
521 all_status_counts = models.Job.objects.get_status_counts(ids)
522 for job in jobs:
523 job['status_counts'] = all_status_counts[job['id']]
524 return rpc_utils.prepare_for_serialization(jobs)
mblighe8819cd2008-02-15 16:48:40 +0000525
526
showard945072f2008-09-03 20:34:59 +0000527def get_info_for_clone(id, preserve_metahosts):
showarda8709c52008-07-03 19:44:54 +0000528 """\
529 Retrieves all the information needed to clone a job.
530 """
531 info = {}
532 job = models.Job.objects.get(id=id)
showard45195442009-02-17 20:55:48 +0000533 query = job.hostqueueentry_set.filter()
showard945072f2008-09-03 20:34:59 +0000534
535 hosts = []
536 meta_hosts = []
showardc92da832009-04-07 18:14:34 +0000537 atomic_group_name = None
showard945072f2008-09-03 20:34:59 +0000538
539 # For each queue entry, if the entry contains a host, add the entry into the
540 # hosts list if either:
541 # It is not a metahost.
542 # It was an assigned metahost, and the user wants to keep the specific
543 # assignments.
544 # Otherwise, add the metahost to the metahosts list.
545 for queue_entry in query:
546 if (queue_entry.host and (preserve_metahosts
547 or not queue_entry.meta_host)):
showard45195442009-02-17 20:55:48 +0000548 if queue_entry.deleted:
549 continue
showard945072f2008-09-03 20:34:59 +0000550 hosts.append(queue_entry.host)
551 else:
552 meta_hosts.append(queue_entry.meta_host.name)
showardc92da832009-04-07 18:14:34 +0000553 if atomic_group_name is None:
showard3562a062009-04-13 16:05:30 +0000554 if queue_entry.atomic_group is not None:
555 atomic_group_name = queue_entry.atomic_group.name
showardc92da832009-04-07 18:14:34 +0000556 else:
557 assert atomic_group_name == queue_entry.atomic_group.name, (
558 'DB inconsistency. HostQueueEntries with multiple atomic'
559 ' groups on job %s: %s != %s' % (
560 id, atomic_group_name, queue_entry.atomic_group.name))
showard945072f2008-09-03 20:34:59 +0000561
showardd9992fe2008-07-31 02:15:03 +0000562 host_dicts = []
showarda8709c52008-07-03 19:44:54 +0000563
showardd9992fe2008-07-31 02:15:03 +0000564 for host in hosts:
showardd9992fe2008-07-31 02:15:03 +0000565 # one-time host
566 if host.invalid:
showardbad4f2d2008-08-15 18:13:47 +0000567 host_dict = {}
568 host_dict['hostname'] = host.hostname
569 host_dict['id'] = host.id
showardd9992fe2008-07-31 02:15:03 +0000570 host_dict['platform'] = '(one-time host)'
571 host_dict['locked_text'] = ''
showardd9992fe2008-07-31 02:15:03 +0000572 else:
showardbad4f2d2008-08-15 18:13:47 +0000573 host_dict = get_hosts(id=host.id)[0]
574 other_labels = host_dict['labels']
575 if host_dict['platform']:
576 other_labels.remove(host_dict['platform'])
577 host_dict['other_labels'] = ', '.join(other_labels)
showardd9992fe2008-07-31 02:15:03 +0000578 host_dicts.append(host_dict)
showarda8709c52008-07-03 19:44:54 +0000579
580 meta_host_counts = {}
581 for meta_host in meta_hosts:
582 meta_host_counts.setdefault(meta_host, 0)
583 meta_host_counts[meta_host] += 1
584
585 info['job'] = job.get_object_dict()
showard2e9415f2008-12-05 19:48:38 +0000586 info['job']['dependencies'] = [label.name for label
587 in job.dependency_labels.all()]
showarda8709c52008-07-03 19:44:54 +0000588 info['meta_host_counts'] = meta_host_counts
showardd9992fe2008-07-31 02:15:03 +0000589 info['hosts'] = host_dicts
showardc92da832009-04-07 18:14:34 +0000590 info['atomic_group_name'] = atomic_group_name
showarda8709c52008-07-03 19:44:54 +0000591
592 return rpc_utils.prepare_for_serialization(info)
593
594
showard34dc5fa2008-04-24 20:58:40 +0000595# host queue entries
596
597def get_host_queue_entries(**filter_data):
jadmanski0afbb632008-06-06 21:10:57 +0000598 """\
showardc92da832009-04-07 18:14:34 +0000599 @returns A sequence of nested dictionaries of host and job information.
jadmanski0afbb632008-06-06 21:10:57 +0000600 """
showardc92da832009-04-07 18:14:34 +0000601 return rpc_utils.prepare_rows_as_nested_dicts(
602 models.HostQueueEntry.query_objects(filter_data),
603 ('host', 'atomic_group', 'job'))
showard34dc5fa2008-04-24 20:58:40 +0000604
605
606def get_num_host_queue_entries(**filter_data):
jadmanski0afbb632008-06-06 21:10:57 +0000607 """\
608 Get the number of host queue entries associated with this job.
609 """
610 return models.HostQueueEntry.query_count(filter_data)
showard34dc5fa2008-04-24 20:58:40 +0000611
612
showard1e935f12008-07-11 00:11:36 +0000613def get_hqe_percentage_complete(**filter_data):
614 """
showardc92da832009-04-07 18:14:34 +0000615 Computes the fraction of host queue entries matching the given filter data
showard1e935f12008-07-11 00:11:36 +0000616 that are complete.
617 """
618 query = models.HostQueueEntry.query_objects(filter_data)
619 complete_count = query.filter(complete=True).count()
620 total_count = query.count()
621 if total_count == 0:
622 return 1
623 return float(complete_count) / total_count
624
625
mblighe8819cd2008-02-15 16:48:40 +0000626# other
627
showarde0b63622008-08-04 20:58:47 +0000628def echo(data=""):
629 """\
630 Returns a passed in string. For doing a basic test to see if RPC calls
631 can successfully be made.
632 """
633 return data
634
635
mblighe8819cd2008-02-15 16:48:40 +0000636def get_static_data():
jadmanski0afbb632008-06-06 21:10:57 +0000637 """\
638 Returns a dictionary containing a bunch of data that shouldn't change
639 often and is otherwise inaccessible. This includes:
showardc92da832009-04-07 18:14:34 +0000640
641 priorities: List of job priority choices.
642 default_priority: Default priority value for new jobs.
643 users: Sorted list of all users.
644 labels: Sorted list of all labels.
645 atomic_groups: Sorted list of all atomic groups.
646 tests: Sorted list of all tests.
647 profilers: Sorted list of all profilers.
648 current_user: Logged-in username.
649 host_statuses: Sorted list of possible Host statuses.
650 job_statuses: Sorted list of possible HostQueueEntry statuses.
651 job_timeout_default: The default job timeout length in hours.
652 reboot_before_options: A list of valid RebootBefore string enums.
653 reboot_after_options: A list of valid RebootAfter string enums.
654 motd: Server's message of the day.
655 status_dictionary: A mapping from one word job status names to a more
656 informative description.
jadmanski0afbb632008-06-06 21:10:57 +0000657 """
showard21baa452008-10-21 00:08:39 +0000658
659 job_fields = models.Job.get_field_dict()
660
jadmanski0afbb632008-06-06 21:10:57 +0000661 result = {}
662 result['priorities'] = models.Job.Priority.choices()
showard21baa452008-10-21 00:08:39 +0000663 default_priority = job_fields['priority'].default
jadmanski0afbb632008-06-06 21:10:57 +0000664 default_string = models.Job.Priority.get_string(default_priority)
665 result['default_priority'] = default_string
666 result['users'] = get_users(sort_by=['login'])
667 result['labels'] = get_labels(sort_by=['-platform', 'name'])
showardc92da832009-04-07 18:14:34 +0000668 result['atomic_groups'] = get_atomic_groups(sort_by=['name'])
jadmanski0afbb632008-06-06 21:10:57 +0000669 result['tests'] = get_tests(sort_by=['name'])
showard2b9a88b2008-06-13 20:55:03 +0000670 result['profilers'] = get_profilers(sort_by=['name'])
showard0fc38302008-10-23 00:44:07 +0000671 result['current_user'] = rpc_utils.prepare_for_serialization(
672 thread_local.get_user().get_object_dict())
showard2b9a88b2008-06-13 20:55:03 +0000673 result['host_statuses'] = sorted(models.Host.Status.names)
mbligh5a198b92008-12-11 19:33:29 +0000674 result['job_statuses'] = sorted(models.HostQueueEntry.Status.names)
showardb1e51872008-10-07 11:08:18 +0000675 result['job_timeout_default'] = models.Job.DEFAULT_TIMEOUT
showard0fc38302008-10-23 00:44:07 +0000676 result['reboot_before_options'] = models.RebootBefore.names
677 result['reboot_after_options'] = models.RebootAfter.names
showard8fbae652009-01-20 23:23:10 +0000678 result['motd'] = rpc_utils.get_motd()
showard8ac29b42008-07-17 17:01:55 +0000679
showard95128e52008-08-04 20:59:34 +0000680 result['status_dictionary'] = {"Abort": "Abort",
681 "Aborted": "Aborted",
showard8ac29b42008-07-17 17:01:55 +0000682 "Verifying": "Verifying Host",
683 "Pending": "Waiting on other hosts",
684 "Running": "Running autoserv",
685 "Completed": "Autoserv completed",
686 "Failed": "Failed to complete",
687 "Aborting": "Abort in progress",
showardd823b362008-07-24 16:35:46 +0000688 "Queued": "Queued",
showard5deb6772008-11-04 21:54:33 +0000689 "Starting": "Next in host's queue",
690 "Stopped": "Other host(s) failed verify",
691 "Parsing": "Awaiting parse of final results"}
jadmanski0afbb632008-06-06 21:10:57 +0000692 return result