mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 1 | """\ |
| 2 | Functions to expose over the RPC interface. |
| 3 | |
| 4 | For all modify* and delete* functions that ask for an 'id' parameter to |
| 5 | identify the object to operate on, the id may be either |
| 6 | * the database row ID |
| 7 | * the name of the object (label name, hostname, user login, etc.) |
| 8 | * a dictionary containing uniquely identifying field (this option should seldom |
| 9 | be used) |
| 10 | |
| 11 | When specifying foreign key fields (i.e. adding hosts to a label, or adding |
| 12 | users to an ACL group), the given value may be either the database row ID or the |
| 13 | name of the object. |
| 14 | |
| 15 | All get* functions return lists of dictionaries. Each dictionary represents one |
| 16 | object and maps field names to values. |
| 17 | |
| 18 | Some examples: |
| 19 | modify_host(2, hostname='myhost') # modify hostname of host with database ID 2 |
| 20 | modify_host('ipaj2', hostname='myhost') # modify hostname of host 'ipaj2' |
| 21 | modify_test('sleeptest', test_type='Client', params=', seconds=60') |
| 22 | delete_acl_group(1) # delete by ID |
| 23 | delete_acl_group('Everyone') # delete by name |
| 24 | acl_group_add_users('Everyone', ['mbligh', 'showard']) |
| 25 | get_jobs(owner='showard', status='Queued') |
| 26 | |
| 27 | See doctests/rpc_test.txt for (lots) more examples. |
| 28 | """ |
| 29 | |
| 30 | __author__ = 'showard@google.com (Steve Howard)' |
| 31 | |
| 32 | import models, control_file, rpc_utils |
| 33 | |
| 34 | # labels |
| 35 | |
| 36 | def add_label(name, kernel_config=None, platform=None): |
| 37 | return models.Label.add_object(name=name, kernel_config=kernel_config, |
| 38 | platform=platform).id |
| 39 | |
| 40 | |
| 41 | def modify_label(id, **data): |
| 42 | models.Label.smart_get(id).update_object(data) |
| 43 | |
| 44 | |
| 45 | def delete_label(id): |
| 46 | models.Label.smart_get(id).delete() |
| 47 | |
| 48 | |
| 49 | def get_labels(**filter_data): |
| 50 | return rpc_utils.prepare_for_serialization( |
| 51 | models.Label.list_objects(filter_data)) |
| 52 | |
| 53 | |
| 54 | # hosts |
| 55 | |
| 56 | def add_host(hostname, status=None, locked=None): |
| 57 | return models.Host.add_object(hostname=hostname, status=status, |
| 58 | locked=locked).id |
| 59 | |
| 60 | |
| 61 | def modify_host(id, **data): |
| 62 | models.Host.smart_get(id).update_object(data) |
| 63 | |
| 64 | |
| 65 | def host_add_labels(id, labels): |
| 66 | labels = [models.Label.smart_get(label) for label in labels] |
| 67 | models.Host.smart_get(id).labels.add(*labels) |
| 68 | |
| 69 | |
| 70 | def host_remove_labels(id, labels): |
| 71 | labels = [models.Label.smart_get(label) for label in labels] |
| 72 | models.Host.smart_get(id).labels.remove(*labels) |
| 73 | |
| 74 | |
| 75 | def delete_host(id): |
| 76 | models.Host.smart_get(id).delete() |
| 77 | |
| 78 | |
| 79 | def get_hosts(**filter_data): |
| 80 | hosts = models.Host.list_objects(filter_data) |
| 81 | for host in hosts: |
| 82 | host_obj = models.Host.objects.get(id=host['id']) |
| 83 | host['labels'] = [label.name |
| 84 | for label in host_obj.labels.all()] |
| 85 | platform = host_obj.platform() |
| 86 | host['platform'] = platform and platform.name or None |
| 87 | return rpc_utils.prepare_for_serialization(hosts) |
| 88 | |
| 89 | |
| 90 | |
| 91 | # tests |
| 92 | |
| 93 | def add_test(name, test_type, path, test_class=None, description=None): |
| 94 | return models.Test.add_object(name=name, test_type=test_type, path=path, |
| 95 | test_class=test_class, |
| 96 | description=description).id |
| 97 | |
| 98 | |
| 99 | def modify_test(id, **data): |
| 100 | models.Test.smart_get(id).update_object(data) |
| 101 | |
| 102 | |
| 103 | def delete_test(id): |
| 104 | models.Test.smart_get(id).delete() |
| 105 | |
| 106 | |
| 107 | def get_tests(**filter_data): |
| 108 | return rpc_utils.prepare_for_serialization( |
| 109 | models.Test.list_objects(filter_data)) |
| 110 | |
| 111 | |
| 112 | # users |
| 113 | |
| 114 | def add_user(login, access_level=None): |
| 115 | return models.User.add_object(login=login, access_level=access_level).id |
| 116 | |
| 117 | |
| 118 | def modify_user(id, **data): |
| 119 | models.User.smart_get(id).update_object(data) |
| 120 | |
| 121 | |
| 122 | def delete_user(id): |
| 123 | models.User.smart_get(id).delete() |
| 124 | |
| 125 | |
| 126 | def get_users(**filter_data): |
| 127 | return rpc_utils.prepare_for_serialization( |
| 128 | models.User.list_objects(filter_data)) |
| 129 | |
| 130 | |
| 131 | # acl groups |
| 132 | |
| 133 | def add_acl_group(name, description=None): |
| 134 | return models.AclGroup.add_object(name=name, description=description).id |
| 135 | |
| 136 | |
| 137 | def modify_acl_group(id, **data): |
| 138 | models.AclGroup.smart_get(id).update_object(data) |
| 139 | |
| 140 | |
| 141 | def acl_group_add_users(id, users): |
| 142 | users = [models.User.smart_get(user) for user in users] |
| 143 | models.AclGroup.smart_get(id).users.add(*users) |
| 144 | |
| 145 | |
| 146 | def acl_group_remove_users(id, users): |
| 147 | users = [models.User.smart_get(user) for user in users] |
| 148 | models.AclGroup.smart_get(id).users.remove(*users) |
| 149 | |
| 150 | |
| 151 | def acl_group_add_hosts(id, hosts): |
| 152 | hosts = [models.Host.smart_get(host) for host in hosts] |
| 153 | models.AclGroup.smart_get(id).hosts.add(*hosts) |
| 154 | |
| 155 | |
| 156 | def acl_group_remove_hosts(id, hosts): |
| 157 | hosts = [models.Host.smart_get(host) for host in hosts] |
| 158 | models.AclGroup.smart_get(id).hosts.remove(*hosts) |
| 159 | |
| 160 | |
| 161 | def delete_acl_group(id): |
| 162 | models.AclGroup.smart_get(id).delete() |
| 163 | |
| 164 | |
| 165 | def get_acl_groups(**filter_data): |
| 166 | acl_groups = models.AclGroup.list_objects(filter_data) |
| 167 | for acl_group in acl_groups: |
| 168 | acl_group_obj = models.AclGroup.objects.get(id=acl_group['id']) |
| 169 | acl_group['users'] = [user.login |
| 170 | for user in acl_group_obj.users.all()] |
| 171 | acl_group['hosts'] = [host.hostname |
| 172 | for host in acl_group_obj.hosts.all()] |
| 173 | return rpc_utils.prepare_for_serialization(acl_groups) |
| 174 | |
| 175 | |
| 176 | # jobs |
| 177 | |
| 178 | def generate_control_file(tests, kernel=None, label=None): |
| 179 | """\ |
| 180 | Generates a client-side control file to load a kernel and run a set of |
showard | 8fd5824 | 2008-03-10 21:29:07 +0000 | [diff] [blame^] | 181 | tests. Returns a tuple (control_file, is_server, is_synchronous): |
| 182 | control_file - the control file text |
| 183 | is_server - is the control file a server-side control file? |
| 184 | is_synchronous - should the control file be run synchronously? |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 185 | |
| 186 | tests: list of tests to run |
| 187 | kernel: kernel to install in generated control file |
| 188 | label: name of label to grab kernel config from |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 189 | """ |
| 190 | if not tests: |
showard | 8fd5824 | 2008-03-10 21:29:07 +0000 | [diff] [blame^] | 191 | return '', False, False |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 192 | |
showard | 8fd5824 | 2008-03-10 21:29:07 +0000 | [diff] [blame^] | 193 | is_server, is_synchronous, test_objects, label = ( |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 194 | rpc_utils.prepare_generate_control_file(tests, kernel, label)) |
| 195 | if is_server: |
showard | 8fd5824 | 2008-03-10 21:29:07 +0000 | [diff] [blame^] | 196 | cf_text = control_file.generate_server_control(test_objects) |
| 197 | cf_text = control_file.generate_client_control(test_objects, kernel, |
| 198 | label) |
| 199 | return cf_text, is_server, is_synchronous |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 200 | |
| 201 | |
| 202 | def create_job(name, priority, control_file, control_type, is_synchronous=None, |
| 203 | hosts=None, meta_hosts=None): |
| 204 | """\ |
| 205 | Create and enqueue a job. |
| 206 | |
| 207 | priority: Low, Medium, High, Urgent |
| 208 | control_file: contents of control file |
| 209 | control_type: type of control file, Client or Server |
| 210 | is_synchronous: boolean indicating if a job is synchronous |
| 211 | hosts: list of hosts to run job on |
| 212 | meta_hosts: list where each entry is a label name, and for each entry |
| 213 | one host will be chosen from that label to run the job |
| 214 | on. |
| 215 | """ |
| 216 | owner = rpc_utils.get_user().login |
| 217 | # input validation |
| 218 | if not hosts and not meta_hosts: |
| 219 | raise models.ValidationError({ |
| 220 | 'arguments' : "You must pass at least one of 'hosts' or " |
| 221 | "'meta_hosts'" |
| 222 | }) |
| 223 | |
| 224 | # convert hostnames & meta hosts to host/label objects |
| 225 | host_objects = [] |
| 226 | for host in hosts or []: |
| 227 | this_host = models.Host.smart_get(host) |
| 228 | host_objects.append(this_host) |
| 229 | for label in meta_hosts or []: |
| 230 | this_label = models.Label.smart_get(label) |
| 231 | host_objects.append(this_label) |
| 232 | |
| 233 | # default is_synchronous to some appropriate value |
| 234 | ControlType = models.Job.ControlType |
| 235 | control_type = ControlType.get_value(control_type) |
| 236 | if is_synchronous is None: |
| 237 | is_synchronous = (control_type == ControlType.SERVER) |
| 238 | # convert the synch flag to an actual type |
| 239 | if is_synchronous: |
showard | 8fd5824 | 2008-03-10 21:29:07 +0000 | [diff] [blame^] | 240 | synch_type = models.Test.SynchType.SYNCHRONOUS |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 241 | else: |
showard | 8fd5824 | 2008-03-10 21:29:07 +0000 | [diff] [blame^] | 242 | synch_type = models.Test.SynchType.ASYNCHRONOUS |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 243 | |
| 244 | job = models.Job.create(owner=owner, name=name, priority=priority, |
| 245 | control_file=control_file, |
| 246 | control_type=control_type, |
| 247 | synch_type=synch_type, |
| 248 | hosts=host_objects) |
| 249 | job.queue(host_objects) |
| 250 | return job.id |
| 251 | |
| 252 | |
mbligh | 3cab4a7 | 2008-03-05 23:19:09 +0000 | [diff] [blame] | 253 | def requeue_job(id): |
| 254 | """\ |
| 255 | Create and enqueue a copy of the given job. |
| 256 | """ |
| 257 | job = models.Job.objects.get(id=id) |
| 258 | new_job = job.requeue(rpc_utils.get_user().login) |
| 259 | return new_job.id |
| 260 | |
| 261 | |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 262 | def abort_job(id): |
| 263 | """\ |
| 264 | Abort the job with the given id number. |
| 265 | """ |
| 266 | job = models.Job.objects.get(id=id) |
| 267 | job.abort() |
| 268 | |
| 269 | |
| 270 | def get_jobs(not_yet_run=False, running=False, finished=False, **filter_data): |
| 271 | """\ |
| 272 | Extra filter args for get_jobs: |
| 273 | -not_yet_run: Include only jobs that have not yet started running. |
| 274 | -running: Include only jobs that have start running but for which not |
| 275 | all hosts have completed. |
| 276 | -finished: Include only jobs for which all hosts have completed (or |
| 277 | aborted). |
| 278 | At most one of these fields should be specified. |
| 279 | """ |
| 280 | filter_data['extra_args'] = rpc_utils.extra_job_filters(not_yet_run, |
| 281 | running, |
| 282 | finished) |
| 283 | return rpc_utils.prepare_for_serialization( |
| 284 | models.Job.list_objects(filter_data)) |
| 285 | |
| 286 | |
| 287 | def get_num_jobs(not_yet_run=False, running=False, finished=False, |
| 288 | **filter_data): |
| 289 | """\ |
| 290 | Get the number of jobs matching the given filters. query_start and |
| 291 | query_limit parameters are ignored. See get_jobs() for documentation of |
| 292 | extra filter parameters. |
| 293 | """ |
| 294 | filter_data.pop('query_start', None) |
| 295 | filter_data.pop('query_limit', None) |
| 296 | filter_data['extra_args'] = rpc_utils.extra_job_filters(not_yet_run, |
| 297 | running, |
| 298 | finished) |
| 299 | return models.Job.query_objects(filter_data).count() |
| 300 | |
| 301 | |
| 302 | def job_status(id): |
| 303 | """\ |
| 304 | Get status of job with the given id number. Returns a dictionary |
| 305 | mapping hostnames to dictionaries with two keys each: |
| 306 | * 'status' : the job status for that host |
| 307 | * 'meta_count' : For meta host entires, gives a count of how many |
| 308 | entries there are for this label (the hostname is |
| 309 | then a label name). For real host entries, |
| 310 | meta_count is None. |
| 311 | """ |
| 312 | job = models.Job.objects.get(id=id) |
| 313 | hosts_status = {} |
| 314 | for queue_entry in job.hostqueueentry_set.all(): |
| 315 | is_meta = queue_entry.is_meta_host_entry() |
| 316 | if is_meta: |
| 317 | name = queue_entry.meta_host.name |
| 318 | hosts_status.setdefault(name, {'meta_count': 0}) |
| 319 | hosts_status[name]['meta_count'] += 1 |
| 320 | else: |
| 321 | name = queue_entry.host.hostname |
| 322 | hosts_status[name] = {'meta_count': None} |
| 323 | hosts_status[name]['status'] = queue_entry.status |
| 324 | return hosts_status |
| 325 | |
| 326 | |
| 327 | def get_jobs_summary(**filter_data): |
| 328 | """\ |
| 329 | Like get_jobs(), but adds a 'stauts_counts' field, which is a dictionary |
| 330 | mapping status strings to the number of hosts currently with that |
| 331 | status, i.e. {'Queued' : 4, 'Running' : 2}. |
| 332 | """ |
| 333 | jobs = get_jobs(**filter_data) |
| 334 | ids = [job['id'] for job in jobs] |
| 335 | all_status_counts = models.Job.objects.get_status_counts(ids) |
| 336 | for job in jobs: |
| 337 | job['status_counts'] = all_status_counts[job['id']] |
| 338 | return rpc_utils.prepare_for_serialization(jobs) |
| 339 | |
| 340 | |
| 341 | # other |
| 342 | |
| 343 | def get_static_data(): |
| 344 | """\ |
| 345 | Returns a dictionary containing a bunch of data that shouldn't change |
| 346 | often. This includes: |
| 347 | priorities: list of job priority choices |
| 348 | default_priority: default priority value for new jobs |
| 349 | users: sorted list of all usernames |
| 350 | labels: sorted list of all label names |
| 351 | tests: sorted list of all test names |
| 352 | user_login: username |
| 353 | """ |
| 354 | result = {} |
| 355 | result['priorities'] = models.Job.Priority.choices() |
| 356 | default_priority = models.Job.get_field_dict()['priority'].default |
| 357 | default_string = models.Job.Priority.get_string(default_priority) |
| 358 | result['default_priority'] = default_string |
| 359 | result['users'] = [user.login for user in |
| 360 | models.User.objects.all().order_by('login')] |
| 361 | result['labels'] = [label.name for label in |
| 362 | models.Label.objects.all().order_by('name')] |
| 363 | result['tests'] = get_tests(sort_by='name') |
| 364 | result['user_login'] = rpc_utils.get_user().login |
| 365 | return result |