blob: 3384ab004951471e0c45862a641ba01fa48bf67e [file] [log] [blame]
showardfb2a7fa2008-07-17 17:04:12 +00001from datetime import datetime
showard7c785282008-05-29 19:45:12 +00002from django.db import models as dbmodels, connection
showarddf062562008-07-03 19:56:37 +00003from frontend.afe import model_logic
showard3dd47c22008-07-10 00:41:36 +00004from frontend import settings, thread_local
showardb1e51872008-10-07 11:08:18 +00005from autotest_lib.client.common_lib import enum, host_protections, global_config
showard2bab8f42008-11-12 18:15:22 +00006from autotest_lib.client.common_lib import debug
7
8logger = debug.get_logger()
mblighe8819cd2008-02-15 16:48:40 +00009
showard0fc38302008-10-23 00:44:07 +000010# job options and user preferences
11RebootBefore = enum.Enum('Never', 'If dirty', 'Always')
12DEFAULT_REBOOT_BEFORE = RebootBefore.IF_DIRTY
13RebootAfter = enum.Enum('Never', 'If all tests passed', 'Always')
14DEFAULT_REBOOT_AFTER = RebootBefore.ALWAYS
mblighe8819cd2008-02-15 16:48:40 +000015
mblighe8819cd2008-02-15 16:48:40 +000016class AclAccessViolation(Exception):
jadmanski0afbb632008-06-06 21:10:57 +000017 """\
18 Raised when an operation is attempted with proper permissions as
19 dictated by ACLs.
20 """
mblighe8819cd2008-02-15 16:48:40 +000021
22
showard7c785282008-05-29 19:45:12 +000023class Label(model_logic.ModelWithInvalid, dbmodels.Model):
jadmanski0afbb632008-06-06 21:10:57 +000024 """\
25 Required:
26 name: label name
mblighe8819cd2008-02-15 16:48:40 +000027
jadmanski0afbb632008-06-06 21:10:57 +000028 Optional:
29 kernel_config: url/path to kernel config to use for jobs run on this
30 label
31 platform: if True, this is a platform label (defaults to False)
showard989f25d2008-10-01 11:38:11 +000032 only_if_needed: if True, a machine with this label can only be used if that
33 that label is requested by the job/test.
jadmanski0afbb632008-06-06 21:10:57 +000034 """
35 name = dbmodels.CharField(maxlength=255, unique=True)
36 kernel_config = dbmodels.CharField(maxlength=255, blank=True)
37 platform = dbmodels.BooleanField(default=False)
38 invalid = dbmodels.BooleanField(default=False,
39 editable=settings.FULL_ADMIN)
showardb1e51872008-10-07 11:08:18 +000040 only_if_needed = dbmodels.BooleanField(default=False)
mblighe8819cd2008-02-15 16:48:40 +000041
jadmanski0afbb632008-06-06 21:10:57 +000042 name_field = 'name'
43 objects = model_logic.ExtendedManager()
44 valid_objects = model_logic.ValidObjectsManager()
mbligh5244cbb2008-04-24 20:39:52 +000045
jadmanski0afbb632008-06-06 21:10:57 +000046 def clean_object(self):
47 self.host_set.clear()
mblighe8819cd2008-02-15 16:48:40 +000048
49
jadmanski0afbb632008-06-06 21:10:57 +000050 def enqueue_job(self, job):
51 'Enqueue a job on any host of this label.'
52 queue_entry = HostQueueEntry(meta_host=self, job=job,
53 status=Job.Status.QUEUED,
54 priority=job.priority)
55 queue_entry.save()
mblighe8819cd2008-02-15 16:48:40 +000056
57
jadmanski0afbb632008-06-06 21:10:57 +000058 class Meta:
59 db_table = 'labels'
mblighe8819cd2008-02-15 16:48:40 +000060
jadmanski0afbb632008-06-06 21:10:57 +000061 class Admin:
62 list_display = ('name', 'kernel_config')
63 # see Host.Admin
64 manager = model_logic.ValidObjectsManager()
mblighe8819cd2008-02-15 16:48:40 +000065
jadmanski0afbb632008-06-06 21:10:57 +000066 def __str__(self):
67 return self.name
mblighe8819cd2008-02-15 16:48:40 +000068
69
showardfb2a7fa2008-07-17 17:04:12 +000070class User(dbmodels.Model, model_logic.ModelExtensions):
71 """\
72 Required:
73 login :user login name
74
75 Optional:
76 access_level: 0=User (default), 1=Admin, 100=Root
77 """
78 ACCESS_ROOT = 100
79 ACCESS_ADMIN = 1
80 ACCESS_USER = 0
81
82 login = dbmodels.CharField(maxlength=255, unique=True)
83 access_level = dbmodels.IntegerField(default=ACCESS_USER, blank=True)
84
showard0fc38302008-10-23 00:44:07 +000085 # user preferences
86 reboot_before = dbmodels.SmallIntegerField(choices=RebootBefore.choices(),
87 blank=True,
88 default=DEFAULT_REBOOT_BEFORE)
89 reboot_after = dbmodels.SmallIntegerField(choices=RebootAfter.choices(),
90 blank=True,
91 default=DEFAULT_REBOOT_AFTER)
showard97db5ba2008-11-12 18:18:02 +000092 show_experimental = dbmodels.BooleanField(default=False)
showard0fc38302008-10-23 00:44:07 +000093
showardfb2a7fa2008-07-17 17:04:12 +000094 name_field = 'login'
95 objects = model_logic.ExtendedManager()
96
97
98 def save(self):
99 # is this a new object being saved for the first time?
100 first_time = (self.id is None)
101 user = thread_local.get_user()
showard0fc38302008-10-23 00:44:07 +0000102 if user and not user.is_superuser() and user.login != self.login:
103 raise AclAccessViolation("You cannot modify user " + self.login)
showardfb2a7fa2008-07-17 17:04:12 +0000104 super(User, self).save()
105 if first_time:
106 everyone = AclGroup.objects.get(name='Everyone')
107 everyone.users.add(self)
108
109
110 def is_superuser(self):
111 return self.access_level >= self.ACCESS_ROOT
112
113
114 class Meta:
115 db_table = 'users'
116
117 class Admin:
118 list_display = ('login', 'access_level')
119 search_fields = ('login',)
120
121 def __str__(self):
122 return self.login
123
124
showard7c785282008-05-29 19:45:12 +0000125class Host(model_logic.ModelWithInvalid, dbmodels.Model):
jadmanski0afbb632008-06-06 21:10:57 +0000126 """\
127 Required:
128 hostname
mblighe8819cd2008-02-15 16:48:40 +0000129
jadmanski0afbb632008-06-06 21:10:57 +0000130 optional:
showard21baa452008-10-21 00:08:39 +0000131 locked: if true, host is locked and will not be queued
mblighe8819cd2008-02-15 16:48:40 +0000132
jadmanski0afbb632008-06-06 21:10:57 +0000133 Internal:
134 synch_id: currently unused
135 status: string describing status of host
showard21baa452008-10-21 00:08:39 +0000136 invalid: true if the host has been deleted
137 protection: indicates what can be done to this host during repair
138 locked_by: user that locked the host, or null if the host is unlocked
139 lock_time: DateTime at which the host was locked
140 dirty: true if the host has been used without being rebooted
jadmanski0afbb632008-06-06 21:10:57 +0000141 """
142 Status = enum.Enum('Verifying', 'Running', 'Ready', 'Repairing',
showard45ae8192008-11-05 19:32:53 +0000143 'Repair Failed', 'Dead', 'Cleaning', 'Pending',
jadmanski0afbb632008-06-06 21:10:57 +0000144 string_values=True)
mblighe8819cd2008-02-15 16:48:40 +0000145
jadmanski0afbb632008-06-06 21:10:57 +0000146 hostname = dbmodels.CharField(maxlength=255, unique=True)
147 labels = dbmodels.ManyToManyField(Label, blank=True,
148 filter_interface=dbmodels.HORIZONTAL)
149 locked = dbmodels.BooleanField(default=False)
150 synch_id = dbmodels.IntegerField(blank=True, null=True,
151 editable=settings.FULL_ADMIN)
152 status = dbmodels.CharField(maxlength=255, default=Status.READY,
153 choices=Status.choices(),
154 editable=settings.FULL_ADMIN)
155 invalid = dbmodels.BooleanField(default=False,
156 editable=settings.FULL_ADMIN)
showardd5afc2f2008-08-12 17:19:44 +0000157 protection = dbmodels.SmallIntegerField(null=False, blank=True,
showarddf062562008-07-03 19:56:37 +0000158 choices=host_protections.choices,
159 default=host_protections.default)
showardfb2a7fa2008-07-17 17:04:12 +0000160 locked_by = dbmodels.ForeignKey(User, null=True, blank=True, editable=False)
161 lock_time = dbmodels.DateTimeField(null=True, blank=True, editable=False)
showard21baa452008-10-21 00:08:39 +0000162 dirty = dbmodels.BooleanField(default=True, editable=settings.FULL_ADMIN)
mblighe8819cd2008-02-15 16:48:40 +0000163
jadmanski0afbb632008-06-06 21:10:57 +0000164 name_field = 'hostname'
165 objects = model_logic.ExtendedManager()
166 valid_objects = model_logic.ValidObjectsManager()
mbligh5244cbb2008-04-24 20:39:52 +0000167
showard2bab8f42008-11-12 18:15:22 +0000168
169 def __init__(self, *args, **kwargs):
170 super(Host, self).__init__(*args, **kwargs)
171 self._record_attributes(['status'])
172
173
showardb8471e32008-07-03 19:51:08 +0000174 @staticmethod
175 def create_one_time_host(hostname):
176 query = Host.objects.filter(hostname=hostname)
177 if query.count() == 0:
178 host = Host(hostname=hostname, invalid=True)
showarda8411af2008-08-07 22:35:58 +0000179 host.do_validate()
showardb8471e32008-07-03 19:51:08 +0000180 else:
181 host = query[0]
182 if not host.invalid:
183 raise model_logic.ValidationError({
184 'hostname' : '%s already exists!' % hostname
185 })
186 host.clean_object()
showarde65d2af2008-07-30 23:34:21 +0000187 AclGroup.objects.get(name='Everyone').hosts.add(host)
showardb8471e32008-07-03 19:51:08 +0000188 host.status = Host.Status.READY
showard1ab512b2008-07-30 23:39:04 +0000189 host.protection = host_protections.Protection.DO_NOT_REPAIR
showardb8471e32008-07-03 19:51:08 +0000190 host.save()
191 return host
mbligh5244cbb2008-04-24 20:39:52 +0000192
jadmanski0afbb632008-06-06 21:10:57 +0000193 def clean_object(self):
194 self.aclgroup_set.clear()
195 self.labels.clear()
mblighe8819cd2008-02-15 16:48:40 +0000196
197
jadmanski0afbb632008-06-06 21:10:57 +0000198 def save(self):
199 # extra spaces in the hostname can be a sneaky source of errors
200 self.hostname = self.hostname.strip()
201 # is this a new object being saved for the first time?
202 first_time = (self.id is None)
showard3dd47c22008-07-10 00:41:36 +0000203 if not first_time:
204 AclGroup.check_for_acl_violation_hosts([self])
showardfb2a7fa2008-07-17 17:04:12 +0000205 if self.locked and not self.locked_by:
206 self.locked_by = thread_local.get_user()
207 self.lock_time = datetime.now()
showard21baa452008-10-21 00:08:39 +0000208 self.dirty = True
showardfb2a7fa2008-07-17 17:04:12 +0000209 elif not self.locked and self.locked_by:
210 self.locked_by = None
211 self.lock_time = None
jadmanski0afbb632008-06-06 21:10:57 +0000212 super(Host, self).save()
213 if first_time:
214 everyone = AclGroup.objects.get(name='Everyone')
215 everyone.hosts.add(self)
showard2bab8f42008-11-12 18:15:22 +0000216 self._check_for_updated_attributes()
217
mblighe8819cd2008-02-15 16:48:40 +0000218
showardb8471e32008-07-03 19:51:08 +0000219 def delete(self):
showard3dd47c22008-07-10 00:41:36 +0000220 AclGroup.check_for_acl_violation_hosts([self])
showardb8471e32008-07-03 19:51:08 +0000221 for queue_entry in self.hostqueueentry_set.all():
222 queue_entry.deleted = True
showard9a1f2e12008-10-02 11:14:29 +0000223 queue_entry.abort(thread_local.get_user())
showardb8471e32008-07-03 19:51:08 +0000224 super(Host, self).delete()
225
mblighe8819cd2008-02-15 16:48:40 +0000226
showard2bab8f42008-11-12 18:15:22 +0000227 def on_attribute_changed(self, attribute, old_value):
228 assert attribute == 'status'
229 logger.debug(self.hostname + ' -> ' + self.status)
230
231
jadmanski0afbb632008-06-06 21:10:57 +0000232 def enqueue_job(self, job):
233 ' Enqueue a job on this host.'
234 queue_entry = HostQueueEntry(host=self, job=job,
235 status=Job.Status.QUEUED,
236 priority=job.priority)
237 # allow recovery of dead hosts from the frontend
238 if not self.active_queue_entry() and self.is_dead():
239 self.status = Host.Status.READY
240 self.save()
241 queue_entry.save()
mblighe8819cd2008-02-15 16:48:40 +0000242
showard08f981b2008-06-24 21:59:03 +0000243 block = IneligibleHostQueue(job=job, host=self)
244 block.save()
245
mblighe8819cd2008-02-15 16:48:40 +0000246
jadmanski0afbb632008-06-06 21:10:57 +0000247 def platform(self):
248 # TODO(showard): slighly hacky?
249 platforms = self.labels.filter(platform=True)
250 if len(platforms) == 0:
251 return None
252 return platforms[0]
253 platform.short_description = 'Platform'
mblighe8819cd2008-02-15 16:48:40 +0000254
255
jadmanski0afbb632008-06-06 21:10:57 +0000256 def is_dead(self):
257 return self.status == Host.Status.REPAIR_FAILED
mbligh3cab4a72008-03-05 23:19:09 +0000258
259
jadmanski0afbb632008-06-06 21:10:57 +0000260 def active_queue_entry(self):
261 active = list(self.hostqueueentry_set.filter(active=True))
262 if not active:
263 return None
264 assert len(active) == 1, ('More than one active entry for '
265 'host ' + self.hostname)
266 return active[0]
mblighe8819cd2008-02-15 16:48:40 +0000267
268
jadmanski0afbb632008-06-06 21:10:57 +0000269 class Meta:
270 db_table = 'hosts'
mblighe8819cd2008-02-15 16:48:40 +0000271
jadmanski0afbb632008-06-06 21:10:57 +0000272 class Admin:
273 # TODO(showard) - showing platform requires a SQL query for
274 # each row (since labels are many-to-many) - should we remove
275 # it?
276 list_display = ('hostname', 'platform', 'locked', 'status')
showarddf062562008-07-03 19:56:37 +0000277 list_filter = ('labels', 'locked', 'protection')
jadmanski0afbb632008-06-06 21:10:57 +0000278 search_fields = ('hostname', 'status')
279 # undocumented Django feature - if you set manager here, the
280 # admin code will use it, otherwise it'll use a default Manager
281 manager = model_logic.ValidObjectsManager()
mblighe8819cd2008-02-15 16:48:40 +0000282
jadmanski0afbb632008-06-06 21:10:57 +0000283 def __str__(self):
284 return self.hostname
mblighe8819cd2008-02-15 16:48:40 +0000285
286
showard7c785282008-05-29 19:45:12 +0000287class Test(dbmodels.Model, model_logic.ModelExtensions):
jadmanski0afbb632008-06-06 21:10:57 +0000288 """\
289 Required:
showard909c7a62008-07-15 21:52:38 +0000290 author: author name
291 description: description of the test
jadmanski0afbb632008-06-06 21:10:57 +0000292 name: test name
showard909c7a62008-07-15 21:52:38 +0000293 time: short, medium, long
294 test_class: This describes the class for your the test belongs in.
295 test_category: This describes the category for your tests
jadmanski0afbb632008-06-06 21:10:57 +0000296 test_type: Client or Server
297 path: path to pass to run_test()
showard909c7a62008-07-15 21:52:38 +0000298 sync_count: is a number >=1 (1 being the default). If it's 1, then it's an
299 async job. If it's >1 it's sync job for that number of machines
showard2bab8f42008-11-12 18:15:22 +0000300 i.e. if sync_count = 2 it is a sync job that requires two
301 machines.
jadmanski0afbb632008-06-06 21:10:57 +0000302 Optional:
showard909c7a62008-07-15 21:52:38 +0000303 dependencies: What the test requires to run. Comma deliminated list
showard989f25d2008-10-01 11:38:11 +0000304 dependency_labels: many-to-many relationship with labels corresponding to
305 test dependencies.
showard909c7a62008-07-15 21:52:38 +0000306 experimental: If this is set to True production servers will ignore the test
307 run_verify: Whether or not the scheduler should run the verify stage
jadmanski0afbb632008-06-06 21:10:57 +0000308 """
showard909c7a62008-07-15 21:52:38 +0000309 TestTime = enum.Enum('SHORT', 'MEDIUM', 'LONG', start_value=1)
jadmanski0afbb632008-06-06 21:10:57 +0000310 # TODO(showard) - this should be merged with Job.ControlType (but right
311 # now they use opposite values)
312 Types = enum.Enum('Client', 'Server', start_value=1)
mblighe8819cd2008-02-15 16:48:40 +0000313
jadmanski0afbb632008-06-06 21:10:57 +0000314 name = dbmodels.CharField(maxlength=255, unique=True)
showard909c7a62008-07-15 21:52:38 +0000315 author = dbmodels.CharField(maxlength=255)
316 test_class = dbmodels.CharField(maxlength=255)
317 test_category = dbmodels.CharField(maxlength=255)
318 dependencies = dbmodels.CharField(maxlength=255, blank=True)
jadmanski0afbb632008-06-06 21:10:57 +0000319 description = dbmodels.TextField(blank=True)
showard909c7a62008-07-15 21:52:38 +0000320 experimental = dbmodels.BooleanField(default=True)
321 run_verify = dbmodels.BooleanField(default=True)
322 test_time = dbmodels.SmallIntegerField(choices=TestTime.choices(),
323 default=TestTime.MEDIUM)
jadmanski0afbb632008-06-06 21:10:57 +0000324 test_type = dbmodels.SmallIntegerField(choices=Types.choices())
showard909c7a62008-07-15 21:52:38 +0000325 sync_count = dbmodels.IntegerField(default=1)
showard909c7a62008-07-15 21:52:38 +0000326 path = dbmodels.CharField(maxlength=255, unique=True)
showard989f25d2008-10-01 11:38:11 +0000327 dependency_labels = dbmodels.ManyToManyField(
328 Label, blank=True, filter_interface=dbmodels.HORIZONTAL)
mblighe8819cd2008-02-15 16:48:40 +0000329
jadmanski0afbb632008-06-06 21:10:57 +0000330 name_field = 'name'
331 objects = model_logic.ExtendedManager()
mblighe8819cd2008-02-15 16:48:40 +0000332
333
jadmanski0afbb632008-06-06 21:10:57 +0000334 class Meta:
335 db_table = 'autotests'
mblighe8819cd2008-02-15 16:48:40 +0000336
jadmanski0afbb632008-06-06 21:10:57 +0000337 class Admin:
338 fields = (
339 (None, {'fields' :
showard909c7a62008-07-15 21:52:38 +0000340 ('name', 'author', 'test_category', 'test_class',
showard2bab8f42008-11-12 18:15:22 +0000341 'test_time', 'sync_count', 'test_type', 'sync_count',
showard909c7a62008-07-15 21:52:38 +0000342 'path', 'dependencies', 'experimental', 'run_verify',
343 'description')}),
jadmanski0afbb632008-06-06 21:10:57 +0000344 )
showard2bab8f42008-11-12 18:15:22 +0000345 list_display = ('name', 'test_type', 'description', 'sync_count')
jadmanski0afbb632008-06-06 21:10:57 +0000346 search_fields = ('name',)
mblighe8819cd2008-02-15 16:48:40 +0000347
jadmanski0afbb632008-06-06 21:10:57 +0000348 def __str__(self):
349 return self.name
mblighe8819cd2008-02-15 16:48:40 +0000350
351
showard2b9a88b2008-06-13 20:55:03 +0000352class Profiler(dbmodels.Model, model_logic.ModelExtensions):
353 """\
354 Required:
355 name: profiler name
356 test_type: Client or Server
357
358 Optional:
359 description: arbirary text description
360 """
361 name = dbmodels.CharField(maxlength=255, unique=True)
362 description = dbmodels.TextField(blank=True)
363
364 name_field = 'name'
365 objects = model_logic.ExtendedManager()
366
367
368 class Meta:
369 db_table = 'profilers'
370
371 class Admin:
372 list_display = ('name', 'description')
373 search_fields = ('name',)
374
375 def __str__(self):
376 return self.name
377
378
showard7c785282008-05-29 19:45:12 +0000379class AclGroup(dbmodels.Model, model_logic.ModelExtensions):
jadmanski0afbb632008-06-06 21:10:57 +0000380 """\
381 Required:
382 name: name of ACL group
mblighe8819cd2008-02-15 16:48:40 +0000383
jadmanski0afbb632008-06-06 21:10:57 +0000384 Optional:
385 description: arbitrary description of group
386 """
387 name = dbmodels.CharField(maxlength=255, unique=True)
388 description = dbmodels.CharField(maxlength=255, blank=True)
showard04f2cd82008-07-25 20:53:31 +0000389 users = dbmodels.ManyToManyField(User, blank=True,
jadmanski0afbb632008-06-06 21:10:57 +0000390 filter_interface=dbmodels.HORIZONTAL)
391 hosts = dbmodels.ManyToManyField(Host,
392 filter_interface=dbmodels.HORIZONTAL)
mblighe8819cd2008-02-15 16:48:40 +0000393
jadmanski0afbb632008-06-06 21:10:57 +0000394 name_field = 'name'
395 objects = model_logic.ExtendedManager()
showardeb3be4d2008-04-21 20:59:26 +0000396
showard08f981b2008-06-24 21:59:03 +0000397 @staticmethod
showard3dd47c22008-07-10 00:41:36 +0000398 def check_for_acl_violation_hosts(hosts):
399 user = thread_local.get_user()
400 if user.is_superuser():
showard9dbdcda2008-10-14 17:34:36 +0000401 return
showard3dd47c22008-07-10 00:41:36 +0000402 accessible_host_ids = set(
403 host.id for host in Host.objects.filter(acl_group__users=user))
404 for host in hosts:
405 # Check if the user has access to this host,
406 # but only if it is not a metahost
407 if (isinstance(host, Host)
408 and int(host.id) not in accessible_host_ids):
409 raise AclAccessViolation("You do not have access to %s"
410 % str(host))
411
showard9dbdcda2008-10-14 17:34:36 +0000412
413 @staticmethod
showarddc817512008-11-12 18:16:41 +0000414 def check_abort_permissions(queue_entries):
415 """
416 look for queue entries that aren't abortable, meaning
417 * the job isn't owned by this user, and
418 * the machine isn't ACL-accessible, or
419 * the machine is in the "Everyone" ACL
420 """
showard9dbdcda2008-10-14 17:34:36 +0000421 user = thread_local.get_user()
422 if user.is_superuser():
423 return
showarddc817512008-11-12 18:16:41 +0000424 not_owned = queue_entries.exclude(job__owner=user.login)
425 # I do this using ID sets instead of just Django filters because
426 # filtering on M2M fields is broken in Django 0.96. It's better in 1.0.
427 accessible_ids = set(
428 entry.id for entry
429 in not_owned.filter(host__acl_group__users__login=user.login))
430 public_ids = set(entry.id for entry
431 in not_owned.filter(host__acl_group__name='Everyone'))
432 cannot_abort = [entry for entry in not_owned.select_related()
433 if entry.id not in accessible_ids
434 or entry.id in public_ids]
435 if len(cannot_abort) == 0:
436 return
437 entry_names = ', '.join('%s-%s/%s' % (entry.job.id, entry.job.owner,
showard3f15eed2008-11-14 22:40:48 +0000438 entry.host_or_metahost_name())
showarddc817512008-11-12 18:16:41 +0000439 for entry in cannot_abort)
440 raise AclAccessViolation('You cannot abort the following job entries: '
441 + entry_names)
showard9dbdcda2008-10-14 17:34:36 +0000442
443
showard3dd47c22008-07-10 00:41:36 +0000444 def check_for_acl_violation_acl_group(self):
445 user = thread_local.get_user()
446 if user.is_superuser():
447 return None
448 if not user in self.users.all():
449 raise AclAccessViolation("You do not have access to %s"
450 % self.name)
451
452 @staticmethod
showard08f981b2008-06-24 21:59:03 +0000453 def on_host_membership_change():
454 everyone = AclGroup.objects.get(name='Everyone')
455
showard3dd47c22008-07-10 00:41:36 +0000456 # find hosts that aren't in any ACL group and add them to Everyone
showard08f981b2008-06-24 21:59:03 +0000457 # TODO(showard): this is a bit of a hack, since the fact that this query
458 # works is kind of a coincidence of Django internals. This trick
459 # doesn't work in general (on all foreign key relationships). I'll
460 # replace it with a better technique when the need arises.
461 orphaned_hosts = Host.valid_objects.filter(acl_group__id__isnull=True)
462 everyone.hosts.add(*orphaned_hosts.distinct())
463
464 # find hosts in both Everyone and another ACL group, and remove them
465 # from Everyone
466 hosts_in_everyone = Host.valid_objects.filter_custom_join(
467 '_everyone', acl_group__name='Everyone')
468 acled_hosts = hosts_in_everyone.exclude(acl_group__name='Everyone')
469 everyone.hosts.remove(*acled_hosts.distinct())
470
471
472 def delete(self):
showard3dd47c22008-07-10 00:41:36 +0000473 if (self.name == 'Everyone'):
474 raise AclAccessViolation("You cannot delete 'Everyone'!")
475 self.check_for_acl_violation_acl_group()
showard08f981b2008-06-24 21:59:03 +0000476 super(AclGroup, self).delete()
477 self.on_host_membership_change()
478
479
showard04f2cd82008-07-25 20:53:31 +0000480 def add_current_user_if_empty(self):
481 if not self.users.count():
482 self.users.add(thread_local.get_user())
483
484
showard08f981b2008-06-24 21:59:03 +0000485 # if you have a model attribute called "Manipulator", Django will
486 # automatically insert it into the beginning of the superclass list
487 # for the model's manipulators
488 class Manipulator(object):
489 """
490 Custom manipulator to get notification when ACLs are changed through
491 the admin interface.
492 """
493 def save(self, new_data):
showard3dd47c22008-07-10 00:41:36 +0000494 user = thread_local.get_user()
showard31c570e2008-07-23 19:29:17 +0000495 if hasattr(self, 'original_object'):
496 if (not user.is_superuser()
497 and self.original_object.name == 'Everyone'):
498 raise AclAccessViolation("You cannot modify 'Everyone'!")
499 self.original_object.check_for_acl_violation_acl_group()
showard08f981b2008-06-24 21:59:03 +0000500 obj = super(AclGroup.Manipulator, self).save(new_data)
showard04f2cd82008-07-25 20:53:31 +0000501 if not hasattr(self, 'original_object'):
502 obj.users.add(thread_local.get_user())
503 obj.add_current_user_if_empty()
showard08f981b2008-06-24 21:59:03 +0000504 obj.on_host_membership_change()
505 return obj
showardeb3be4d2008-04-21 20:59:26 +0000506
jadmanski0afbb632008-06-06 21:10:57 +0000507 class Meta:
508 db_table = 'acl_groups'
mblighe8819cd2008-02-15 16:48:40 +0000509
jadmanski0afbb632008-06-06 21:10:57 +0000510 class Admin:
511 list_display = ('name', 'description')
512 search_fields = ('name',)
mblighe8819cd2008-02-15 16:48:40 +0000513
jadmanski0afbb632008-06-06 21:10:57 +0000514 def __str__(self):
515 return self.name
mblighe8819cd2008-02-15 16:48:40 +0000516
517# hack to make the column name in the many-to-many DB tables match the one
518# generated by ruby
519AclGroup._meta.object_name = 'acl_group'
520
521
showard7c785282008-05-29 19:45:12 +0000522class JobManager(model_logic.ExtendedManager):
jadmanski0afbb632008-06-06 21:10:57 +0000523 'Custom manager to provide efficient status counts querying.'
524 def get_status_counts(self, job_ids):
525 """\
526 Returns a dictionary mapping the given job IDs to their status
527 count dictionaries.
528 """
529 if not job_ids:
530 return {}
531 id_list = '(%s)' % ','.join(str(job_id) for job_id in job_ids)
532 cursor = connection.cursor()
533 cursor.execute("""
534 SELECT job_id, status, COUNT(*)
535 FROM host_queue_entries
536 WHERE job_id IN %s
537 GROUP BY job_id, status
538 """ % id_list)
539 all_job_counts = {}
540 for job_id in job_ids:
541 all_job_counts[job_id] = {}
542 for job_id, status, count in cursor.fetchall():
543 all_job_counts[job_id][status] = count
544 return all_job_counts
mblighe8819cd2008-02-15 16:48:40 +0000545
546
showard989f25d2008-10-01 11:38:11 +0000547 def populate_dependencies(self, jobs):
showard9a1f2e12008-10-02 11:14:29 +0000548 if not jobs:
549 return
showard989f25d2008-10-01 11:38:11 +0000550 job_ids = ','.join(str(job['id']) for job in jobs)
551 cursor = connection.cursor()
552 cursor.execute("""
showard56e93772008-10-06 10:06:22 +0000553 SELECT jobs.id, labels.name
showard989f25d2008-10-01 11:38:11 +0000554 FROM jobs
555 INNER JOIN jobs_dependency_labels
556 ON jobs.id = jobs_dependency_labels.job_id
557 INNER JOIN labels ON jobs_dependency_labels.label_id = labels.id
558 WHERE jobs.id IN (%s)
showard989f25d2008-10-01 11:38:11 +0000559 """ % job_ids)
showard56e93772008-10-06 10:06:22 +0000560 job_dependencies = {}
561 for job_id, dependency in cursor.fetchall():
562 job_dependencies.setdefault(job_id, []).append(dependency)
showard989f25d2008-10-01 11:38:11 +0000563 for job in jobs:
showard56e93772008-10-06 10:06:22 +0000564 dependencies = ','.join(job_dependencies.get(job['id'], []))
565 job['dependencies'] = dependencies
showard989f25d2008-10-01 11:38:11 +0000566
567
showard7c785282008-05-29 19:45:12 +0000568class Job(dbmodels.Model, model_logic.ModelExtensions):
jadmanski0afbb632008-06-06 21:10:57 +0000569 """\
570 owner: username of job owner
571 name: job name (does not have to be unique)
572 priority: Low, Medium, High, Urgent (or 0-3)
573 control_file: contents of control file
574 control_type: Client or Server
575 created_on: date of job creation
576 submitted_on: date of job submission
showard2bab8f42008-11-12 18:15:22 +0000577 synch_count: how many hosts should be used per autoserv execution
showard909c7a62008-07-15 21:52:38 +0000578 run_verify: Whether or not to run the verify phase
showard3bb499f2008-07-03 19:42:20 +0000579 timeout: hours until job times out
showard542e8402008-09-19 20:16:18 +0000580 email_list: list of people to email on completion delimited by any of:
581 white space, ',', ':', ';'
showard989f25d2008-10-01 11:38:11 +0000582 dependency_labels: many-to-many relationship with labels corresponding to
583 job dependencies
showard21baa452008-10-21 00:08:39 +0000584 reboot_before: Never, If dirty, or Always
585 reboot_after: Never, If all tests passed, or Always
jadmanski0afbb632008-06-06 21:10:57 +0000586 """
showardb1e51872008-10-07 11:08:18 +0000587 DEFAULT_TIMEOUT = global_config.global_config.get_config_value(
588 'AUTOTEST_WEB', 'job_timeout_default', default=240)
589
jadmanski0afbb632008-06-06 21:10:57 +0000590 Priority = enum.Enum('Low', 'Medium', 'High', 'Urgent')
591 ControlType = enum.Enum('Server', 'Client', start_value=1)
592 Status = enum.Enum('Created', 'Queued', 'Pending', 'Running',
593 'Completed', 'Abort', 'Aborting', 'Aborted',
showardd823b362008-07-24 16:35:46 +0000594 'Failed', 'Starting', string_values=True)
mblighe8819cd2008-02-15 16:48:40 +0000595
jadmanski0afbb632008-06-06 21:10:57 +0000596 owner = dbmodels.CharField(maxlength=255)
597 name = dbmodels.CharField(maxlength=255)
598 priority = dbmodels.SmallIntegerField(choices=Priority.choices(),
599 blank=True, # to allow 0
600 default=Priority.MEDIUM)
601 control_file = dbmodels.TextField()
602 control_type = dbmodels.SmallIntegerField(choices=ControlType.choices(),
showardb1e51872008-10-07 11:08:18 +0000603 blank=True, # to allow 0
604 default=ControlType.CLIENT)
showard68c7aa02008-10-09 16:49:11 +0000605 created_on = dbmodels.DateTimeField()
showard2bab8f42008-11-12 18:15:22 +0000606 synch_count = dbmodels.IntegerField(null=True, default=1)
showardb1e51872008-10-07 11:08:18 +0000607 timeout = dbmodels.IntegerField(default=DEFAULT_TIMEOUT)
showard9976ce92008-10-15 20:28:13 +0000608 run_verify = dbmodels.BooleanField(default=True)
showard542e8402008-09-19 20:16:18 +0000609 email_list = dbmodels.CharField(maxlength=250, blank=True)
showard989f25d2008-10-01 11:38:11 +0000610 dependency_labels = dbmodels.ManyToManyField(
611 Label, blank=True, filter_interface=dbmodels.HORIZONTAL)
showard21baa452008-10-21 00:08:39 +0000612 reboot_before = dbmodels.SmallIntegerField(choices=RebootBefore.choices(),
613 blank=True,
showard0fc38302008-10-23 00:44:07 +0000614 default=DEFAULT_REBOOT_BEFORE)
showard21baa452008-10-21 00:08:39 +0000615 reboot_after = dbmodels.SmallIntegerField(choices=RebootAfter.choices(),
616 blank=True,
showard0fc38302008-10-23 00:44:07 +0000617 default=DEFAULT_REBOOT_AFTER)
mblighe8819cd2008-02-15 16:48:40 +0000618
619
jadmanski0afbb632008-06-06 21:10:57 +0000620 # custom manager
621 objects = JobManager()
mblighe8819cd2008-02-15 16:48:40 +0000622
623
jadmanski0afbb632008-06-06 21:10:57 +0000624 def is_server_job(self):
625 return self.control_type == self.ControlType.SERVER
mblighe8819cd2008-02-15 16:48:40 +0000626
627
jadmanski0afbb632008-06-06 21:10:57 +0000628 @classmethod
629 def create(cls, owner, name, priority, control_file, control_type,
showard2bab8f42008-11-12 18:15:22 +0000630 hosts, synch_count, timeout, run_verify, email_list,
showard21baa452008-10-21 00:08:39 +0000631 dependencies, reboot_before, reboot_after):
jadmanski0afbb632008-06-06 21:10:57 +0000632 """\
633 Creates a job by taking some information (the listed args)
634 and filling in the rest of the necessary information.
635 """
showard3dd47c22008-07-10 00:41:36 +0000636 AclGroup.check_for_acl_violation_hosts(hosts)
jadmanski0afbb632008-06-06 21:10:57 +0000637 job = cls.add_object(
638 owner=owner, name=name, priority=priority,
639 control_file=control_file, control_type=control_type,
showard2bab8f42008-11-12 18:15:22 +0000640 synch_count=synch_count, timeout=timeout,
showard68c7aa02008-10-09 16:49:11 +0000641 run_verify=run_verify, email_list=email_list,
showard21baa452008-10-21 00:08:39 +0000642 reboot_before=reboot_before, reboot_after=reboot_after,
showard68c7aa02008-10-09 16:49:11 +0000643 created_on=datetime.now())
mblighe8819cd2008-02-15 16:48:40 +0000644
showard2bab8f42008-11-12 18:15:22 +0000645 if job.synch_count > len(hosts):
646 raise model_logic.ValidationError(
647 {'hosts': 'only %d hosts provided for job with synch_count = %d'
648 % (len(hosts), job.synch_count)})
jadmanski0afbb632008-06-06 21:10:57 +0000649 job.save()
showard989f25d2008-10-01 11:38:11 +0000650 job.dependency_labels = dependencies
jadmanski0afbb632008-06-06 21:10:57 +0000651 return job
mblighe8819cd2008-02-15 16:48:40 +0000652
653
jadmanski0afbb632008-06-06 21:10:57 +0000654 def queue(self, hosts):
655 'Enqueue a job on the given hosts.'
656 for host in hosts:
657 host.enqueue_job(self)
mblighe8819cd2008-02-15 16:48:40 +0000658
659
jadmanski0afbb632008-06-06 21:10:57 +0000660 def user(self):
661 try:
662 return User.objects.get(login=self.owner)
663 except self.DoesNotExist:
664 return None
mblighe8819cd2008-02-15 16:48:40 +0000665
666
showard98863972008-10-29 21:14:56 +0000667 def abort(self, aborted_by):
668 for queue_entry in self.hostqueueentry_set.all():
669 queue_entry.abort(aborted_by)
670
671
jadmanski0afbb632008-06-06 21:10:57 +0000672 class Meta:
673 db_table = 'jobs'
mblighe8819cd2008-02-15 16:48:40 +0000674
jadmanski0afbb632008-06-06 21:10:57 +0000675 if settings.FULL_ADMIN:
676 class Admin:
677 list_display = ('id', 'owner', 'name', 'control_type')
mblighe8819cd2008-02-15 16:48:40 +0000678
jadmanski0afbb632008-06-06 21:10:57 +0000679 def __str__(self):
680 return '%s (%s-%s)' % (self.name, self.id, self.owner)
mblighe8819cd2008-02-15 16:48:40 +0000681
682
showard7c785282008-05-29 19:45:12 +0000683class IneligibleHostQueue(dbmodels.Model, model_logic.ModelExtensions):
jadmanski0afbb632008-06-06 21:10:57 +0000684 job = dbmodels.ForeignKey(Job)
685 host = dbmodels.ForeignKey(Host)
mblighe8819cd2008-02-15 16:48:40 +0000686
jadmanski0afbb632008-06-06 21:10:57 +0000687 objects = model_logic.ExtendedManager()
showardeb3be4d2008-04-21 20:59:26 +0000688
jadmanski0afbb632008-06-06 21:10:57 +0000689 class Meta:
690 db_table = 'ineligible_host_queues'
mblighe8819cd2008-02-15 16:48:40 +0000691
jadmanski0afbb632008-06-06 21:10:57 +0000692 if settings.FULL_ADMIN:
693 class Admin:
694 list_display = ('id', 'job', 'host')
mblighe8819cd2008-02-15 16:48:40 +0000695
696
showard7c785282008-05-29 19:45:12 +0000697class HostQueueEntry(dbmodels.Model, model_logic.ModelExtensions):
showarda3ab0d52008-11-03 19:03:47 +0000698 Status = enum.Enum('Queued', 'Starting', 'Verifying', 'Pending', 'Running',
showard97aed502008-11-04 02:01:24 +0000699 'Parsing', 'Abort', 'Aborting', 'Aborted', 'Completed',
700 'Failed', 'Stopped', string_values=True)
showarda3ab0d52008-11-03 19:03:47 +0000701 ABORT_STATUSES = (Status.ABORT, Status.ABORTING, Status.ABORTED)
showard2bab8f42008-11-12 18:15:22 +0000702 ACTIVE_STATUSES = (Status.STARTING, Status.VERIFYING, Status.PENDING,
showarda58594a2008-11-18 17:53:56 +0000703 Status.RUNNING, Status.ABORTING)
showard2bab8f42008-11-12 18:15:22 +0000704 COMPLETE_STATUSES = (Status.PARSING, Status.ABORTED, Status.COMPLETED,
705 Status.FAILED, Status.STOPPED)
showarda3ab0d52008-11-03 19:03:47 +0000706
jadmanski0afbb632008-06-06 21:10:57 +0000707 job = dbmodels.ForeignKey(Job)
708 host = dbmodels.ForeignKey(Host, blank=True, null=True)
709 priority = dbmodels.SmallIntegerField()
710 status = dbmodels.CharField(maxlength=255)
711 meta_host = dbmodels.ForeignKey(Label, blank=True, null=True,
712 db_column='meta_host')
713 active = dbmodels.BooleanField(default=False)
714 complete = dbmodels.BooleanField(default=False)
showardb8471e32008-07-03 19:51:08 +0000715 deleted = dbmodels.BooleanField(default=False)
showard2bab8f42008-11-12 18:15:22 +0000716 execution_subdir = dbmodels.CharField(maxlength=255, blank=True, default='')
mblighe8819cd2008-02-15 16:48:40 +0000717
jadmanski0afbb632008-06-06 21:10:57 +0000718 objects = model_logic.ExtendedManager()
showardeb3be4d2008-04-21 20:59:26 +0000719
mblighe8819cd2008-02-15 16:48:40 +0000720
showard2bab8f42008-11-12 18:15:22 +0000721 def __init__(self, *args, **kwargs):
722 super(HostQueueEntry, self).__init__(*args, **kwargs)
723 self._record_attributes(['status'])
724
725
726 def save(self):
727 self._set_active_and_complete()
728 super(HostQueueEntry, self).save()
729 self._check_for_updated_attributes()
730
731
showard3f15eed2008-11-14 22:40:48 +0000732 def host_or_metahost_name(self):
733 if self.host:
734 return self.host.hostname
735 else:
736 assert self.meta_host
737 return self.meta_host.name
738
739
showard2bab8f42008-11-12 18:15:22 +0000740 def _set_active_and_complete(self):
showarda58594a2008-11-18 17:53:56 +0000741 if self.status == self.Status.ABORT:
742 # must leave active flag unchanged so scheduler knows if entry was
743 # active before abort.
744 return
745 elif self.status in self.ACTIVE_STATUSES:
showard2bab8f42008-11-12 18:15:22 +0000746 self.active, self.complete = True, False
747 elif self.status in self.COMPLETE_STATUSES:
748 self.active, self.complete = False, True
749 else:
750 self.active, self.complete = False, False
751
752
753 def on_attribute_changed(self, attribute, old_value):
754 assert attribute == 'status'
755 logger.debug('%s/%d (%d) -> %s' % (self.host, self.job.id, self.id,
756 self.status))
757
758
jadmanski0afbb632008-06-06 21:10:57 +0000759 def is_meta_host_entry(self):
760 'True if this is a entry has a meta_host instead of a host.'
761 return self.host is None and self.meta_host is not None
mblighe8819cd2008-02-15 16:48:40 +0000762
showarda3ab0d52008-11-03 19:03:47 +0000763
showard4c119042008-09-29 19:16:18 +0000764 def log_abort(self, user):
showard98863972008-10-29 21:14:56 +0000765 if user is None:
766 # automatic system abort (i.e. job timeout)
767 return
showard4c119042008-09-29 19:16:18 +0000768 abort_log = AbortedHostQueueEntry(queue_entry=self, aborted_by=user)
769 abort_log.save()
770
showarda3ab0d52008-11-03 19:03:47 +0000771
showard4c119042008-09-29 19:16:18 +0000772 def abort(self, user):
showarda3ab0d52008-11-03 19:03:47 +0000773 # this isn't completely immune to race conditions since it's not atomic,
774 # but it should be safe given the scheduler's behavior.
775 if not self.complete and self.status not in self.ABORT_STATUSES:
showardb8471e32008-07-03 19:51:08 +0000776 self.status = Job.Status.ABORT
showard4c119042008-09-29 19:16:18 +0000777 self.log_abort(user)
showarda3ab0d52008-11-03 19:03:47 +0000778 self.save()
mblighe8819cd2008-02-15 16:48:40 +0000779
jadmanski0afbb632008-06-06 21:10:57 +0000780 class Meta:
781 db_table = 'host_queue_entries'
mblighe8819cd2008-02-15 16:48:40 +0000782
jadmanski0afbb632008-06-06 21:10:57 +0000783 if settings.FULL_ADMIN:
784 class Admin:
785 list_display = ('id', 'job', 'host', 'status',
786 'meta_host')
showard4c119042008-09-29 19:16:18 +0000787
788
789class AbortedHostQueueEntry(dbmodels.Model, model_logic.ModelExtensions):
790 queue_entry = dbmodels.OneToOneField(HostQueueEntry, primary_key=True)
791 aborted_by = dbmodels.ForeignKey(User)
showard68c7aa02008-10-09 16:49:11 +0000792 aborted_on = dbmodels.DateTimeField()
showard4c119042008-09-29 19:16:18 +0000793
794 objects = model_logic.ExtendedManager()
795
showard68c7aa02008-10-09 16:49:11 +0000796
797 def save(self):
798 self.aborted_on = datetime.now()
799 super(AbortedHostQueueEntry, self).save()
800
showard4c119042008-09-29 19:16:18 +0000801 class Meta:
802 db_table = 'aborted_host_queue_entries'