blob: a9c6bf0ced534a8d841b589d6855b60f021a4d89 [file] [log] [blame]
showard35444862008-08-07 22:35:30 +00001from django.db import models as dbmodels, connection
showard35444862008-08-07 22:35:30 +00002from autotest_lib.frontend.afe import model_logic, readonly_connection
3
showard8b0ea222009-12-23 19:23:03 +00004_quote_name = connection.ops.quote_name
5
showard35444862008-08-07 22:35:30 +00006class TempManager(model_logic.ExtendedManager):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -08007 """A Temp Manager."""
showard35444862008-08-07 22:35:30 +00008 _GROUP_COUNT_NAME = 'group_count'
9
10 def _get_key_unless_is_function(self, field):
11 if '(' in field:
12 return field
showard7c199df2008-10-03 10:17:15 +000013 return self.get_key_on_this_table(field)
showard35444862008-08-07 22:35:30 +000014
15
showardf2489522008-10-23 23:08:00 +000016 def _get_field_names(self, fields, extra_select_fields={}):
17 field_names = []
18 for field in fields:
19 if field in extra_select_fields:
showardd2b0c882009-10-19 18:34:11 +000020 field_names.append(extra_select_fields[field][0])
showardf2489522008-10-23 23:08:00 +000021 else:
22 field_names.append(self._get_key_unless_is_function(field))
23 return field_names
showard35444862008-08-07 22:35:30 +000024
25
showard8bfb5cb2009-10-07 20:49:15 +000026 def _get_group_query_sql(self, query, group_by):
Dale Curtis74a314b2011-06-23 14:55:46 -070027 compiler = query.query.get_compiler(using=query.db)
28 sql, params = compiler.as_sql()
29
showardd50ffb42008-09-04 02:47:45 +000030
31 # insert GROUP BY clause into query
showard8bfb5cb2009-10-07 20:49:15 +000032 group_fields = self._get_field_names(group_by, query.query.extra_select)
showard7c199df2008-10-03 10:17:15 +000033 group_by_clause = ' GROUP BY ' + ', '.join(group_fields)
showarda5288b42009-07-28 20:06:08 +000034 group_by_position = sql.rfind('ORDER BY')
showardd50ffb42008-09-04 02:47:45 +000035 if group_by_position == -1:
showarda5288b42009-07-28 20:06:08 +000036 group_by_position = len(sql)
37 sql = (sql[:group_by_position] +
38 group_by_clause + ' ' +
39 sql[group_by_position:])
showard35444862008-08-07 22:35:30 +000040
showarda5288b42009-07-28 20:06:08 +000041 return sql, params
showard35444862008-08-07 22:35:30 +000042
43
showard06b82fc2009-06-30 01:59:42 +000044 def _get_column_names(self, cursor):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -080045 """Gets the column names from the cursor description.
46
47 This method exists so that it can be mocked in the unit test for
48 sqlite3 compatibility.
49
showard06b82fc2009-06-30 01:59:42 +000050 """
51 return [column_info[0] for column_info in cursor.description]
52
53
showard8bfb5cb2009-10-07 20:49:15 +000054 def execute_group_query(self, query, group_by):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -080055 """Performs the given query grouped by the specified fields.
56
57 The given query's extra select fields are added.
58
59 @param query: The query to perform.
60 @param group_by: The fields by which to group.
61
62 @return A list of dicts, where each dict corresponds to single row and
63 contains a key for each grouped field as well as all of the extra
64 select fields.
65
showard35444862008-08-07 22:35:30 +000066 """
showard8bfb5cb2009-10-07 20:49:15 +000067 sql, params = self._get_group_query_sql(query, group_by)
showard56e93772008-10-06 10:06:22 +000068 cursor = readonly_connection.connection().cursor()
showard8a6eb0c2008-10-01 11:38:59 +000069 cursor.execute(sql, params)
showard06b82fc2009-06-30 01:59:42 +000070 field_names = self._get_column_names(cursor)
showard8a6eb0c2008-10-01 11:38:59 +000071 row_dicts = [dict(zip(field_names, row)) for row in cursor.fetchall()]
72 return row_dicts
73
74
75 def get_count_sql(self, query):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -080076 """Get SQL to select a per-group count of unique matches for a query.
77
78 @param query: The query to use.
79
80 @return A tuple (field alias, field SQL).
81
showard8a6eb0c2008-10-01 11:38:59 +000082 """
showarda5288b42009-07-28 20:06:08 +000083 if query.query.distinct:
showard7c199df2008-10-03 10:17:15 +000084 pk_field = self.get_key_on_this_table()
showard8a6eb0c2008-10-01 11:38:59 +000085 count_sql = 'COUNT(DISTINCT %s)' % pk_field
86 else:
87 count_sql = 'COUNT(1)'
showard7c199df2008-10-03 10:17:15 +000088 return self._GROUP_COUNT_NAME, count_sql
showard35444862008-08-07 22:35:30 +000089
90
91 def _get_num_groups_sql(self, query, group_by):
showardd2b0c882009-10-19 18:34:11 +000092 group_fields = self._get_field_names(group_by, query.query.extra_select)
showarda5288b42009-07-28 20:06:08 +000093 query = query.order_by() # this can mess up the query and isn't needed
94
Dale Curtis74a314b2011-06-23 14:55:46 -070095 compiler = query.query.get_compiler(using=query.db)
96 sql, params = compiler.as_sql()
showarda5288b42009-07-28 20:06:08 +000097 from_ = sql[sql.find(' FROM'):]
showardd2b0c882009-10-19 18:34:11 +000098 return ('SELECT DISTINCT %s %s' % (','.join(group_fields),
showarda5288b42009-07-28 20:06:08 +000099 from_),
showard35444862008-08-07 22:35:30 +0000100 params)
101
102
showardd2b0c882009-10-19 18:34:11 +0000103 def _cursor_rowcount(self, cursor):
104 """To be stubbed by tests"""
105 return cursor.rowcount
106
107
showard35444862008-08-07 22:35:30 +0000108 def get_num_groups(self, query, group_by):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800109 """Gets the number of distinct groups for a query.
110
111 @param query: The query to use.
112 @param group_by: The fields by which to group.
113
114 @return The number of distinct groups for the given query grouped by
115 the fields in group_by.
116
showard35444862008-08-07 22:35:30 +0000117 """
118 sql, params = self._get_num_groups_sql(query, group_by)
showard56e93772008-10-06 10:06:22 +0000119 cursor = readonly_connection.connection().cursor()
showard35444862008-08-07 22:35:30 +0000120 cursor.execute(sql, params)
showardd2b0c882009-10-19 18:34:11 +0000121 return self._cursor_rowcount(cursor)
showard35444862008-08-07 22:35:30 +0000122
123
124class Machine(dbmodels.Model):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800125 """Models a machine."""
showardf8b19042009-05-12 17:22:49 +0000126 machine_idx = dbmodels.AutoField(primary_key=True)
showard356d2372009-11-10 01:29:53 +0000127 hostname = dbmodels.CharField(unique=True, max_length=255)
showarda5288b42009-07-28 20:06:08 +0000128 machine_group = dbmodels.CharField(blank=True, max_length=240)
129 owner = dbmodels.CharField(blank=True, max_length=240)
showard35444862008-08-07 22:35:30 +0000130
131 class Meta:
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800132 """Metadata for class Machine."""
showardeab66ce2009-12-23 00:03:56 +0000133 db_table = 'tko_machines'
showard35444862008-08-07 22:35:30 +0000134
135
136class Kernel(dbmodels.Model):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800137 """Models a kernel."""
showardf8b19042009-05-12 17:22:49 +0000138 kernel_idx = dbmodels.AutoField(primary_key=True)
showarda5288b42009-07-28 20:06:08 +0000139 kernel_hash = dbmodels.CharField(max_length=105, editable=False)
140 base = dbmodels.CharField(max_length=90)
141 printable = dbmodels.CharField(max_length=300)
showard35444862008-08-07 22:35:30 +0000142
143 class Meta:
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800144 """Metadata for class Kernel."""
showardeab66ce2009-12-23 00:03:56 +0000145 db_table = 'tko_kernels'
showard35444862008-08-07 22:35:30 +0000146
147
148class Patch(dbmodels.Model):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800149 """Models a patch."""
showard35444862008-08-07 22:35:30 +0000150 kernel = dbmodels.ForeignKey(Kernel, db_column='kernel_idx')
showarda5288b42009-07-28 20:06:08 +0000151 name = dbmodels.CharField(blank=True, max_length=240)
152 url = dbmodels.CharField(blank=True, max_length=900)
153 the_hash = dbmodels.CharField(blank=True, max_length=105, db_column='hash')
showard35444862008-08-07 22:35:30 +0000154
155 class Meta:
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800156 """Metadata for class Patch."""
showardeab66ce2009-12-23 00:03:56 +0000157 db_table = 'tko_patches'
showard35444862008-08-07 22:35:30 +0000158
159
160class Status(dbmodels.Model):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800161 """Models a status."""
showardf8b19042009-05-12 17:22:49 +0000162 status_idx = dbmodels.AutoField(primary_key=True)
showarda5288b42009-07-28 20:06:08 +0000163 word = dbmodels.CharField(max_length=30)
showard35444862008-08-07 22:35:30 +0000164
165 class Meta:
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800166 """Metadata for class Status."""
showardeab66ce2009-12-23 00:03:56 +0000167 db_table = 'tko_status'
showard35444862008-08-07 22:35:30 +0000168
169
jamesren708f1c02010-03-31 21:43:57 +0000170class Job(dbmodels.Model, model_logic.ModelExtensions):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800171 """Models a job."""
showardf8b19042009-05-12 17:22:49 +0000172 job_idx = dbmodels.AutoField(primary_key=True)
showard356d2372009-11-10 01:29:53 +0000173 tag = dbmodels.CharField(unique=True, max_length=100)
showarda5288b42009-07-28 20:06:08 +0000174 label = dbmodels.CharField(max_length=300)
175 username = dbmodels.CharField(max_length=240)
showard35444862008-08-07 22:35:30 +0000176 machine = dbmodels.ForeignKey(Machine, db_column='machine_idx')
177 queued_time = dbmodels.DateTimeField(null=True, blank=True)
178 started_time = dbmodels.DateTimeField(null=True, blank=True)
179 finished_time = dbmodels.DateTimeField(null=True, blank=True)
showardc1c1caf2009-09-08 16:26:50 +0000180 afe_job_id = dbmodels.IntegerField(null=True, default=None)
showard35444862008-08-07 22:35:30 +0000181
jamesren708f1c02010-03-31 21:43:57 +0000182 objects = model_logic.ExtendedManager()
183
showard35444862008-08-07 22:35:30 +0000184 class Meta:
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800185 """Metadata for class Job."""
showardeab66ce2009-12-23 00:03:56 +0000186 db_table = 'tko_jobs'
showard35444862008-08-07 22:35:30 +0000187
188
showardc1a98d12010-01-15 00:22:22 +0000189class JobKeyval(dbmodels.Model):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800190 """Models a job keyval."""
showardc1a98d12010-01-15 00:22:22 +0000191 job = dbmodels.ForeignKey(Job)
192 key = dbmodels.CharField(max_length=90)
193 value = dbmodels.CharField(blank=True, max_length=300)
194
showardc1a98d12010-01-15 00:22:22 +0000195 class Meta:
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800196 """Metadata for class JobKeyval."""
showardc1a98d12010-01-15 00:22:22 +0000197 db_table = 'tko_job_keyvals'
198
199
showardf8b19042009-05-12 17:22:49 +0000200class Test(dbmodels.Model, model_logic.ModelExtensions,
201 model_logic.ModelWithAttributes):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800202 """Models a test."""
showardf8b19042009-05-12 17:22:49 +0000203 test_idx = dbmodels.AutoField(primary_key=True)
showard35444862008-08-07 22:35:30 +0000204 job = dbmodels.ForeignKey(Job, db_column='job_idx')
lmr79697282010-03-31 23:18:02 +0000205 test = dbmodels.CharField(max_length=300)
206 subdir = dbmodels.CharField(blank=True, max_length=300)
showard35444862008-08-07 22:35:30 +0000207 kernel = dbmodels.ForeignKey(Kernel, db_column='kernel_idx')
208 status = dbmodels.ForeignKey(Status, db_column='status')
showarda5288b42009-07-28 20:06:08 +0000209 reason = dbmodels.CharField(blank=True, max_length=3072)
showard35444862008-08-07 22:35:30 +0000210 machine = dbmodels.ForeignKey(Machine, db_column='machine_idx')
211 finished_time = dbmodels.DateTimeField(null=True, blank=True)
212 started_time = dbmodels.DateTimeField(null=True, blank=True)
213
showardf8b19042009-05-12 17:22:49 +0000214 objects = model_logic.ExtendedManager()
215
216 def _get_attribute_model_and_args(self, attribute):
217 return TestAttribute, dict(test=self, attribute=attribute,
218 user_created=True)
219
220
221 def set_attribute(self, attribute, value):
222 # ensure non-user-created attributes remain immutable
223 try:
224 TestAttribute.objects.get(test=self, attribute=attribute,
225 user_created=False)
226 raise ValueError('Attribute %s already exists for test %s and is '
227 'immutable' % (attribute, self.test_idx))
228 except TestAttribute.DoesNotExist:
229 super(Test, self).set_attribute(attribute, value)
230
showard35444862008-08-07 22:35:30 +0000231 class Meta:
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800232 """Metadata for class Test."""
showardeab66ce2009-12-23 00:03:56 +0000233 db_table = 'tko_tests'
showard35444862008-08-07 22:35:30 +0000234
235
showarde732ee72008-09-23 19:15:43 +0000236class TestAttribute(dbmodels.Model, model_logic.ModelExtensions):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800237 """Models a test attribute."""
showardf8b19042009-05-12 17:22:49 +0000238 test = dbmodels.ForeignKey(Test, db_column='test_idx')
showarda5288b42009-07-28 20:06:08 +0000239 attribute = dbmodels.CharField(max_length=90)
240 value = dbmodels.CharField(blank=True, max_length=300)
showardf8b19042009-05-12 17:22:49 +0000241 user_created = dbmodels.BooleanField(default=False)
242
243 objects = model_logic.ExtendedManager()
showard35444862008-08-07 22:35:30 +0000244
245 class Meta:
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800246 """Metadata for class TestAttribute."""
showardeab66ce2009-12-23 00:03:56 +0000247 db_table = 'tko_test_attributes'
showard35444862008-08-07 22:35:30 +0000248
249
jadmanski430dca92008-12-16 20:56:53 +0000250class IterationAttribute(dbmodels.Model, model_logic.ModelExtensions):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800251 """Models an iteration attribute."""
252 # This isn't really a primary key, but it's necessary to appease Django
253 # and is harmless as long as we're careful.
showarde732ee72008-09-23 19:15:43 +0000254 test = dbmodels.ForeignKey(Test, db_column='test_idx', primary_key=True)
showard35444862008-08-07 22:35:30 +0000255 iteration = dbmodels.IntegerField()
showarda5288b42009-07-28 20:06:08 +0000256 attribute = dbmodels.CharField(max_length=90)
257 value = dbmodels.CharField(blank=True, max_length=300)
showard35444862008-08-07 22:35:30 +0000258
showardf8b19042009-05-12 17:22:49 +0000259 objects = model_logic.ExtendedManager()
260
showard35444862008-08-07 22:35:30 +0000261 class Meta:
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800262 """Metadata for class IterationAttribute."""
showardeab66ce2009-12-23 00:03:56 +0000263 db_table = 'tko_iteration_attributes'
showard35444862008-08-07 22:35:30 +0000264
265
jadmanski430dca92008-12-16 20:56:53 +0000266class IterationResult(dbmodels.Model, model_logic.ModelExtensions):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800267 """Models an iteration result."""
268 # See comment on IterationAttribute regarding primary_key=True.
jadmanski430dca92008-12-16 20:56:53 +0000269 test = dbmodels.ForeignKey(Test, db_column='test_idx', primary_key=True)
showard35444862008-08-07 22:35:30 +0000270 iteration = dbmodels.IntegerField()
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800271 attribute = dbmodels.CharField(max_length=256)
jamesren26f9a9f2010-04-02 17:44:55 +0000272 value = dbmodels.FloatField(null=True, blank=True)
showard35444862008-08-07 22:35:30 +0000273
showardf8b19042009-05-12 17:22:49 +0000274 objects = model_logic.ExtendedManager()
275
showard35444862008-08-07 22:35:30 +0000276 class Meta:
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800277 """Metadata for class IterationResult."""
showardeab66ce2009-12-23 00:03:56 +0000278 db_table = 'tko_iteration_result'
showard35444862008-08-07 22:35:30 +0000279
280
281class TestLabel(dbmodels.Model, model_logic.ModelExtensions):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800282 """Models a test label."""
showarda5288b42009-07-28 20:06:08 +0000283 name = dbmodels.CharField(max_length=80, unique=True)
showard35444862008-08-07 22:35:30 +0000284 description = dbmodels.TextField(blank=True)
showardeab66ce2009-12-23 00:03:56 +0000285 tests = dbmodels.ManyToManyField(Test, blank=True,
286 db_table='tko_test_labels_tests')
showard35444862008-08-07 22:35:30 +0000287
288 name_field = 'name'
showardf8b19042009-05-12 17:22:49 +0000289 objects = model_logic.ExtendedManager()
showard35444862008-08-07 22:35:30 +0000290
291 class Meta:
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800292 """Metadata for class TestLabel."""
showardeab66ce2009-12-23 00:03:56 +0000293 db_table = 'tko_test_labels'
showard35444862008-08-07 22:35:30 +0000294
295
296class SavedQuery(dbmodels.Model, model_logic.ModelExtensions):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800297 """Models a saved query."""
298 # TODO: change this to foreign key once DBs are merged.
showarda5288b42009-07-28 20:06:08 +0000299 owner = dbmodels.CharField(max_length=80)
300 name = dbmodels.CharField(max_length=100)
showard35444862008-08-07 22:35:30 +0000301 url_token = dbmodels.TextField()
302
303 class Meta:
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800304 """Metadata for class SavedQuery."""
showardeab66ce2009-12-23 00:03:56 +0000305 db_table = 'tko_saved_queries'
showard35444862008-08-07 22:35:30 +0000306
307
showardce12f552008-09-19 00:48:59 +0000308class EmbeddedGraphingQuery(dbmodels.Model, model_logic.ModelExtensions):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800309 """Models an embedded graphing query."""
showardce12f552008-09-19 00:48:59 +0000310 url_token = dbmodels.TextField(null=False, blank=False)
showarda5288b42009-07-28 20:06:08 +0000311 graph_type = dbmodels.CharField(max_length=16, null=False, blank=False)
showardce12f552008-09-19 00:48:59 +0000312 params = dbmodels.TextField(null=False, blank=False)
313 last_updated = dbmodels.DateTimeField(null=False, blank=False,
314 editable=False)
315 # refresh_time shows the time at which a thread is updating the cached
316 # image, or NULL if no one is updating the image. This is used so that only
317 # one thread is updating the cached image at a time (see
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800318 # graphing_utils.handle_plot_request).
showardce12f552008-09-19 00:48:59 +0000319 refresh_time = dbmodels.DateTimeField(editable=False)
320 cached_png = dbmodels.TextField(editable=False)
321
322 class Meta:
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800323 """Metadata for class EmbeddedGraphingQuery."""
showardeab66ce2009-12-23 00:03:56 +0000324 db_table = 'tko_embedded_graphing_queries'
showardce12f552008-09-19 00:48:59 +0000325
326
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800327# Views.
showard35444862008-08-07 22:35:30 +0000328
329class TestViewManager(TempManager):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800330 """A Test View Manager."""
331
showard35444862008-08-07 22:35:30 +0000332 def get_query_set(self):
333 query = super(TestViewManager, self).get_query_set()
showard5bf7c502008-08-20 01:22:22 +0000334
showard35444862008-08-07 22:35:30 +0000335 # add extra fields to selects, using the SQL itself as the "alias"
336 extra_select = dict((sql, sql)
337 for sql in self.model.extra_fields.iterkeys())
338 return query.extra(select=extra_select)
339
340
showardf2489522008-10-23 23:08:00 +0000341 def _get_include_exclude_suffix(self, exclude):
342 if exclude:
showard2aa318e2009-08-20 23:43:10 +0000343 return '_exclude'
344 return '_include'
345
346
347 def _add_attribute_join(self, query_set, join_condition,
348 suffix=None, exclude=False):
349 if suffix is None:
350 suffix = self._get_include_exclude_suffix(exclude)
showardeab66ce2009-12-23 00:03:56 +0000351 return self.add_join(query_set, 'tko_test_attributes',
352 join_key='test_idx',
showard2aa318e2009-08-20 23:43:10 +0000353 join_condition=join_condition,
354 suffix=suffix, exclude=exclude)
355
356
357 def _add_label_pivot_table_join(self, query_set, suffix, join_condition='',
358 exclude=False, force_left_join=False):
showardeab66ce2009-12-23 00:03:56 +0000359 return self.add_join(query_set, 'tko_test_labels_tests',
360 join_key='test_id',
showard2aa318e2009-08-20 23:43:10 +0000361 join_condition=join_condition,
362 suffix=suffix, exclude=exclude,
363 force_left_join=force_left_join)
showardf2489522008-10-23 23:08:00 +0000364
365
showard64aeecd2008-09-19 21:32:58 +0000366 def _add_label_joins(self, query_set, suffix=''):
showard2aa318e2009-08-20 23:43:10 +0000367 query_set = self._add_label_pivot_table_join(
368 query_set, suffix=suffix, force_left_join=True)
showardd50ffb42008-09-04 02:47:45 +0000369
showard2aa318e2009-08-20 23:43:10 +0000370 # since we're not joining from the original table, we can't use
371 # self.add_join() again
showardeab66ce2009-12-23 00:03:56 +0000372 second_join_alias = 'tko_test_labels' + suffix
showardd50ffb42008-09-04 02:47:45 +0000373 second_join_condition = ('%s.id = %s.testlabel_id' %
showard64aeecd2008-09-19 21:32:58 +0000374 (second_join_alias,
showardeab66ce2009-12-23 00:03:56 +0000375 'tko_test_labels_tests' + suffix))
showard7e67b432010-01-20 01:13:04 +0000376 query_set.query.add_custom_join('tko_test_labels',
377 second_join_condition,
378 query_set.query.LOUTER,
379 alias=second_join_alias)
380 return query_set
showardd50ffb42008-09-04 02:47:45 +0000381
showard64aeecd2008-09-19 21:32:58 +0000382
showardd50ffb42008-09-04 02:47:45 +0000383 def _get_label_ids_from_names(self, label_names):
showard2aa318e2009-08-20 23:43:10 +0000384 label_ids = list( # listifying avoids a double query below
showard8b0ea222009-12-23 19:23:03 +0000385 TestLabel.objects.filter(name__in=label_names)
386 .values_list('name', 'id'))
showard2aa318e2009-08-20 23:43:10 +0000387 if len(label_ids) < len(set(label_names)):
lmr79697282010-03-31 23:18:02 +0000388 raise ValueError('Not all labels found: %s' %
389 ', '.join(label_names))
showard8b0ea222009-12-23 19:23:03 +0000390 return dict(name_and_id for name_and_id in label_ids)
showard2aa318e2009-08-20 23:43:10 +0000391
392
393 def _include_or_exclude_labels(self, query_set, label_names, exclude=False):
showard8b0ea222009-12-23 19:23:03 +0000394 label_ids = self._get_label_ids_from_names(label_names).itervalues()
showard2aa318e2009-08-20 23:43:10 +0000395 suffix = self._get_include_exclude_suffix(exclude)
showardeab66ce2009-12-23 00:03:56 +0000396 condition = ('tko_test_labels_tests%s.testlabel_id IN (%s)' %
showard8b0ea222009-12-23 19:23:03 +0000397 (suffix,
398 ','.join(str(label_id) for label_id in label_ids)))
showard2aa318e2009-08-20 23:43:10 +0000399 return self._add_label_pivot_table_join(query_set,
400 join_condition=condition,
401 suffix=suffix,
402 exclude=exclude)
showard02813502008-08-20 20:52:56 +0000403
404
showard8b0ea222009-12-23 19:23:03 +0000405 def _add_custom_select(self, query_set, select_name, select_sql):
406 return query_set.extra(select={select_name: select_sql})
407
408
409 def _add_select_value(self, query_set, alias):
410 return self._add_custom_select(query_set, alias,
411 _quote_name(alias) + '.value')
412
413
414 def _add_select_ifnull(self, query_set, alias, non_null_value):
415 select_sql = "IF(%s.id IS NOT NULL, '%s', NULL)" % (_quote_name(alias),
416 non_null_value)
417 return self._add_custom_select(query_set, alias, select_sql)
418
419
jamesren708f1c02010-03-31 21:43:57 +0000420 def _join_test_label_column(self, query_set, label_name, label_id):
421 alias = 'test_label_' + label_name
showard7e67b432010-01-20 01:13:04 +0000422 label_query = TestLabel.objects.filter(name=label_name)
423 query_set = Test.objects.join_custom_field(query_set, label_query,
424 alias)
showard8b0ea222009-12-23 19:23:03 +0000425
426 query_set = self._add_select_ifnull(query_set, alias, label_name)
427 return query_set
428
429
jamesren708f1c02010-03-31 21:43:57 +0000430 def _join_test_label_columns(self, query_set, label_names):
showard8b0ea222009-12-23 19:23:03 +0000431 label_id_map = self._get_label_ids_from_names(label_names)
432 for label_name in label_names:
jamesren708f1c02010-03-31 21:43:57 +0000433 query_set = self._join_test_label_column(query_set, label_name,
434 label_id_map[label_name])
showard8b0ea222009-12-23 19:23:03 +0000435 return query_set
436
437
jamesren708f1c02010-03-31 21:43:57 +0000438 def _join_test_attribute(self, query_set, attribute, alias=None,
439 extra_join_condition=None):
showard8b0ea222009-12-23 19:23:03 +0000440 """
441 Join the given TestView QuerySet to TestAttribute. The resulting query
442 has an additional column for the given attribute named
443 "attribute_<attribute name>".
444 """
showard8b0ea222009-12-23 19:23:03 +0000445 if not alias:
jamesren708f1c02010-03-31 21:43:57 +0000446 alias = 'test_attribute_' + attribute
showard7e67b432010-01-20 01:13:04 +0000447 attribute_query = TestAttribute.objects.filter(attribute=attribute)
showard8b0ea222009-12-23 19:23:03 +0000448 if extra_join_condition:
showard7e67b432010-01-20 01:13:04 +0000449 attribute_query = attribute_query.extra(
450 where=[extra_join_condition])
451 query_set = Test.objects.join_custom_field(query_set, attribute_query,
452 alias)
showard8b0ea222009-12-23 19:23:03 +0000453
454 query_set = self._add_select_value(query_set, alias)
455 return query_set
456
457
458 def _join_machine_label_columns(self, query_set, machine_label_names):
459 for label_name in machine_label_names:
460 alias = 'machine_label_' + label_name
461 condition = "FIND_IN_SET('%s', %s)" % (
462 label_name, _quote_name(alias) + '.value')
jamesren708f1c02010-03-31 21:43:57 +0000463 query_set = self._join_test_attribute(
464 query_set, 'host-labels',
465 alias=alias, extra_join_condition=condition)
showard8b0ea222009-12-23 19:23:03 +0000466 query_set = self._add_select_ifnull(query_set, alias, label_name)
467 return query_set
468
469
470 def _join_one_iteration_key(self, query_set, result_key, first_alias=None):
jamesren708f1c02010-03-31 21:43:57 +0000471 alias = 'iteration_result_' + result_key
showard7e67b432010-01-20 01:13:04 +0000472 iteration_query = IterationResult.objects.filter(attribute=result_key)
showard8b0ea222009-12-23 19:23:03 +0000473 if first_alias:
474 # after the first join, we need to match up iteration indices,
475 # otherwise each join will expand the query by the number of
476 # iterations and we'll have extraneous rows
showard7e67b432010-01-20 01:13:04 +0000477 iteration_query = iteration_query.extra(
478 where=['%s.iteration = %s.iteration'
479 % (_quote_name(alias), _quote_name(first_alias))])
showard8b0ea222009-12-23 19:23:03 +0000480
showard7e67b432010-01-20 01:13:04 +0000481 query_set = Test.objects.join_custom_field(query_set, iteration_query,
482 alias, left_join=False)
showard8b0ea222009-12-23 19:23:03 +0000483 # select the iteration value and index for this join
484 query_set = self._add_select_value(query_set, alias)
485 if not first_alias:
486 # for first join, add iteration index select too
487 query_set = self._add_custom_select(
488 query_set, 'iteration_index',
489 _quote_name(alias) + '.iteration')
490
491 return query_set, alias
492
493
jamesren708f1c02010-03-31 21:43:57 +0000494 def _join_iteration_results(self, test_view_query_set, result_keys):
showard8b0ea222009-12-23 19:23:03 +0000495 """Join the given TestView QuerySet to IterationResult for one result.
496
497 The resulting query looks like a TestView query but has one row per
498 iteration. Each row includes all the attributes of TestView, an
499 attribute for each key in result_keys and an iteration_index attribute.
500
501 We accomplish this by joining the TestView query to IterationResult
502 once per result key. Each join is restricted on the result key (and on
503 the test index, like all one-to-many joins). For the first join, this
504 is the only restriction, so each TestView row expands to a row per
505 iteration (per iteration that includes the key, of course). For each
506 subsequent join, we also restrict the iteration index to match that of
507 the initial join. This makes each subsequent join produce exactly one
508 result row for each input row. (This assumes each iteration contains
509 the same set of keys. Results are undefined if that's not true.)
510 """
511 if not result_keys:
512 return test_view_query_set
513
514 query_set, first_alias = self._join_one_iteration_key(
515 test_view_query_set, result_keys[0])
516 for result_key in result_keys[1:]:
517 query_set, _ = self._join_one_iteration_key(query_set, result_key,
518 first_alias=first_alias)
519 return query_set
520
521
jamesren708f1c02010-03-31 21:43:57 +0000522 def _join_job_keyvals(self, query_set, job_keyvals):
523 for job_keyval in job_keyvals:
524 alias = 'job_keyval_' + job_keyval
525 keyval_query = JobKeyval.objects.filter(key=job_keyval)
526 query_set = Job.objects.join_custom_field(query_set, keyval_query,
527 alias)
528 query_set = self._add_select_value(query_set, alias)
529 return query_set
530
531
532 def _join_iteration_attributes(self, query_set, iteration_attributes):
533 for attribute in iteration_attributes:
534 alias = 'iteration_attribute_' + attribute
535 attribute_query = IterationAttribute.objects.filter(
536 attribute=attribute)
537 query_set = Test.objects.join_custom_field(query_set,
538 attribute_query, alias)
539 query_set = self._add_select_value(query_set, alias)
540 return query_set
541
542
showard8b0ea222009-12-23 19:23:03 +0000543 def get_query_set_with_joins(self, filter_data):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800544 """Add joins for querying over test-related items.
showard8b0ea222009-12-23 19:23:03 +0000545
546 These parameters are supported going forward:
547 * test_attribute_fields: list of attribute names. Each attribute will
548 be available as a column attribute_<name>.value.
549 * test_label_fields: list of label names. Each label will be available
550 as a column label_<name>.id, non-null iff the label is present.
jamesren708f1c02010-03-31 21:43:57 +0000551 * iteration_result_fields: list of iteration result names. Each
showard8b0ea222009-12-23 19:23:03 +0000552 result will be available as a column iteration_<name>.value.
553 Note that this changes the semantics to return iterations
554 instead of tests -- if a test has multiple iterations, a row
555 will be returned for each one. The iteration index is also
556 available as iteration_<name>.iteration.
557 * machine_label_fields: list of machine label names. Each will be
558 available as a column machine_label_<name>.id, non-null iff the
559 label is present on the machine used in the test.
jamesren708f1c02010-03-31 21:43:57 +0000560 * job_keyval_fields: list of job keyval names. Each value will be
561 available as a column job_keyval_<name>.id, non-null iff the
562 keyval is present in the AFE job.
563 * iteration_attribute_fields: list of iteration attribute names. Each
564 attribute will be available as a column
565 iteration_attribute<name>.id, non-null iff the attribute is
566 present.
showard8b0ea222009-12-23 19:23:03 +0000567
568 These parameters are deprecated:
569 * include_labels
570 * exclude_labels
571 * include_attributes_where
572 * exclude_attributes_where
573
574 Additionally, this method adds joins if the following strings are
575 present in extra_where (this is also deprecated):
576 * test_labels
577 * test_attributes_host_labels
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800578
579 @param filter_data: Data by which to filter.
580
581 @return A QuerySet.
582
showard8b0ea222009-12-23 19:23:03 +0000583 """
showard35444862008-08-07 22:35:30 +0000584 query_set = self.get_query_set()
showard8b0ea222009-12-23 19:23:03 +0000585
586 test_attributes = filter_data.pop('test_attribute_fields', [])
587 for attribute in test_attributes:
jamesren708f1c02010-03-31 21:43:57 +0000588 query_set = self._join_test_attribute(query_set, attribute)
showard8b0ea222009-12-23 19:23:03 +0000589
590 test_labels = filter_data.pop('test_label_fields', [])
jamesren708f1c02010-03-31 21:43:57 +0000591 query_set = self._join_test_label_columns(query_set, test_labels)
showard8b0ea222009-12-23 19:23:03 +0000592
593 machine_labels = filter_data.pop('machine_label_fields', [])
594 query_set = self._join_machine_label_columns(query_set, machine_labels)
595
jamesren708f1c02010-03-31 21:43:57 +0000596 iteration_keys = filter_data.pop('iteration_result_fields', [])
597 query_set = self._join_iteration_results(query_set, iteration_keys)
598
599 job_keyvals = filter_data.pop('job_keyval_fields', [])
600 query_set = self._join_job_keyvals(query_set, job_keyvals)
601
602 iteration_attributes = filter_data.pop('iteration_attribute_fields', [])
603 query_set = self._join_iteration_attributes(query_set,
604 iteration_attributes)
showard8b0ea222009-12-23 19:23:03 +0000605
606 # everything that follows is deprecated behavior
607
showardd50ffb42008-09-04 02:47:45 +0000608 joined = False
showard2aa318e2009-08-20 23:43:10 +0000609
showardf2489522008-10-23 23:08:00 +0000610 extra_where = filter_data.get('extra_where', '')
showardeab66ce2009-12-23 00:03:56 +0000611 if 'tko_test_labels' in extra_where:
showard02813502008-08-20 20:52:56 +0000612 query_set = self._add_label_joins(query_set)
showardd50ffb42008-09-04 02:47:45 +0000613 joined = True
614
showard8b0ea222009-12-23 19:23:03 +0000615 include_labels = filter_data.pop('include_labels', [])
616 exclude_labels = filter_data.pop('exclude_labels', [])
showard2aa318e2009-08-20 23:43:10 +0000617 if include_labels:
618 query_set = self._include_or_exclude_labels(query_set,
619 include_labels)
showardfc8c6ae2008-11-11 19:06:01 +0000620 joined = True
showard2aa318e2009-08-20 23:43:10 +0000621 if exclude_labels:
622 query_set = self._include_or_exclude_labels(query_set,
623 exclude_labels,
624 exclude=True)
showard64aeecd2008-09-19 21:32:58 +0000625 joined = True
626
627 include_attributes_where = filter_data.pop('include_attributes_where',
628 '')
629 exclude_attributes_where = filter_data.pop('exclude_attributes_where',
630 '')
631 if include_attributes_where:
632 query_set = self._add_attribute_join(
showard2aa318e2009-08-20 23:43:10 +0000633 query_set,
634 join_condition=self.escape_user_sql(include_attributes_where))
showard64aeecd2008-09-19 21:32:58 +0000635 joined = True
636 if exclude_attributes_where:
637 query_set = self._add_attribute_join(
showard2aa318e2009-08-20 23:43:10 +0000638 query_set,
639 join_condition=self.escape_user_sql(exclude_attributes_where),
showard64aeecd2008-09-19 21:32:58 +0000640 exclude=True)
641 joined = True
showardd50ffb42008-09-04 02:47:45 +0000642
643 if not joined:
showard35444862008-08-07 22:35:30 +0000644 filter_data['no_distinct'] = True
showardd50ffb42008-09-04 02:47:45 +0000645
showard8b0ea222009-12-23 19:23:03 +0000646 if 'tko_test_attributes_host_labels' in extra_where:
showardf2489522008-10-23 23:08:00 +0000647 query_set = self._add_attribute_join(
648 query_set, suffix='_host_labels',
showardeab66ce2009-12-23 00:03:56 +0000649 join_condition='tko_test_attributes_host_labels.attribute = '
showardf2489522008-10-23 23:08:00 +0000650 '"host-labels"')
651
showard35444862008-08-07 22:35:30 +0000652 return query_set
653
654
showard8bfb5cb2009-10-07 20:49:15 +0000655 def query_test_ids(self, filter_data, apply_presentation=True):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800656 """Queries for test IDs.
657
658 @param filter_data: Data by which to filter.
659 @param apply_presentation: Whether or not to apply presentation
660 parameters.
661
662 @return A list of test IDs.
663
664 """
showard8bfb5cb2009-10-07 20:49:15 +0000665 query = self.model.query_objects(filter_data,
666 apply_presentation=apply_presentation)
667 dicts = query.values('test_idx')
showard02813502008-08-20 20:52:56 +0000668 return [item['test_idx'] for item in dicts]
669
670
showard02813502008-08-20 20:52:56 +0000671 def query_test_label_ids(self, filter_data):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800672 """Queries for test label IDs.
673
674 @param filter_data: Data by which to filter.
675
676 @return A list of test label IDs.
677
678 """
showardd50ffb42008-09-04 02:47:45 +0000679 query_set = self.model.query_objects(filter_data)
680 query_set = self._add_label_joins(query_set, suffix='_list')
showardeab66ce2009-12-23 00:03:56 +0000681 rows = self._custom_select_query(query_set, ['tko_test_labels_list.id'])
showardd50ffb42008-09-04 02:47:45 +0000682 return [row[0] for row in rows if row[0] is not None]
showard02813502008-08-20 20:52:56 +0000683
684
showardeaccf8f2009-04-16 03:11:33 +0000685 def escape_user_sql(self, sql):
686 sql = super(TestViewManager, self).escape_user_sql(sql)
687 return sql.replace('test_idx', self.get_key_on_this_table('test_idx'))
688
689
showard35444862008-08-07 22:35:30 +0000690class TestView(dbmodels.Model, model_logic.ModelExtensions):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800691 """Models a test view."""
showard35444862008-08-07 22:35:30 +0000692 extra_fields = {
showardf4c702e2009-07-08 21:14:27 +0000693 'DATE(job_queued_time)': 'job queued day',
694 'DATE(test_finished_time)': 'test finished day',
showard35444862008-08-07 22:35:30 +0000695 }
696
697 group_fields = [
showardf4c702e2009-07-08 21:14:27 +0000698 'test_name',
699 'status',
700 'kernel',
701 'hostname',
702 'job_tag',
703 'job_name',
704 'platform',
705 'reason',
706 'job_owner',
707 'job_queued_time',
708 'DATE(job_queued_time)',
709 'test_started_time',
710 'test_finished_time',
711 'DATE(test_finished_time)',
showard35444862008-08-07 22:35:30 +0000712 ]
713
714 test_idx = dbmodels.IntegerField('test index', primary_key=True)
715 job_idx = dbmodels.IntegerField('job index', null=True, blank=True)
showarda5288b42009-07-28 20:06:08 +0000716 test_name = dbmodels.CharField(blank=True, max_length=90)
717 subdir = dbmodels.CharField('subdirectory', blank=True, max_length=180)
showard35444862008-08-07 22:35:30 +0000718 kernel_idx = dbmodels.IntegerField('kernel index')
719 status_idx = dbmodels.IntegerField('status index')
showarda5288b42009-07-28 20:06:08 +0000720 reason = dbmodels.CharField(blank=True, max_length=3072)
showard35444862008-08-07 22:35:30 +0000721 machine_idx = dbmodels.IntegerField('host index')
722 test_started_time = dbmodels.DateTimeField(null=True, blank=True)
723 test_finished_time = dbmodels.DateTimeField(null=True, blank=True)
showarda5288b42009-07-28 20:06:08 +0000724 job_tag = dbmodels.CharField(blank=True, max_length=300)
725 job_name = dbmodels.CharField(blank=True, max_length=300)
726 job_owner = dbmodels.CharField('owner', blank=True, max_length=240)
showard35444862008-08-07 22:35:30 +0000727 job_queued_time = dbmodels.DateTimeField(null=True, blank=True)
728 job_started_time = dbmodels.DateTimeField(null=True, blank=True)
729 job_finished_time = dbmodels.DateTimeField(null=True, blank=True)
showardc1c1caf2009-09-08 16:26:50 +0000730 afe_job_id = dbmodels.IntegerField(null=True)
showarda5288b42009-07-28 20:06:08 +0000731 hostname = dbmodels.CharField(blank=True, max_length=300)
732 platform = dbmodels.CharField(blank=True, max_length=240)
733 machine_owner = dbmodels.CharField(blank=True, max_length=240)
734 kernel_hash = dbmodels.CharField(blank=True, max_length=105)
735 kernel_base = dbmodels.CharField(blank=True, max_length=90)
736 kernel = dbmodels.CharField(blank=True, max_length=300)
737 status = dbmodels.CharField(blank=True, max_length=30)
showard35444862008-08-07 22:35:30 +0000738
739 objects = TestViewManager()
740
741 def save(self):
742 raise NotImplementedError('TestView is read-only')
743
744
745 def delete(self):
746 raise NotImplementedError('TestView is read-only')
747
748
749 @classmethod
showard8bfb5cb2009-10-07 20:49:15 +0000750 def query_objects(cls, filter_data, initial_query=None,
751 apply_presentation=True):
showard35444862008-08-07 22:35:30 +0000752 if initial_query is None:
showard64aeecd2008-09-19 21:32:58 +0000753 initial_query = cls.objects.get_query_set_with_joins(filter_data)
showard8bfb5cb2009-10-07 20:49:15 +0000754 return super(TestView, cls).query_objects(
755 filter_data, initial_query=initial_query,
756 apply_presentation=apply_presentation)
showard35444862008-08-07 22:35:30 +0000757
showard35444862008-08-07 22:35:30 +0000758 class Meta:
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800759 """Metadata for class TestView."""
showardeab66ce2009-12-23 00:03:56 +0000760 db_table = 'tko_test_view_2'