blob: 286279bfa6d49945cd7d2dca4d95b2b17b65ecb2 [file] [log] [blame]
showard35444862008-08-07 22:35:30 +00001from django.db import models as dbmodels, connection
showard35444862008-08-07 22:35:30 +00002from autotest_lib.frontend.afe import model_logic, readonly_connection
3
showard8b0ea222009-12-23 19:23:03 +00004_quote_name = connection.ops.quote_name
5
showard35444862008-08-07 22:35:30 +00006class TempManager(model_logic.ExtendedManager):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -08007 """A Temp Manager."""
showard35444862008-08-07 22:35:30 +00008 _GROUP_COUNT_NAME = 'group_count'
9
10 def _get_key_unless_is_function(self, field):
11 if '(' in field:
12 return field
showard7c199df2008-10-03 10:17:15 +000013 return self.get_key_on_this_table(field)
showard35444862008-08-07 22:35:30 +000014
15
showardf2489522008-10-23 23:08:00 +000016 def _get_field_names(self, fields, extra_select_fields={}):
17 field_names = []
18 for field in fields:
19 if field in extra_select_fields:
showardd2b0c882009-10-19 18:34:11 +000020 field_names.append(extra_select_fields[field][0])
showardf2489522008-10-23 23:08:00 +000021 else:
22 field_names.append(self._get_key_unless_is_function(field))
23 return field_names
showard35444862008-08-07 22:35:30 +000024
25
showard8bfb5cb2009-10-07 20:49:15 +000026 def _get_group_query_sql(self, query, group_by):
Dale Curtis74a314b2011-06-23 14:55:46 -070027 compiler = query.query.get_compiler(using=query.db)
28 sql, params = compiler.as_sql()
29
showardd50ffb42008-09-04 02:47:45 +000030
31 # insert GROUP BY clause into query
showard8bfb5cb2009-10-07 20:49:15 +000032 group_fields = self._get_field_names(group_by, query.query.extra_select)
showard7c199df2008-10-03 10:17:15 +000033 group_by_clause = ' GROUP BY ' + ', '.join(group_fields)
showarda5288b42009-07-28 20:06:08 +000034 group_by_position = sql.rfind('ORDER BY')
showardd50ffb42008-09-04 02:47:45 +000035 if group_by_position == -1:
showarda5288b42009-07-28 20:06:08 +000036 group_by_position = len(sql)
37 sql = (sql[:group_by_position] +
38 group_by_clause + ' ' +
39 sql[group_by_position:])
showard35444862008-08-07 22:35:30 +000040
showarda5288b42009-07-28 20:06:08 +000041 return sql, params
showard35444862008-08-07 22:35:30 +000042
43
showard06b82fc2009-06-30 01:59:42 +000044 def _get_column_names(self, cursor):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -080045 """Gets the column names from the cursor description.
46
47 This method exists so that it can be mocked in the unit test for
48 sqlite3 compatibility.
49
showard06b82fc2009-06-30 01:59:42 +000050 """
51 return [column_info[0] for column_info in cursor.description]
52
53
showard8bfb5cb2009-10-07 20:49:15 +000054 def execute_group_query(self, query, group_by):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -080055 """Performs the given query grouped by the specified fields.
56
57 The given query's extra select fields are added.
58
59 @param query: The query to perform.
60 @param group_by: The fields by which to group.
61
62 @return A list of dicts, where each dict corresponds to single row and
63 contains a key for each grouped field as well as all of the extra
64 select fields.
65
showard35444862008-08-07 22:35:30 +000066 """
showard8bfb5cb2009-10-07 20:49:15 +000067 sql, params = self._get_group_query_sql(query, group_by)
showard56e93772008-10-06 10:06:22 +000068 cursor = readonly_connection.connection().cursor()
showard8a6eb0c2008-10-01 11:38:59 +000069 cursor.execute(sql, params)
showard06b82fc2009-06-30 01:59:42 +000070 field_names = self._get_column_names(cursor)
showard8a6eb0c2008-10-01 11:38:59 +000071 row_dicts = [dict(zip(field_names, row)) for row in cursor.fetchall()]
72 return row_dicts
73
74
75 def get_count_sql(self, query):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -080076 """Get SQL to select a per-group count of unique matches for a query.
77
78 @param query: The query to use.
79
80 @return A tuple (field alias, field SQL).
81
showard8a6eb0c2008-10-01 11:38:59 +000082 """
showarda5288b42009-07-28 20:06:08 +000083 if query.query.distinct:
showard7c199df2008-10-03 10:17:15 +000084 pk_field = self.get_key_on_this_table()
showard8a6eb0c2008-10-01 11:38:59 +000085 count_sql = 'COUNT(DISTINCT %s)' % pk_field
86 else:
87 count_sql = 'COUNT(1)'
showard7c199df2008-10-03 10:17:15 +000088 return self._GROUP_COUNT_NAME, count_sql
showard35444862008-08-07 22:35:30 +000089
90
91 def _get_num_groups_sql(self, query, group_by):
showardd2b0c882009-10-19 18:34:11 +000092 group_fields = self._get_field_names(group_by, query.query.extra_select)
showarda5288b42009-07-28 20:06:08 +000093 query = query.order_by() # this can mess up the query and isn't needed
94
Dale Curtis74a314b2011-06-23 14:55:46 -070095 compiler = query.query.get_compiler(using=query.db)
96 sql, params = compiler.as_sql()
showarda5288b42009-07-28 20:06:08 +000097 from_ = sql[sql.find(' FROM'):]
showardd2b0c882009-10-19 18:34:11 +000098 return ('SELECT DISTINCT %s %s' % (','.join(group_fields),
showarda5288b42009-07-28 20:06:08 +000099 from_),
showard35444862008-08-07 22:35:30 +0000100 params)
101
102
showardd2b0c882009-10-19 18:34:11 +0000103 def _cursor_rowcount(self, cursor):
104 """To be stubbed by tests"""
105 return cursor.rowcount
106
107
showard35444862008-08-07 22:35:30 +0000108 def get_num_groups(self, query, group_by):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800109 """Gets the number of distinct groups for a query.
110
111 @param query: The query to use.
112 @param group_by: The fields by which to group.
113
114 @return The number of distinct groups for the given query grouped by
115 the fields in group_by.
116
showard35444862008-08-07 22:35:30 +0000117 """
118 sql, params = self._get_num_groups_sql(query, group_by)
showard56e93772008-10-06 10:06:22 +0000119 cursor = readonly_connection.connection().cursor()
showard35444862008-08-07 22:35:30 +0000120 cursor.execute(sql, params)
showardd2b0c882009-10-19 18:34:11 +0000121 return self._cursor_rowcount(cursor)
showard35444862008-08-07 22:35:30 +0000122
123
124class Machine(dbmodels.Model):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800125 """Models a machine."""
showardf8b19042009-05-12 17:22:49 +0000126 machine_idx = dbmodels.AutoField(primary_key=True)
showard356d2372009-11-10 01:29:53 +0000127 hostname = dbmodels.CharField(unique=True, max_length=255)
showarda5288b42009-07-28 20:06:08 +0000128 machine_group = dbmodels.CharField(blank=True, max_length=240)
129 owner = dbmodels.CharField(blank=True, max_length=240)
showard35444862008-08-07 22:35:30 +0000130
131 class Meta:
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800132 """Metadata for class Machine."""
showardeab66ce2009-12-23 00:03:56 +0000133 db_table = 'tko_machines'
showard35444862008-08-07 22:35:30 +0000134
135
136class Kernel(dbmodels.Model):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800137 """Models a kernel."""
showardf8b19042009-05-12 17:22:49 +0000138 kernel_idx = dbmodels.AutoField(primary_key=True)
showarda5288b42009-07-28 20:06:08 +0000139 kernel_hash = dbmodels.CharField(max_length=105, editable=False)
140 base = dbmodels.CharField(max_length=90)
141 printable = dbmodels.CharField(max_length=300)
showard35444862008-08-07 22:35:30 +0000142
143 class Meta:
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800144 """Metadata for class Kernel."""
showardeab66ce2009-12-23 00:03:56 +0000145 db_table = 'tko_kernels'
showard35444862008-08-07 22:35:30 +0000146
147
148class Patch(dbmodels.Model):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800149 """Models a patch."""
showard35444862008-08-07 22:35:30 +0000150 kernel = dbmodels.ForeignKey(Kernel, db_column='kernel_idx')
showarda5288b42009-07-28 20:06:08 +0000151 name = dbmodels.CharField(blank=True, max_length=240)
152 url = dbmodels.CharField(blank=True, max_length=900)
153 the_hash = dbmodels.CharField(blank=True, max_length=105, db_column='hash')
showard35444862008-08-07 22:35:30 +0000154
155 class Meta:
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800156 """Metadata for class Patch."""
showardeab66ce2009-12-23 00:03:56 +0000157 db_table = 'tko_patches'
showard35444862008-08-07 22:35:30 +0000158
159
160class Status(dbmodels.Model):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800161 """Models a status."""
showardf8b19042009-05-12 17:22:49 +0000162 status_idx = dbmodels.AutoField(primary_key=True)
showarda5288b42009-07-28 20:06:08 +0000163 word = dbmodels.CharField(max_length=30)
showard35444862008-08-07 22:35:30 +0000164
165 class Meta:
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800166 """Metadata for class Status."""
showardeab66ce2009-12-23 00:03:56 +0000167 db_table = 'tko_status'
showard35444862008-08-07 22:35:30 +0000168
169
jamesren708f1c02010-03-31 21:43:57 +0000170class Job(dbmodels.Model, model_logic.ModelExtensions):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800171 """Models a job."""
showardf8b19042009-05-12 17:22:49 +0000172 job_idx = dbmodels.AutoField(primary_key=True)
showard356d2372009-11-10 01:29:53 +0000173 tag = dbmodels.CharField(unique=True, max_length=100)
showarda5288b42009-07-28 20:06:08 +0000174 label = dbmodels.CharField(max_length=300)
175 username = dbmodels.CharField(max_length=240)
showard35444862008-08-07 22:35:30 +0000176 machine = dbmodels.ForeignKey(Machine, db_column='machine_idx')
177 queued_time = dbmodels.DateTimeField(null=True, blank=True)
178 started_time = dbmodels.DateTimeField(null=True, blank=True)
179 finished_time = dbmodels.DateTimeField(null=True, blank=True)
showardc1c1caf2009-09-08 16:26:50 +0000180 afe_job_id = dbmodels.IntegerField(null=True, default=None)
showard35444862008-08-07 22:35:30 +0000181
jamesren708f1c02010-03-31 21:43:57 +0000182 objects = model_logic.ExtendedManager()
183
showard35444862008-08-07 22:35:30 +0000184 class Meta:
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800185 """Metadata for class Job."""
showardeab66ce2009-12-23 00:03:56 +0000186 db_table = 'tko_jobs'
showard35444862008-08-07 22:35:30 +0000187
188
showardc1a98d12010-01-15 00:22:22 +0000189class JobKeyval(dbmodels.Model):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800190 """Models a job keyval."""
showardc1a98d12010-01-15 00:22:22 +0000191 job = dbmodels.ForeignKey(Job)
192 key = dbmodels.CharField(max_length=90)
193 value = dbmodels.CharField(blank=True, max_length=300)
194
showardc1a98d12010-01-15 00:22:22 +0000195 class Meta:
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800196 """Metadata for class JobKeyval."""
showardc1a98d12010-01-15 00:22:22 +0000197 db_table = 'tko_job_keyvals'
198
199
showardf8b19042009-05-12 17:22:49 +0000200class Test(dbmodels.Model, model_logic.ModelExtensions,
201 model_logic.ModelWithAttributes):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800202 """Models a test."""
showardf8b19042009-05-12 17:22:49 +0000203 test_idx = dbmodels.AutoField(primary_key=True)
showard35444862008-08-07 22:35:30 +0000204 job = dbmodels.ForeignKey(Job, db_column='job_idx')
lmr79697282010-03-31 23:18:02 +0000205 test = dbmodels.CharField(max_length=300)
206 subdir = dbmodels.CharField(blank=True, max_length=300)
showard35444862008-08-07 22:35:30 +0000207 kernel = dbmodels.ForeignKey(Kernel, db_column='kernel_idx')
208 status = dbmodels.ForeignKey(Status, db_column='status')
showarda5288b42009-07-28 20:06:08 +0000209 reason = dbmodels.CharField(blank=True, max_length=3072)
showard35444862008-08-07 22:35:30 +0000210 machine = dbmodels.ForeignKey(Machine, db_column='machine_idx')
211 finished_time = dbmodels.DateTimeField(null=True, blank=True)
212 started_time = dbmodels.DateTimeField(null=True, blank=True)
Fang Deng9ec66802014-04-28 19:04:33 +0000213 invalid = dbmodels.BooleanField(default=False)
214 invalidates_test = dbmodels.ForeignKey(
215 'self', null=True, db_column='invalidates_test_idx',
216 related_name='invalidates_test_set')
showard35444862008-08-07 22:35:30 +0000217
showardf8b19042009-05-12 17:22:49 +0000218 objects = model_logic.ExtendedManager()
219
220 def _get_attribute_model_and_args(self, attribute):
221 return TestAttribute, dict(test=self, attribute=attribute,
222 user_created=True)
223
224
225 def set_attribute(self, attribute, value):
226 # ensure non-user-created attributes remain immutable
227 try:
228 TestAttribute.objects.get(test=self, attribute=attribute,
229 user_created=False)
230 raise ValueError('Attribute %s already exists for test %s and is '
231 'immutable' % (attribute, self.test_idx))
232 except TestAttribute.DoesNotExist:
233 super(Test, self).set_attribute(attribute, value)
234
showard35444862008-08-07 22:35:30 +0000235 class Meta:
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800236 """Metadata for class Test."""
showardeab66ce2009-12-23 00:03:56 +0000237 db_table = 'tko_tests'
showard35444862008-08-07 22:35:30 +0000238
239
showarde732ee72008-09-23 19:15:43 +0000240class TestAttribute(dbmodels.Model, model_logic.ModelExtensions):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800241 """Models a test attribute."""
showardf8b19042009-05-12 17:22:49 +0000242 test = dbmodels.ForeignKey(Test, db_column='test_idx')
showarda5288b42009-07-28 20:06:08 +0000243 attribute = dbmodels.CharField(max_length=90)
244 value = dbmodels.CharField(blank=True, max_length=300)
showardf8b19042009-05-12 17:22:49 +0000245 user_created = dbmodels.BooleanField(default=False)
246
247 objects = model_logic.ExtendedManager()
showard35444862008-08-07 22:35:30 +0000248
249 class Meta:
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800250 """Metadata for class TestAttribute."""
showardeab66ce2009-12-23 00:03:56 +0000251 db_table = 'tko_test_attributes'
showard35444862008-08-07 22:35:30 +0000252
253
jadmanski430dca92008-12-16 20:56:53 +0000254class IterationAttribute(dbmodels.Model, model_logic.ModelExtensions):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800255 """Models an iteration attribute."""
256 # This isn't really a primary key, but it's necessary to appease Django
257 # and is harmless as long as we're careful.
showarde732ee72008-09-23 19:15:43 +0000258 test = dbmodels.ForeignKey(Test, db_column='test_idx', primary_key=True)
showard35444862008-08-07 22:35:30 +0000259 iteration = dbmodels.IntegerField()
showarda5288b42009-07-28 20:06:08 +0000260 attribute = dbmodels.CharField(max_length=90)
261 value = dbmodels.CharField(blank=True, max_length=300)
showard35444862008-08-07 22:35:30 +0000262
showardf8b19042009-05-12 17:22:49 +0000263 objects = model_logic.ExtendedManager()
264
showard35444862008-08-07 22:35:30 +0000265 class Meta:
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800266 """Metadata for class IterationAttribute."""
showardeab66ce2009-12-23 00:03:56 +0000267 db_table = 'tko_iteration_attributes'
showard35444862008-08-07 22:35:30 +0000268
269
jadmanski430dca92008-12-16 20:56:53 +0000270class IterationResult(dbmodels.Model, model_logic.ModelExtensions):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800271 """Models an iteration result."""
272 # See comment on IterationAttribute regarding primary_key=True.
jadmanski430dca92008-12-16 20:56:53 +0000273 test = dbmodels.ForeignKey(Test, db_column='test_idx', primary_key=True)
showard35444862008-08-07 22:35:30 +0000274 iteration = dbmodels.IntegerField()
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800275 attribute = dbmodels.CharField(max_length=256)
jamesren26f9a9f2010-04-02 17:44:55 +0000276 value = dbmodels.FloatField(null=True, blank=True)
showard35444862008-08-07 22:35:30 +0000277
showardf8b19042009-05-12 17:22:49 +0000278 objects = model_logic.ExtendedManager()
279
showard35444862008-08-07 22:35:30 +0000280 class Meta:
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800281 """Metadata for class IterationResult."""
showardeab66ce2009-12-23 00:03:56 +0000282 db_table = 'tko_iteration_result'
showard35444862008-08-07 22:35:30 +0000283
284
285class TestLabel(dbmodels.Model, model_logic.ModelExtensions):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800286 """Models a test label."""
showarda5288b42009-07-28 20:06:08 +0000287 name = dbmodels.CharField(max_length=80, unique=True)
showard35444862008-08-07 22:35:30 +0000288 description = dbmodels.TextField(blank=True)
showardeab66ce2009-12-23 00:03:56 +0000289 tests = dbmodels.ManyToManyField(Test, blank=True,
290 db_table='tko_test_labels_tests')
showard35444862008-08-07 22:35:30 +0000291
292 name_field = 'name'
showardf8b19042009-05-12 17:22:49 +0000293 objects = model_logic.ExtendedManager()
showard35444862008-08-07 22:35:30 +0000294
295 class Meta:
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800296 """Metadata for class TestLabel."""
showardeab66ce2009-12-23 00:03:56 +0000297 db_table = 'tko_test_labels'
showard35444862008-08-07 22:35:30 +0000298
299
300class SavedQuery(dbmodels.Model, model_logic.ModelExtensions):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800301 """Models a saved query."""
302 # TODO: change this to foreign key once DBs are merged.
showarda5288b42009-07-28 20:06:08 +0000303 owner = dbmodels.CharField(max_length=80)
304 name = dbmodels.CharField(max_length=100)
showard35444862008-08-07 22:35:30 +0000305 url_token = dbmodels.TextField()
306
307 class Meta:
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800308 """Metadata for class SavedQuery."""
showardeab66ce2009-12-23 00:03:56 +0000309 db_table = 'tko_saved_queries'
showard35444862008-08-07 22:35:30 +0000310
311
showardce12f552008-09-19 00:48:59 +0000312class EmbeddedGraphingQuery(dbmodels.Model, model_logic.ModelExtensions):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800313 """Models an embedded graphing query."""
showardce12f552008-09-19 00:48:59 +0000314 url_token = dbmodels.TextField(null=False, blank=False)
showarda5288b42009-07-28 20:06:08 +0000315 graph_type = dbmodels.CharField(max_length=16, null=False, blank=False)
showardce12f552008-09-19 00:48:59 +0000316 params = dbmodels.TextField(null=False, blank=False)
317 last_updated = dbmodels.DateTimeField(null=False, blank=False,
318 editable=False)
319 # refresh_time shows the time at which a thread is updating the cached
320 # image, or NULL if no one is updating the image. This is used so that only
321 # one thread is updating the cached image at a time (see
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800322 # graphing_utils.handle_plot_request).
showardce12f552008-09-19 00:48:59 +0000323 refresh_time = dbmodels.DateTimeField(editable=False)
324 cached_png = dbmodels.TextField(editable=False)
325
326 class Meta:
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800327 """Metadata for class EmbeddedGraphingQuery."""
showardeab66ce2009-12-23 00:03:56 +0000328 db_table = 'tko_embedded_graphing_queries'
showardce12f552008-09-19 00:48:59 +0000329
330
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800331# Views.
showard35444862008-08-07 22:35:30 +0000332
333class TestViewManager(TempManager):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800334 """A Test View Manager."""
335
showard35444862008-08-07 22:35:30 +0000336 def get_query_set(self):
337 query = super(TestViewManager, self).get_query_set()
showard5bf7c502008-08-20 01:22:22 +0000338
showard35444862008-08-07 22:35:30 +0000339 # add extra fields to selects, using the SQL itself as the "alias"
340 extra_select = dict((sql, sql)
341 for sql in self.model.extra_fields.iterkeys())
342 return query.extra(select=extra_select)
343
344
showardf2489522008-10-23 23:08:00 +0000345 def _get_include_exclude_suffix(self, exclude):
346 if exclude:
showard2aa318e2009-08-20 23:43:10 +0000347 return '_exclude'
348 return '_include'
349
350
351 def _add_attribute_join(self, query_set, join_condition,
352 suffix=None, exclude=False):
353 if suffix is None:
354 suffix = self._get_include_exclude_suffix(exclude)
showardeab66ce2009-12-23 00:03:56 +0000355 return self.add_join(query_set, 'tko_test_attributes',
356 join_key='test_idx',
showard2aa318e2009-08-20 23:43:10 +0000357 join_condition=join_condition,
358 suffix=suffix, exclude=exclude)
359
360
361 def _add_label_pivot_table_join(self, query_set, suffix, join_condition='',
362 exclude=False, force_left_join=False):
showardeab66ce2009-12-23 00:03:56 +0000363 return self.add_join(query_set, 'tko_test_labels_tests',
364 join_key='test_id',
showard2aa318e2009-08-20 23:43:10 +0000365 join_condition=join_condition,
366 suffix=suffix, exclude=exclude,
367 force_left_join=force_left_join)
showardf2489522008-10-23 23:08:00 +0000368
369
showard64aeecd2008-09-19 21:32:58 +0000370 def _add_label_joins(self, query_set, suffix=''):
showard2aa318e2009-08-20 23:43:10 +0000371 query_set = self._add_label_pivot_table_join(
372 query_set, suffix=suffix, force_left_join=True)
showardd50ffb42008-09-04 02:47:45 +0000373
showard2aa318e2009-08-20 23:43:10 +0000374 # since we're not joining from the original table, we can't use
375 # self.add_join() again
showardeab66ce2009-12-23 00:03:56 +0000376 second_join_alias = 'tko_test_labels' + suffix
showardd50ffb42008-09-04 02:47:45 +0000377 second_join_condition = ('%s.id = %s.testlabel_id' %
showard64aeecd2008-09-19 21:32:58 +0000378 (second_join_alias,
showardeab66ce2009-12-23 00:03:56 +0000379 'tko_test_labels_tests' + suffix))
showard7e67b432010-01-20 01:13:04 +0000380 query_set.query.add_custom_join('tko_test_labels',
381 second_join_condition,
382 query_set.query.LOUTER,
383 alias=second_join_alias)
384 return query_set
showardd50ffb42008-09-04 02:47:45 +0000385
showard64aeecd2008-09-19 21:32:58 +0000386
showardd50ffb42008-09-04 02:47:45 +0000387 def _get_label_ids_from_names(self, label_names):
showard2aa318e2009-08-20 23:43:10 +0000388 label_ids = list( # listifying avoids a double query below
showard8b0ea222009-12-23 19:23:03 +0000389 TestLabel.objects.filter(name__in=label_names)
390 .values_list('name', 'id'))
showard2aa318e2009-08-20 23:43:10 +0000391 if len(label_ids) < len(set(label_names)):
lmr79697282010-03-31 23:18:02 +0000392 raise ValueError('Not all labels found: %s' %
393 ', '.join(label_names))
showard8b0ea222009-12-23 19:23:03 +0000394 return dict(name_and_id for name_and_id in label_ids)
showard2aa318e2009-08-20 23:43:10 +0000395
396
397 def _include_or_exclude_labels(self, query_set, label_names, exclude=False):
showard8b0ea222009-12-23 19:23:03 +0000398 label_ids = self._get_label_ids_from_names(label_names).itervalues()
showard2aa318e2009-08-20 23:43:10 +0000399 suffix = self._get_include_exclude_suffix(exclude)
showardeab66ce2009-12-23 00:03:56 +0000400 condition = ('tko_test_labels_tests%s.testlabel_id IN (%s)' %
showard8b0ea222009-12-23 19:23:03 +0000401 (suffix,
402 ','.join(str(label_id) for label_id in label_ids)))
showard2aa318e2009-08-20 23:43:10 +0000403 return self._add_label_pivot_table_join(query_set,
404 join_condition=condition,
405 suffix=suffix,
406 exclude=exclude)
showard02813502008-08-20 20:52:56 +0000407
408
showard8b0ea222009-12-23 19:23:03 +0000409 def _add_custom_select(self, query_set, select_name, select_sql):
410 return query_set.extra(select={select_name: select_sql})
411
412
413 def _add_select_value(self, query_set, alias):
414 return self._add_custom_select(query_set, alias,
415 _quote_name(alias) + '.value')
416
417
418 def _add_select_ifnull(self, query_set, alias, non_null_value):
419 select_sql = "IF(%s.id IS NOT NULL, '%s', NULL)" % (_quote_name(alias),
420 non_null_value)
421 return self._add_custom_select(query_set, alias, select_sql)
422
423
jamesren708f1c02010-03-31 21:43:57 +0000424 def _join_test_label_column(self, query_set, label_name, label_id):
425 alias = 'test_label_' + label_name
showard7e67b432010-01-20 01:13:04 +0000426 label_query = TestLabel.objects.filter(name=label_name)
427 query_set = Test.objects.join_custom_field(query_set, label_query,
428 alias)
showard8b0ea222009-12-23 19:23:03 +0000429
430 query_set = self._add_select_ifnull(query_set, alias, label_name)
431 return query_set
432
433
jamesren708f1c02010-03-31 21:43:57 +0000434 def _join_test_label_columns(self, query_set, label_names):
showard8b0ea222009-12-23 19:23:03 +0000435 label_id_map = self._get_label_ids_from_names(label_names)
436 for label_name in label_names:
jamesren708f1c02010-03-31 21:43:57 +0000437 query_set = self._join_test_label_column(query_set, label_name,
438 label_id_map[label_name])
showard8b0ea222009-12-23 19:23:03 +0000439 return query_set
440
441
jamesren708f1c02010-03-31 21:43:57 +0000442 def _join_test_attribute(self, query_set, attribute, alias=None,
443 extra_join_condition=None):
showard8b0ea222009-12-23 19:23:03 +0000444 """
445 Join the given TestView QuerySet to TestAttribute. The resulting query
446 has an additional column for the given attribute named
447 "attribute_<attribute name>".
448 """
showard8b0ea222009-12-23 19:23:03 +0000449 if not alias:
jamesren708f1c02010-03-31 21:43:57 +0000450 alias = 'test_attribute_' + attribute
showard7e67b432010-01-20 01:13:04 +0000451 attribute_query = TestAttribute.objects.filter(attribute=attribute)
showard8b0ea222009-12-23 19:23:03 +0000452 if extra_join_condition:
showard7e67b432010-01-20 01:13:04 +0000453 attribute_query = attribute_query.extra(
454 where=[extra_join_condition])
455 query_set = Test.objects.join_custom_field(query_set, attribute_query,
456 alias)
showard8b0ea222009-12-23 19:23:03 +0000457
458 query_set = self._add_select_value(query_set, alias)
459 return query_set
460
461
462 def _join_machine_label_columns(self, query_set, machine_label_names):
463 for label_name in machine_label_names:
464 alias = 'machine_label_' + label_name
465 condition = "FIND_IN_SET('%s', %s)" % (
466 label_name, _quote_name(alias) + '.value')
jamesren708f1c02010-03-31 21:43:57 +0000467 query_set = self._join_test_attribute(
468 query_set, 'host-labels',
469 alias=alias, extra_join_condition=condition)
showard8b0ea222009-12-23 19:23:03 +0000470 query_set = self._add_select_ifnull(query_set, alias, label_name)
471 return query_set
472
473
474 def _join_one_iteration_key(self, query_set, result_key, first_alias=None):
jamesren708f1c02010-03-31 21:43:57 +0000475 alias = 'iteration_result_' + result_key
showard7e67b432010-01-20 01:13:04 +0000476 iteration_query = IterationResult.objects.filter(attribute=result_key)
showard8b0ea222009-12-23 19:23:03 +0000477 if first_alias:
478 # after the first join, we need to match up iteration indices,
479 # otherwise each join will expand the query by the number of
480 # iterations and we'll have extraneous rows
showard7e67b432010-01-20 01:13:04 +0000481 iteration_query = iteration_query.extra(
482 where=['%s.iteration = %s.iteration'
483 % (_quote_name(alias), _quote_name(first_alias))])
showard8b0ea222009-12-23 19:23:03 +0000484
showard7e67b432010-01-20 01:13:04 +0000485 query_set = Test.objects.join_custom_field(query_set, iteration_query,
486 alias, left_join=False)
showard8b0ea222009-12-23 19:23:03 +0000487 # select the iteration value and index for this join
488 query_set = self._add_select_value(query_set, alias)
489 if not first_alias:
490 # for first join, add iteration index select too
491 query_set = self._add_custom_select(
492 query_set, 'iteration_index',
493 _quote_name(alias) + '.iteration')
494
495 return query_set, alias
496
497
jamesren708f1c02010-03-31 21:43:57 +0000498 def _join_iteration_results(self, test_view_query_set, result_keys):
showard8b0ea222009-12-23 19:23:03 +0000499 """Join the given TestView QuerySet to IterationResult for one result.
500
501 The resulting query looks like a TestView query but has one row per
502 iteration. Each row includes all the attributes of TestView, an
503 attribute for each key in result_keys and an iteration_index attribute.
504
505 We accomplish this by joining the TestView query to IterationResult
506 once per result key. Each join is restricted on the result key (and on
507 the test index, like all one-to-many joins). For the first join, this
508 is the only restriction, so each TestView row expands to a row per
509 iteration (per iteration that includes the key, of course). For each
510 subsequent join, we also restrict the iteration index to match that of
511 the initial join. This makes each subsequent join produce exactly one
512 result row for each input row. (This assumes each iteration contains
513 the same set of keys. Results are undefined if that's not true.)
514 """
515 if not result_keys:
516 return test_view_query_set
517
518 query_set, first_alias = self._join_one_iteration_key(
519 test_view_query_set, result_keys[0])
520 for result_key in result_keys[1:]:
521 query_set, _ = self._join_one_iteration_key(query_set, result_key,
522 first_alias=first_alias)
523 return query_set
524
525
jamesren708f1c02010-03-31 21:43:57 +0000526 def _join_job_keyvals(self, query_set, job_keyvals):
527 for job_keyval in job_keyvals:
528 alias = 'job_keyval_' + job_keyval
529 keyval_query = JobKeyval.objects.filter(key=job_keyval)
530 query_set = Job.objects.join_custom_field(query_set, keyval_query,
531 alias)
532 query_set = self._add_select_value(query_set, alias)
533 return query_set
534
535
536 def _join_iteration_attributes(self, query_set, iteration_attributes):
537 for attribute in iteration_attributes:
538 alias = 'iteration_attribute_' + attribute
539 attribute_query = IterationAttribute.objects.filter(
540 attribute=attribute)
541 query_set = Test.objects.join_custom_field(query_set,
542 attribute_query, alias)
543 query_set = self._add_select_value(query_set, alias)
544 return query_set
545
546
showard8b0ea222009-12-23 19:23:03 +0000547 def get_query_set_with_joins(self, filter_data):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800548 """Add joins for querying over test-related items.
showard8b0ea222009-12-23 19:23:03 +0000549
550 These parameters are supported going forward:
551 * test_attribute_fields: list of attribute names. Each attribute will
552 be available as a column attribute_<name>.value.
553 * test_label_fields: list of label names. Each label will be available
554 as a column label_<name>.id, non-null iff the label is present.
jamesren708f1c02010-03-31 21:43:57 +0000555 * iteration_result_fields: list of iteration result names. Each
showard8b0ea222009-12-23 19:23:03 +0000556 result will be available as a column iteration_<name>.value.
557 Note that this changes the semantics to return iterations
558 instead of tests -- if a test has multiple iterations, a row
559 will be returned for each one. The iteration index is also
560 available as iteration_<name>.iteration.
561 * machine_label_fields: list of machine label names. Each will be
562 available as a column machine_label_<name>.id, non-null iff the
563 label is present on the machine used in the test.
jamesren708f1c02010-03-31 21:43:57 +0000564 * job_keyval_fields: list of job keyval names. Each value will be
565 available as a column job_keyval_<name>.id, non-null iff the
566 keyval is present in the AFE job.
567 * iteration_attribute_fields: list of iteration attribute names. Each
568 attribute will be available as a column
569 iteration_attribute<name>.id, non-null iff the attribute is
570 present.
showard8b0ea222009-12-23 19:23:03 +0000571
572 These parameters are deprecated:
573 * include_labels
574 * exclude_labels
575 * include_attributes_where
576 * exclude_attributes_where
577
578 Additionally, this method adds joins if the following strings are
579 present in extra_where (this is also deprecated):
580 * test_labels
581 * test_attributes_host_labels
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800582
583 @param filter_data: Data by which to filter.
584
585 @return A QuerySet.
586
showard8b0ea222009-12-23 19:23:03 +0000587 """
showard35444862008-08-07 22:35:30 +0000588 query_set = self.get_query_set()
showard8b0ea222009-12-23 19:23:03 +0000589
590 test_attributes = filter_data.pop('test_attribute_fields', [])
591 for attribute in test_attributes:
jamesren708f1c02010-03-31 21:43:57 +0000592 query_set = self._join_test_attribute(query_set, attribute)
showard8b0ea222009-12-23 19:23:03 +0000593
594 test_labels = filter_data.pop('test_label_fields', [])
jamesren708f1c02010-03-31 21:43:57 +0000595 query_set = self._join_test_label_columns(query_set, test_labels)
showard8b0ea222009-12-23 19:23:03 +0000596
597 machine_labels = filter_data.pop('machine_label_fields', [])
598 query_set = self._join_machine_label_columns(query_set, machine_labels)
599
jamesren708f1c02010-03-31 21:43:57 +0000600 iteration_keys = filter_data.pop('iteration_result_fields', [])
601 query_set = self._join_iteration_results(query_set, iteration_keys)
602
603 job_keyvals = filter_data.pop('job_keyval_fields', [])
604 query_set = self._join_job_keyvals(query_set, job_keyvals)
605
606 iteration_attributes = filter_data.pop('iteration_attribute_fields', [])
607 query_set = self._join_iteration_attributes(query_set,
608 iteration_attributes)
showard8b0ea222009-12-23 19:23:03 +0000609
610 # everything that follows is deprecated behavior
611
showardd50ffb42008-09-04 02:47:45 +0000612 joined = False
showard2aa318e2009-08-20 23:43:10 +0000613
showardf2489522008-10-23 23:08:00 +0000614 extra_where = filter_data.get('extra_where', '')
showardeab66ce2009-12-23 00:03:56 +0000615 if 'tko_test_labels' in extra_where:
showard02813502008-08-20 20:52:56 +0000616 query_set = self._add_label_joins(query_set)
showardd50ffb42008-09-04 02:47:45 +0000617 joined = True
618
showard8b0ea222009-12-23 19:23:03 +0000619 include_labels = filter_data.pop('include_labels', [])
620 exclude_labels = filter_data.pop('exclude_labels', [])
showard2aa318e2009-08-20 23:43:10 +0000621 if include_labels:
622 query_set = self._include_or_exclude_labels(query_set,
623 include_labels)
showardfc8c6ae2008-11-11 19:06:01 +0000624 joined = True
showard2aa318e2009-08-20 23:43:10 +0000625 if exclude_labels:
626 query_set = self._include_or_exclude_labels(query_set,
627 exclude_labels,
628 exclude=True)
showard64aeecd2008-09-19 21:32:58 +0000629 joined = True
630
631 include_attributes_where = filter_data.pop('include_attributes_where',
632 '')
633 exclude_attributes_where = filter_data.pop('exclude_attributes_where',
634 '')
635 if include_attributes_where:
636 query_set = self._add_attribute_join(
showard2aa318e2009-08-20 23:43:10 +0000637 query_set,
638 join_condition=self.escape_user_sql(include_attributes_where))
showard64aeecd2008-09-19 21:32:58 +0000639 joined = True
640 if exclude_attributes_where:
641 query_set = self._add_attribute_join(
showard2aa318e2009-08-20 23:43:10 +0000642 query_set,
643 join_condition=self.escape_user_sql(exclude_attributes_where),
showard64aeecd2008-09-19 21:32:58 +0000644 exclude=True)
645 joined = True
showardd50ffb42008-09-04 02:47:45 +0000646
647 if not joined:
showard35444862008-08-07 22:35:30 +0000648 filter_data['no_distinct'] = True
showardd50ffb42008-09-04 02:47:45 +0000649
showard8b0ea222009-12-23 19:23:03 +0000650 if 'tko_test_attributes_host_labels' in extra_where:
showardf2489522008-10-23 23:08:00 +0000651 query_set = self._add_attribute_join(
652 query_set, suffix='_host_labels',
showardeab66ce2009-12-23 00:03:56 +0000653 join_condition='tko_test_attributes_host_labels.attribute = '
showardf2489522008-10-23 23:08:00 +0000654 '"host-labels"')
655
showard35444862008-08-07 22:35:30 +0000656 return query_set
657
658
showard8bfb5cb2009-10-07 20:49:15 +0000659 def query_test_ids(self, filter_data, apply_presentation=True):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800660 """Queries for test IDs.
661
662 @param filter_data: Data by which to filter.
663 @param apply_presentation: Whether or not to apply presentation
664 parameters.
665
666 @return A list of test IDs.
667
668 """
showard8bfb5cb2009-10-07 20:49:15 +0000669 query = self.model.query_objects(filter_data,
670 apply_presentation=apply_presentation)
671 dicts = query.values('test_idx')
showard02813502008-08-20 20:52:56 +0000672 return [item['test_idx'] for item in dicts]
673
674
showard02813502008-08-20 20:52:56 +0000675 def query_test_label_ids(self, filter_data):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800676 """Queries for test label IDs.
677
678 @param filter_data: Data by which to filter.
679
680 @return A list of test label IDs.
681
682 """
showardd50ffb42008-09-04 02:47:45 +0000683 query_set = self.model.query_objects(filter_data)
684 query_set = self._add_label_joins(query_set, suffix='_list')
showardeab66ce2009-12-23 00:03:56 +0000685 rows = self._custom_select_query(query_set, ['tko_test_labels_list.id'])
showardd50ffb42008-09-04 02:47:45 +0000686 return [row[0] for row in rows if row[0] is not None]
showard02813502008-08-20 20:52:56 +0000687
688
showardeaccf8f2009-04-16 03:11:33 +0000689 def escape_user_sql(self, sql):
690 sql = super(TestViewManager, self).escape_user_sql(sql)
691 return sql.replace('test_idx', self.get_key_on_this_table('test_idx'))
692
693
showard35444862008-08-07 22:35:30 +0000694class TestView(dbmodels.Model, model_logic.ModelExtensions):
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800695 """Models a test view."""
showard35444862008-08-07 22:35:30 +0000696 extra_fields = {
showardf4c702e2009-07-08 21:14:27 +0000697 'DATE(job_queued_time)': 'job queued day',
698 'DATE(test_finished_time)': 'test finished day',
showard35444862008-08-07 22:35:30 +0000699 }
700
701 group_fields = [
showardf4c702e2009-07-08 21:14:27 +0000702 'test_name',
703 'status',
704 'kernel',
705 'hostname',
706 'job_tag',
707 'job_name',
708 'platform',
709 'reason',
710 'job_owner',
711 'job_queued_time',
712 'DATE(job_queued_time)',
713 'test_started_time',
714 'test_finished_time',
715 'DATE(test_finished_time)',
showard35444862008-08-07 22:35:30 +0000716 ]
717
718 test_idx = dbmodels.IntegerField('test index', primary_key=True)
719 job_idx = dbmodels.IntegerField('job index', null=True, blank=True)
showarda5288b42009-07-28 20:06:08 +0000720 test_name = dbmodels.CharField(blank=True, max_length=90)
721 subdir = dbmodels.CharField('subdirectory', blank=True, max_length=180)
showard35444862008-08-07 22:35:30 +0000722 kernel_idx = dbmodels.IntegerField('kernel index')
723 status_idx = dbmodels.IntegerField('status index')
showarda5288b42009-07-28 20:06:08 +0000724 reason = dbmodels.CharField(blank=True, max_length=3072)
showard35444862008-08-07 22:35:30 +0000725 machine_idx = dbmodels.IntegerField('host index')
726 test_started_time = dbmodels.DateTimeField(null=True, blank=True)
727 test_finished_time = dbmodels.DateTimeField(null=True, blank=True)
showarda5288b42009-07-28 20:06:08 +0000728 job_tag = dbmodels.CharField(blank=True, max_length=300)
729 job_name = dbmodels.CharField(blank=True, max_length=300)
730 job_owner = dbmodels.CharField('owner', blank=True, max_length=240)
showard35444862008-08-07 22:35:30 +0000731 job_queued_time = dbmodels.DateTimeField(null=True, blank=True)
732 job_started_time = dbmodels.DateTimeField(null=True, blank=True)
733 job_finished_time = dbmodels.DateTimeField(null=True, blank=True)
showardc1c1caf2009-09-08 16:26:50 +0000734 afe_job_id = dbmodels.IntegerField(null=True)
showarda5288b42009-07-28 20:06:08 +0000735 hostname = dbmodels.CharField(blank=True, max_length=300)
736 platform = dbmodels.CharField(blank=True, max_length=240)
737 machine_owner = dbmodels.CharField(blank=True, max_length=240)
738 kernel_hash = dbmodels.CharField(blank=True, max_length=105)
739 kernel_base = dbmodels.CharField(blank=True, max_length=90)
740 kernel = dbmodels.CharField(blank=True, max_length=300)
741 status = dbmodels.CharField(blank=True, max_length=30)
showard35444862008-08-07 22:35:30 +0000742
743 objects = TestViewManager()
744
745 def save(self):
746 raise NotImplementedError('TestView is read-only')
747
748
749 def delete(self):
750 raise NotImplementedError('TestView is read-only')
751
752
753 @classmethod
showard8bfb5cb2009-10-07 20:49:15 +0000754 def query_objects(cls, filter_data, initial_query=None,
755 apply_presentation=True):
showard35444862008-08-07 22:35:30 +0000756 if initial_query is None:
showard64aeecd2008-09-19 21:32:58 +0000757 initial_query = cls.objects.get_query_set_with_joins(filter_data)
showard8bfb5cb2009-10-07 20:49:15 +0000758 return super(TestView, cls).query_objects(
759 filter_data, initial_query=initial_query,
760 apply_presentation=apply_presentation)
showard35444862008-08-07 22:35:30 +0000761
showard35444862008-08-07 22:35:30 +0000762 class Meta:
Dennis Jeffrey8bd3e982013-03-05 12:46:32 -0800763 """Metadata for class TestView."""
showardeab66ce2009-12-23 00:03:56 +0000764 db_table = 'tko_test_view_2'