showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 1 | from django.db import models as dbmodels, connection |
| 2 | from django.utils import datastructures |
| 3 | from autotest_lib.frontend.afe import model_logic, readonly_connection |
| 4 | |
| 5 | class TempManager(model_logic.ExtendedManager): |
| 6 | _GROUP_COUNT_NAME = 'group_count' |
| 7 | |
| 8 | def _get_key_unless_is_function(self, field): |
| 9 | if '(' in field: |
| 10 | return field |
showard | 7c199df | 2008-10-03 10:17:15 +0000 | [diff] [blame] | 11 | return self.get_key_on_this_table(field) |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 12 | |
| 13 | |
showard | f248952 | 2008-10-23 23:08:00 +0000 | [diff] [blame] | 14 | def _get_field_names(self, fields, extra_select_fields={}): |
| 15 | field_names = [] |
| 16 | for field in fields: |
| 17 | if field in extra_select_fields: |
showard | d2b0c88 | 2009-10-19 18:34:11 +0000 | [diff] [blame] | 18 | field_names.append(extra_select_fields[field][0]) |
showard | f248952 | 2008-10-23 23:08:00 +0000 | [diff] [blame] | 19 | else: |
| 20 | field_names.append(self._get_key_unless_is_function(field)) |
| 21 | return field_names |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 22 | |
| 23 | |
showard | 8bfb5cb | 2009-10-07 20:49:15 +0000 | [diff] [blame] | 24 | def _get_group_query_sql(self, query, group_by): |
showard | a5288b4 | 2009-07-28 20:06:08 +0000 | [diff] [blame] | 25 | sql, params = query.query.as_sql() |
showard | d50ffb4 | 2008-09-04 02:47:45 +0000 | [diff] [blame] | 26 | |
| 27 | # insert GROUP BY clause into query |
showard | 8bfb5cb | 2009-10-07 20:49:15 +0000 | [diff] [blame] | 28 | group_fields = self._get_field_names(group_by, query.query.extra_select) |
showard | 7c199df | 2008-10-03 10:17:15 +0000 | [diff] [blame] | 29 | group_by_clause = ' GROUP BY ' + ', '.join(group_fields) |
showard | a5288b4 | 2009-07-28 20:06:08 +0000 | [diff] [blame] | 30 | group_by_position = sql.rfind('ORDER BY') |
showard | d50ffb4 | 2008-09-04 02:47:45 +0000 | [diff] [blame] | 31 | if group_by_position == -1: |
showard | a5288b4 | 2009-07-28 20:06:08 +0000 | [diff] [blame] | 32 | group_by_position = len(sql) |
| 33 | sql = (sql[:group_by_position] + |
| 34 | group_by_clause + ' ' + |
| 35 | sql[group_by_position:]) |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 36 | |
showard | a5288b4 | 2009-07-28 20:06:08 +0000 | [diff] [blame] | 37 | return sql, params |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 38 | |
| 39 | |
showard | 06b82fc | 2009-06-30 01:59:42 +0000 | [diff] [blame] | 40 | def _get_column_names(self, cursor): |
showard | 8bfb5cb | 2009-10-07 20:49:15 +0000 | [diff] [blame] | 41 | """ |
showard | 06b82fc | 2009-06-30 01:59:42 +0000 | [diff] [blame] | 42 | Gets the column names from the cursor description. This method exists |
showard | 8bfb5cb | 2009-10-07 20:49:15 +0000 | [diff] [blame] | 43 | so that it can be mocked in the unit test for sqlite3 compatibility. |
showard | 06b82fc | 2009-06-30 01:59:42 +0000 | [diff] [blame] | 44 | """ |
| 45 | return [column_info[0] for column_info in cursor.description] |
| 46 | |
| 47 | |
showard | 8bfb5cb | 2009-10-07 20:49:15 +0000 | [diff] [blame] | 48 | def execute_group_query(self, query, group_by): |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 49 | """ |
showard | 8a6eb0c | 2008-10-01 11:38:59 +0000 | [diff] [blame] | 50 | Performs the given query grouped by the fields in group_by with the |
showard | 8bfb5cb | 2009-10-07 20:49:15 +0000 | [diff] [blame] | 51 | given query's extra select fields added. Returns a list of dicts, where |
| 52 | each dict corresponds to single row and contains a key for each grouped |
| 53 | field as well as all of the extra select fields. |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 54 | """ |
showard | 8bfb5cb | 2009-10-07 20:49:15 +0000 | [diff] [blame] | 55 | sql, params = self._get_group_query_sql(query, group_by) |
showard | 56e9377 | 2008-10-06 10:06:22 +0000 | [diff] [blame] | 56 | cursor = readonly_connection.connection().cursor() |
showard | 8a6eb0c | 2008-10-01 11:38:59 +0000 | [diff] [blame] | 57 | cursor.execute(sql, params) |
showard | 06b82fc | 2009-06-30 01:59:42 +0000 | [diff] [blame] | 58 | field_names = self._get_column_names(cursor) |
showard | 8a6eb0c | 2008-10-01 11:38:59 +0000 | [diff] [blame] | 59 | row_dicts = [dict(zip(field_names, row)) for row in cursor.fetchall()] |
| 60 | return row_dicts |
| 61 | |
| 62 | |
| 63 | def get_count_sql(self, query): |
| 64 | """ |
| 65 | Get the SQL to properly select a per-group count of unique matches for |
showard | 7c199df | 2008-10-03 10:17:15 +0000 | [diff] [blame] | 66 | a grouped query. Returns a tuple (field alias, field SQL) |
showard | 8a6eb0c | 2008-10-01 11:38:59 +0000 | [diff] [blame] | 67 | """ |
showard | a5288b4 | 2009-07-28 20:06:08 +0000 | [diff] [blame] | 68 | if query.query.distinct: |
showard | 7c199df | 2008-10-03 10:17:15 +0000 | [diff] [blame] | 69 | pk_field = self.get_key_on_this_table() |
showard | 8a6eb0c | 2008-10-01 11:38:59 +0000 | [diff] [blame] | 70 | count_sql = 'COUNT(DISTINCT %s)' % pk_field |
| 71 | else: |
| 72 | count_sql = 'COUNT(1)' |
showard | 7c199df | 2008-10-03 10:17:15 +0000 | [diff] [blame] | 73 | return self._GROUP_COUNT_NAME, count_sql |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 74 | |
| 75 | |
| 76 | def _get_num_groups_sql(self, query, group_by): |
showard | d2b0c88 | 2009-10-19 18:34:11 +0000 | [diff] [blame] | 77 | group_fields = self._get_field_names(group_by, query.query.extra_select) |
showard | a5288b4 | 2009-07-28 20:06:08 +0000 | [diff] [blame] | 78 | query = query.order_by() # this can mess up the query and isn't needed |
| 79 | |
| 80 | sql, params = query.query.as_sql() |
| 81 | from_ = sql[sql.find(' FROM'):] |
showard | d2b0c88 | 2009-10-19 18:34:11 +0000 | [diff] [blame] | 82 | return ('SELECT DISTINCT %s %s' % (','.join(group_fields), |
showard | a5288b4 | 2009-07-28 20:06:08 +0000 | [diff] [blame] | 83 | from_), |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 84 | params) |
| 85 | |
| 86 | |
showard | d2b0c88 | 2009-10-19 18:34:11 +0000 | [diff] [blame] | 87 | def _cursor_rowcount(self, cursor): |
| 88 | """To be stubbed by tests""" |
| 89 | return cursor.rowcount |
| 90 | |
| 91 | |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 92 | def get_num_groups(self, query, group_by): |
| 93 | """ |
| 94 | Returns the number of distinct groups for the given query grouped by the |
| 95 | fields in group_by. |
| 96 | """ |
| 97 | sql, params = self._get_num_groups_sql(query, group_by) |
showard | 56e9377 | 2008-10-06 10:06:22 +0000 | [diff] [blame] | 98 | cursor = readonly_connection.connection().cursor() |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 99 | cursor.execute(sql, params) |
showard | d2b0c88 | 2009-10-19 18:34:11 +0000 | [diff] [blame] | 100 | return self._cursor_rowcount(cursor) |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 101 | |
| 102 | |
| 103 | class Machine(dbmodels.Model): |
showard | f8b1904 | 2009-05-12 17:22:49 +0000 | [diff] [blame] | 104 | machine_idx = dbmodels.AutoField(primary_key=True) |
showard | 356d237 | 2009-11-10 01:29:53 +0000 | [diff] [blame^] | 105 | hostname = dbmodels.CharField(unique=True, max_length=255) |
showard | a5288b4 | 2009-07-28 20:06:08 +0000 | [diff] [blame] | 106 | machine_group = dbmodels.CharField(blank=True, max_length=240) |
| 107 | owner = dbmodels.CharField(blank=True, max_length=240) |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 108 | |
| 109 | class Meta: |
| 110 | db_table = 'machines' |
| 111 | |
| 112 | |
| 113 | class Kernel(dbmodels.Model): |
showard | f8b1904 | 2009-05-12 17:22:49 +0000 | [diff] [blame] | 114 | kernel_idx = dbmodels.AutoField(primary_key=True) |
showard | a5288b4 | 2009-07-28 20:06:08 +0000 | [diff] [blame] | 115 | kernel_hash = dbmodels.CharField(max_length=105, editable=False) |
| 116 | base = dbmodels.CharField(max_length=90) |
| 117 | printable = dbmodels.CharField(max_length=300) |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 118 | |
| 119 | class Meta: |
| 120 | db_table = 'kernels' |
| 121 | |
| 122 | |
| 123 | class Patch(dbmodels.Model): |
| 124 | kernel = dbmodels.ForeignKey(Kernel, db_column='kernel_idx') |
showard | a5288b4 | 2009-07-28 20:06:08 +0000 | [diff] [blame] | 125 | name = dbmodels.CharField(blank=True, max_length=240) |
| 126 | url = dbmodels.CharField(blank=True, max_length=900) |
| 127 | the_hash = dbmodels.CharField(blank=True, max_length=105, db_column='hash') |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 128 | |
| 129 | class Meta: |
| 130 | db_table = 'patches' |
| 131 | |
| 132 | |
| 133 | class Status(dbmodels.Model): |
showard | f8b1904 | 2009-05-12 17:22:49 +0000 | [diff] [blame] | 134 | status_idx = dbmodels.AutoField(primary_key=True) |
showard | a5288b4 | 2009-07-28 20:06:08 +0000 | [diff] [blame] | 135 | word = dbmodels.CharField(max_length=30) |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 136 | |
| 137 | class Meta: |
| 138 | db_table = 'status' |
| 139 | |
| 140 | |
| 141 | class Job(dbmodels.Model): |
showard | f8b1904 | 2009-05-12 17:22:49 +0000 | [diff] [blame] | 142 | job_idx = dbmodels.AutoField(primary_key=True) |
showard | 356d237 | 2009-11-10 01:29:53 +0000 | [diff] [blame^] | 143 | tag = dbmodels.CharField(unique=True, max_length=100) |
showard | a5288b4 | 2009-07-28 20:06:08 +0000 | [diff] [blame] | 144 | label = dbmodels.CharField(max_length=300) |
| 145 | username = dbmodels.CharField(max_length=240) |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 146 | machine = dbmodels.ForeignKey(Machine, db_column='machine_idx') |
| 147 | queued_time = dbmodels.DateTimeField(null=True, blank=True) |
| 148 | started_time = dbmodels.DateTimeField(null=True, blank=True) |
| 149 | finished_time = dbmodels.DateTimeField(null=True, blank=True) |
showard | c1c1caf | 2009-09-08 16:26:50 +0000 | [diff] [blame] | 150 | afe_job_id = dbmodels.IntegerField(null=True, default=None) |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 151 | |
| 152 | class Meta: |
| 153 | db_table = 'jobs' |
| 154 | |
| 155 | |
showard | f8b1904 | 2009-05-12 17:22:49 +0000 | [diff] [blame] | 156 | class Test(dbmodels.Model, model_logic.ModelExtensions, |
| 157 | model_logic.ModelWithAttributes): |
| 158 | test_idx = dbmodels.AutoField(primary_key=True) |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 159 | job = dbmodels.ForeignKey(Job, db_column='job_idx') |
showard | a5288b4 | 2009-07-28 20:06:08 +0000 | [diff] [blame] | 160 | test = dbmodels.CharField(max_length=90) |
| 161 | subdir = dbmodels.CharField(blank=True, max_length=180) |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 162 | kernel = dbmodels.ForeignKey(Kernel, db_column='kernel_idx') |
| 163 | status = dbmodels.ForeignKey(Status, db_column='status') |
showard | a5288b4 | 2009-07-28 20:06:08 +0000 | [diff] [blame] | 164 | reason = dbmodels.CharField(blank=True, max_length=3072) |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 165 | machine = dbmodels.ForeignKey(Machine, db_column='machine_idx') |
| 166 | finished_time = dbmodels.DateTimeField(null=True, blank=True) |
| 167 | started_time = dbmodels.DateTimeField(null=True, blank=True) |
| 168 | |
showard | f8b1904 | 2009-05-12 17:22:49 +0000 | [diff] [blame] | 169 | objects = model_logic.ExtendedManager() |
| 170 | |
| 171 | def _get_attribute_model_and_args(self, attribute): |
| 172 | return TestAttribute, dict(test=self, attribute=attribute, |
| 173 | user_created=True) |
| 174 | |
| 175 | |
| 176 | def set_attribute(self, attribute, value): |
| 177 | # ensure non-user-created attributes remain immutable |
| 178 | try: |
| 179 | TestAttribute.objects.get(test=self, attribute=attribute, |
| 180 | user_created=False) |
| 181 | raise ValueError('Attribute %s already exists for test %s and is ' |
| 182 | 'immutable' % (attribute, self.test_idx)) |
| 183 | except TestAttribute.DoesNotExist: |
| 184 | super(Test, self).set_attribute(attribute, value) |
| 185 | |
| 186 | |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 187 | class Meta: |
| 188 | db_table = 'tests' |
| 189 | |
| 190 | |
showard | e732ee7 | 2008-09-23 19:15:43 +0000 | [diff] [blame] | 191 | class TestAttribute(dbmodels.Model, model_logic.ModelExtensions): |
showard | f8b1904 | 2009-05-12 17:22:49 +0000 | [diff] [blame] | 192 | test = dbmodels.ForeignKey(Test, db_column='test_idx') |
showard | a5288b4 | 2009-07-28 20:06:08 +0000 | [diff] [blame] | 193 | attribute = dbmodels.CharField(max_length=90) |
| 194 | value = dbmodels.CharField(blank=True, max_length=300) |
showard | f8b1904 | 2009-05-12 17:22:49 +0000 | [diff] [blame] | 195 | user_created = dbmodels.BooleanField(default=False) |
| 196 | |
| 197 | objects = model_logic.ExtendedManager() |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 198 | |
| 199 | class Meta: |
| 200 | db_table = 'test_attributes' |
| 201 | |
| 202 | |
jadmanski | 430dca9 | 2008-12-16 20:56:53 +0000 | [diff] [blame] | 203 | class IterationAttribute(dbmodels.Model, model_logic.ModelExtensions): |
showard | f8b1904 | 2009-05-12 17:22:49 +0000 | [diff] [blame] | 204 | # this isn't really a primary key, but it's necessary to appease Django |
| 205 | # and is harmless as long as we're careful |
showard | e732ee7 | 2008-09-23 19:15:43 +0000 | [diff] [blame] | 206 | test = dbmodels.ForeignKey(Test, db_column='test_idx', primary_key=True) |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 207 | iteration = dbmodels.IntegerField() |
showard | a5288b4 | 2009-07-28 20:06:08 +0000 | [diff] [blame] | 208 | attribute = dbmodels.CharField(max_length=90) |
| 209 | value = dbmodels.CharField(blank=True, max_length=300) |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 210 | |
showard | f8b1904 | 2009-05-12 17:22:49 +0000 | [diff] [blame] | 211 | objects = model_logic.ExtendedManager() |
| 212 | |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 213 | class Meta: |
| 214 | db_table = 'iteration_attributes' |
| 215 | |
| 216 | |
jadmanski | 430dca9 | 2008-12-16 20:56:53 +0000 | [diff] [blame] | 217 | class IterationResult(dbmodels.Model, model_logic.ModelExtensions): |
showard | f8b1904 | 2009-05-12 17:22:49 +0000 | [diff] [blame] | 218 | # see comment on IterationAttribute regarding primary_key=True |
jadmanski | 430dca9 | 2008-12-16 20:56:53 +0000 | [diff] [blame] | 219 | test = dbmodels.ForeignKey(Test, db_column='test_idx', primary_key=True) |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 220 | iteration = dbmodels.IntegerField() |
showard | a5288b4 | 2009-07-28 20:06:08 +0000 | [diff] [blame] | 221 | attribute = dbmodels.CharField(max_length=90) |
| 222 | value = dbmodels.DecimalField(null=True, max_digits=12, decimal_places=31, |
| 223 | blank=True) |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 224 | |
showard | f8b1904 | 2009-05-12 17:22:49 +0000 | [diff] [blame] | 225 | objects = model_logic.ExtendedManager() |
| 226 | |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 227 | class Meta: |
| 228 | db_table = 'iteration_result' |
| 229 | |
| 230 | |
| 231 | class TestLabel(dbmodels.Model, model_logic.ModelExtensions): |
showard | a5288b4 | 2009-07-28 20:06:08 +0000 | [diff] [blame] | 232 | name = dbmodels.CharField(max_length=80, unique=True) |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 233 | description = dbmodels.TextField(blank=True) |
showard | a5288b4 | 2009-07-28 20:06:08 +0000 | [diff] [blame] | 234 | tests = dbmodels.ManyToManyField(Test, blank=True) |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 235 | |
| 236 | name_field = 'name' |
showard | f8b1904 | 2009-05-12 17:22:49 +0000 | [diff] [blame] | 237 | objects = model_logic.ExtendedManager() |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 238 | |
| 239 | class Meta: |
| 240 | db_table = 'test_labels' |
| 241 | |
| 242 | |
| 243 | class SavedQuery(dbmodels.Model, model_logic.ModelExtensions): |
| 244 | # TODO: change this to foreign key once DBs are merged |
showard | a5288b4 | 2009-07-28 20:06:08 +0000 | [diff] [blame] | 245 | owner = dbmodels.CharField(max_length=80) |
| 246 | name = dbmodels.CharField(max_length=100) |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 247 | url_token = dbmodels.TextField() |
| 248 | |
| 249 | class Meta: |
| 250 | db_table = 'saved_queries' |
| 251 | |
| 252 | |
showard | ce12f55 | 2008-09-19 00:48:59 +0000 | [diff] [blame] | 253 | class EmbeddedGraphingQuery(dbmodels.Model, model_logic.ModelExtensions): |
| 254 | url_token = dbmodels.TextField(null=False, blank=False) |
showard | a5288b4 | 2009-07-28 20:06:08 +0000 | [diff] [blame] | 255 | graph_type = dbmodels.CharField(max_length=16, null=False, blank=False) |
showard | ce12f55 | 2008-09-19 00:48:59 +0000 | [diff] [blame] | 256 | params = dbmodels.TextField(null=False, blank=False) |
| 257 | last_updated = dbmodels.DateTimeField(null=False, blank=False, |
| 258 | editable=False) |
| 259 | # refresh_time shows the time at which a thread is updating the cached |
| 260 | # image, or NULL if no one is updating the image. This is used so that only |
| 261 | # one thread is updating the cached image at a time (see |
| 262 | # graphing_utils.handle_plot_request) |
| 263 | refresh_time = dbmodels.DateTimeField(editable=False) |
| 264 | cached_png = dbmodels.TextField(editable=False) |
| 265 | |
| 266 | class Meta: |
| 267 | db_table = 'embedded_graphing_queries' |
| 268 | |
| 269 | |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 270 | # views |
| 271 | |
| 272 | class TestViewManager(TempManager): |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 273 | def get_query_set(self): |
| 274 | query = super(TestViewManager, self).get_query_set() |
showard | 5bf7c50 | 2008-08-20 01:22:22 +0000 | [diff] [blame] | 275 | |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 276 | # add extra fields to selects, using the SQL itself as the "alias" |
| 277 | extra_select = dict((sql, sql) |
| 278 | for sql in self.model.extra_fields.iterkeys()) |
| 279 | return query.extra(select=extra_select) |
| 280 | |
| 281 | |
showard | f248952 | 2008-10-23 23:08:00 +0000 | [diff] [blame] | 282 | def _get_include_exclude_suffix(self, exclude): |
| 283 | if exclude: |
showard | 2aa318e | 2009-08-20 23:43:10 +0000 | [diff] [blame] | 284 | return '_exclude' |
| 285 | return '_include' |
| 286 | |
| 287 | |
| 288 | def _add_attribute_join(self, query_set, join_condition, |
| 289 | suffix=None, exclude=False): |
| 290 | if suffix is None: |
| 291 | suffix = self._get_include_exclude_suffix(exclude) |
| 292 | return self.add_join(query_set, 'test_attributes', join_key='test_idx', |
| 293 | join_condition=join_condition, |
| 294 | suffix=suffix, exclude=exclude) |
| 295 | |
| 296 | |
| 297 | def _add_label_pivot_table_join(self, query_set, suffix, join_condition='', |
| 298 | exclude=False, force_left_join=False): |
| 299 | return self.add_join(query_set, 'test_labels_tests', join_key='test_id', |
| 300 | join_condition=join_condition, |
| 301 | suffix=suffix, exclude=exclude, |
| 302 | force_left_join=force_left_join) |
showard | f248952 | 2008-10-23 23:08:00 +0000 | [diff] [blame] | 303 | |
| 304 | |
showard | 64aeecd | 2008-09-19 21:32:58 +0000 | [diff] [blame] | 305 | def _add_label_joins(self, query_set, suffix=''): |
showard | 2aa318e | 2009-08-20 23:43:10 +0000 | [diff] [blame] | 306 | query_set = self._add_label_pivot_table_join( |
| 307 | query_set, suffix=suffix, force_left_join=True) |
showard | d50ffb4 | 2008-09-04 02:47:45 +0000 | [diff] [blame] | 308 | |
showard | 2aa318e | 2009-08-20 23:43:10 +0000 | [diff] [blame] | 309 | # since we're not joining from the original table, we can't use |
| 310 | # self.add_join() again |
showard | d50ffb4 | 2008-09-04 02:47:45 +0000 | [diff] [blame] | 311 | second_join_alias = 'test_labels' + suffix |
| 312 | second_join_condition = ('%s.id = %s.testlabel_id' % |
showard | 64aeecd | 2008-09-19 21:32:58 +0000 | [diff] [blame] | 313 | (second_join_alias, |
| 314 | 'test_labels_tests' + suffix)) |
| 315 | filter_object = self._CustomSqlQ() |
showard | d50ffb4 | 2008-09-04 02:47:45 +0000 | [diff] [blame] | 316 | filter_object.add_join('test_labels', |
| 317 | second_join_condition, |
showard | a5288b4 | 2009-07-28 20:06:08 +0000 | [diff] [blame] | 318 | query_set.query.LOUTER, |
showard | d50ffb4 | 2008-09-04 02:47:45 +0000 | [diff] [blame] | 319 | alias=second_join_alias) |
showard | a5288b4 | 2009-07-28 20:06:08 +0000 | [diff] [blame] | 320 | return self._add_customSqlQ(query_set, filter_object) |
showard | d50ffb4 | 2008-09-04 02:47:45 +0000 | [diff] [blame] | 321 | |
showard | 64aeecd | 2008-09-19 21:32:58 +0000 | [diff] [blame] | 322 | |
showard | d50ffb4 | 2008-09-04 02:47:45 +0000 | [diff] [blame] | 323 | def _get_label_ids_from_names(self, label_names): |
showard | 2aa318e | 2009-08-20 23:43:10 +0000 | [diff] [blame] | 324 | assert label_names |
| 325 | label_ids = list( # listifying avoids a double query below |
| 326 | TestLabel.objects.filter(name__in=label_names).values('id')) |
| 327 | if len(label_ids) < len(set(label_names)): |
| 328 | raise ValueError('Not all labels found: %s' % |
| 329 | ', '.join(label_names)) |
| 330 | return [str(label['id']) for label in label_ids] |
| 331 | |
| 332 | |
| 333 | def _include_or_exclude_labels(self, query_set, label_names, exclude=False): |
| 334 | label_ids = self._get_label_ids_from_names(label_names) |
| 335 | suffix = self._get_include_exclude_suffix(exclude) |
| 336 | condition = ('test_labels_tests%s.testlabel_id IN (%s)' % |
| 337 | (suffix, ','.join(label_ids))) |
| 338 | return self._add_label_pivot_table_join(query_set, |
| 339 | join_condition=condition, |
| 340 | suffix=suffix, |
| 341 | exclude=exclude) |
showard | 0281350 | 2008-08-20 20:52:56 +0000 | [diff] [blame] | 342 | |
| 343 | |
showard | f248952 | 2008-10-23 23:08:00 +0000 | [diff] [blame] | 344 | def get_query_set_with_joins(self, filter_data, include_host_labels=False): |
showard | fc8c6ae | 2008-11-11 19:06:01 +0000 | [diff] [blame] | 345 | include_labels = filter_data.pop('include_labels', []) |
showard | d50ffb4 | 2008-09-04 02:47:45 +0000 | [diff] [blame] | 346 | exclude_labels = filter_data.pop('exclude_labels', []) |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 347 | query_set = self.get_query_set() |
showard | d50ffb4 | 2008-09-04 02:47:45 +0000 | [diff] [blame] | 348 | joined = False |
showard | 2aa318e | 2009-08-20 23:43:10 +0000 | [diff] [blame] | 349 | |
| 350 | # TODO: make this feature obsolete in favor of include_labels and |
| 351 | # exclude_labels |
showard | f248952 | 2008-10-23 23:08:00 +0000 | [diff] [blame] | 352 | extra_where = filter_data.get('extra_where', '') |
| 353 | if 'test_labels' in extra_where: |
showard | 0281350 | 2008-08-20 20:52:56 +0000 | [diff] [blame] | 354 | query_set = self._add_label_joins(query_set) |
showard | d50ffb4 | 2008-09-04 02:47:45 +0000 | [diff] [blame] | 355 | joined = True |
| 356 | |
showard | 2aa318e | 2009-08-20 23:43:10 +0000 | [diff] [blame] | 357 | if include_labels: |
| 358 | query_set = self._include_or_exclude_labels(query_set, |
| 359 | include_labels) |
showard | fc8c6ae | 2008-11-11 19:06:01 +0000 | [diff] [blame] | 360 | joined = True |
showard | 2aa318e | 2009-08-20 23:43:10 +0000 | [diff] [blame] | 361 | if exclude_labels: |
| 362 | query_set = self._include_or_exclude_labels(query_set, |
| 363 | exclude_labels, |
| 364 | exclude=True) |
showard | 64aeecd | 2008-09-19 21:32:58 +0000 | [diff] [blame] | 365 | joined = True |
| 366 | |
| 367 | include_attributes_where = filter_data.pop('include_attributes_where', |
| 368 | '') |
| 369 | exclude_attributes_where = filter_data.pop('exclude_attributes_where', |
| 370 | '') |
| 371 | if include_attributes_where: |
| 372 | query_set = self._add_attribute_join( |
showard | 2aa318e | 2009-08-20 23:43:10 +0000 | [diff] [blame] | 373 | query_set, |
| 374 | join_condition=self.escape_user_sql(include_attributes_where)) |
showard | 64aeecd | 2008-09-19 21:32:58 +0000 | [diff] [blame] | 375 | joined = True |
| 376 | if exclude_attributes_where: |
| 377 | query_set = self._add_attribute_join( |
showard | 2aa318e | 2009-08-20 23:43:10 +0000 | [diff] [blame] | 378 | query_set, |
| 379 | join_condition=self.escape_user_sql(exclude_attributes_where), |
showard | 64aeecd | 2008-09-19 21:32:58 +0000 | [diff] [blame] | 380 | exclude=True) |
| 381 | joined = True |
showard | d50ffb4 | 2008-09-04 02:47:45 +0000 | [diff] [blame] | 382 | |
showard | 8bfb5cb | 2009-10-07 20:49:15 +0000 | [diff] [blame] | 383 | test_attributes = filter_data.pop('test_attributes', []) |
| 384 | for attribute in test_attributes: |
| 385 | query_set = self.join_attribute(query_set, attribute) |
| 386 | joined = True |
| 387 | |
showard | d50ffb4 | 2008-09-04 02:47:45 +0000 | [diff] [blame] | 388 | if not joined: |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 389 | filter_data['no_distinct'] = True |
showard | d50ffb4 | 2008-09-04 02:47:45 +0000 | [diff] [blame] | 390 | |
showard | 2aa318e | 2009-08-20 23:43:10 +0000 | [diff] [blame] | 391 | # TODO: make test_attributes_host_labels obsolete too |
showard | f248952 | 2008-10-23 23:08:00 +0000 | [diff] [blame] | 392 | if include_host_labels or 'test_attributes_host_labels' in extra_where: |
| 393 | query_set = self._add_attribute_join( |
| 394 | query_set, suffix='_host_labels', |
| 395 | join_condition='test_attributes_host_labels.attribute = ' |
| 396 | '"host-labels"') |
| 397 | |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 398 | return query_set |
| 399 | |
| 400 | |
showard | 8bfb5cb | 2009-10-07 20:49:15 +0000 | [diff] [blame] | 401 | def query_test_ids(self, filter_data, apply_presentation=True): |
| 402 | query = self.model.query_objects(filter_data, |
| 403 | apply_presentation=apply_presentation) |
| 404 | dicts = query.values('test_idx') |
showard | 0281350 | 2008-08-20 20:52:56 +0000 | [diff] [blame] | 405 | return [item['test_idx'] for item in dicts] |
| 406 | |
| 407 | |
showard | 0281350 | 2008-08-20 20:52:56 +0000 | [diff] [blame] | 408 | def query_test_label_ids(self, filter_data): |
showard | d50ffb4 | 2008-09-04 02:47:45 +0000 | [diff] [blame] | 409 | query_set = self.model.query_objects(filter_data) |
| 410 | query_set = self._add_label_joins(query_set, suffix='_list') |
| 411 | rows = self._custom_select_query(query_set, ['test_labels_list.id']) |
| 412 | return [row[0] for row in rows if row[0] is not None] |
showard | 0281350 | 2008-08-20 20:52:56 +0000 | [diff] [blame] | 413 | |
| 414 | |
showard | eaccf8f | 2009-04-16 03:11:33 +0000 | [diff] [blame] | 415 | def escape_user_sql(self, sql): |
| 416 | sql = super(TestViewManager, self).escape_user_sql(sql) |
| 417 | return sql.replace('test_idx', self.get_key_on_this_table('test_idx')) |
| 418 | |
| 419 | |
showard | c478040 | 2009-08-31 18:31:34 +0000 | [diff] [blame] | 420 | def _join_one_iteration_key(self, query_set, result_key, index): |
| 421 | suffix = '_%s' % index |
| 422 | table_name = IterationResult._meta.db_table |
| 423 | alias = table_name + suffix |
| 424 | condition_parts = ["%s.attribute = '%s'" % |
| 425 | (alias, self.escape_user_sql(result_key))] |
| 426 | if index > 0: |
| 427 | # after the first join, we need to match up iteration indices, |
| 428 | # otherwise each join will expand the query by the number of |
| 429 | # iterations and we'll have extraneous rows |
| 430 | first_alias = table_name + '_0' |
| 431 | condition_parts.append('%s.iteration = %s.iteration' % |
| 432 | (alias, first_alias)) |
| 433 | |
| 434 | condition = ' and '.join(condition_parts) |
| 435 | # add a join to IterationResult |
| 436 | query_set = self.add_join(query_set, table_name, join_key='test_idx', |
| 437 | join_condition=condition, suffix=suffix) |
| 438 | # select the iteration value for this join |
| 439 | query_set = query_set.extra(select={result_key: '%s.value' % alias}) |
| 440 | if index == 0: |
| 441 | # pull the iteration index from the first join |
| 442 | query_set = query_set.extra( |
| 443 | select={'iteration_index': '%s.iteration' % alias}) |
| 444 | |
| 445 | return query_set |
| 446 | |
| 447 | |
| 448 | def join_iterations(self, test_view_query_set, result_keys): |
| 449 | """ |
| 450 | Join the given TestView QuerySet to IterationResult. The resulting |
| 451 | query looks like a TestView query but has one row per iteration. Each |
| 452 | row includes all the attributes of TestView, an attribute for each key |
| 453 | in result_keys and an iteration_index attribute. |
| 454 | |
| 455 | We accomplish this by joining the TestView query to IterationResult |
| 456 | once per result key. Each join is restricted on the result key (and on |
| 457 | the test index, like all one-to-many joins). For the first join, this |
| 458 | is the only restriction, so each TestView row expands to a row per |
| 459 | iteration (per iteration that includes the key, of course). For each |
| 460 | subsequent join, we also restrict the iteration index to match that of |
| 461 | the initial join. This makes each subsequent join produce exactly one |
| 462 | result row for each input row. (This assumes each iteration contains |
| 463 | the same set of keys.) |
| 464 | """ |
| 465 | query_set = test_view_query_set |
| 466 | for index, result_key in enumerate(result_keys): |
| 467 | query_set = self._join_one_iteration_key(query_set, result_key, |
| 468 | index) |
| 469 | return query_set |
| 470 | |
| 471 | |
showard | 8bfb5cb | 2009-10-07 20:49:15 +0000 | [diff] [blame] | 472 | def join_attribute(self, test_view_query_set, attribute): |
| 473 | """ |
| 474 | Join the given TestView QuerySet to TestAttribute. The resulting query |
| 475 | has an additional column for the given attribute named |
| 476 | "attribute_<attribute name>". |
| 477 | """ |
| 478 | table_name = TestAttribute._meta.db_table |
| 479 | suffix = '_' + attribute |
| 480 | alias = table_name + suffix |
| 481 | condition = "%s.attribute = '%s'" % (alias, |
| 482 | self.escape_user_sql(attribute)) |
| 483 | query_set = self.add_join(test_view_query_set, table_name, |
| 484 | join_key='test_idx', join_condition=condition, |
| 485 | suffix=suffix, force_left_join=True) |
| 486 | |
| 487 | select_name = 'attribute_' + attribute |
| 488 | query_set = query_set.extra(select={select_name: '%s.value' % alias}) |
| 489 | return query_set |
| 490 | |
| 491 | |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 492 | class TestView(dbmodels.Model, model_logic.ModelExtensions): |
| 493 | extra_fields = { |
showard | f4c702e | 2009-07-08 21:14:27 +0000 | [diff] [blame] | 494 | 'DATE(job_queued_time)': 'job queued day', |
| 495 | 'DATE(test_finished_time)': 'test finished day', |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 496 | } |
| 497 | |
| 498 | group_fields = [ |
showard | f4c702e | 2009-07-08 21:14:27 +0000 | [diff] [blame] | 499 | 'test_name', |
| 500 | 'status', |
| 501 | 'kernel', |
| 502 | 'hostname', |
| 503 | 'job_tag', |
| 504 | 'job_name', |
| 505 | 'platform', |
| 506 | 'reason', |
| 507 | 'job_owner', |
| 508 | 'job_queued_time', |
| 509 | 'DATE(job_queued_time)', |
| 510 | 'test_started_time', |
| 511 | 'test_finished_time', |
| 512 | 'DATE(test_finished_time)', |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 513 | ] |
| 514 | |
| 515 | test_idx = dbmodels.IntegerField('test index', primary_key=True) |
| 516 | job_idx = dbmodels.IntegerField('job index', null=True, blank=True) |
showard | a5288b4 | 2009-07-28 20:06:08 +0000 | [diff] [blame] | 517 | test_name = dbmodels.CharField(blank=True, max_length=90) |
| 518 | subdir = dbmodels.CharField('subdirectory', blank=True, max_length=180) |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 519 | kernel_idx = dbmodels.IntegerField('kernel index') |
| 520 | status_idx = dbmodels.IntegerField('status index') |
showard | a5288b4 | 2009-07-28 20:06:08 +0000 | [diff] [blame] | 521 | reason = dbmodels.CharField(blank=True, max_length=3072) |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 522 | machine_idx = dbmodels.IntegerField('host index') |
| 523 | test_started_time = dbmodels.DateTimeField(null=True, blank=True) |
| 524 | test_finished_time = dbmodels.DateTimeField(null=True, blank=True) |
showard | a5288b4 | 2009-07-28 20:06:08 +0000 | [diff] [blame] | 525 | job_tag = dbmodels.CharField(blank=True, max_length=300) |
| 526 | job_name = dbmodels.CharField(blank=True, max_length=300) |
| 527 | job_owner = dbmodels.CharField('owner', blank=True, max_length=240) |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 528 | job_queued_time = dbmodels.DateTimeField(null=True, blank=True) |
| 529 | job_started_time = dbmodels.DateTimeField(null=True, blank=True) |
| 530 | job_finished_time = dbmodels.DateTimeField(null=True, blank=True) |
showard | c1c1caf | 2009-09-08 16:26:50 +0000 | [diff] [blame] | 531 | afe_job_id = dbmodels.IntegerField(null=True) |
showard | a5288b4 | 2009-07-28 20:06:08 +0000 | [diff] [blame] | 532 | hostname = dbmodels.CharField(blank=True, max_length=300) |
| 533 | platform = dbmodels.CharField(blank=True, max_length=240) |
| 534 | machine_owner = dbmodels.CharField(blank=True, max_length=240) |
| 535 | kernel_hash = dbmodels.CharField(blank=True, max_length=105) |
| 536 | kernel_base = dbmodels.CharField(blank=True, max_length=90) |
| 537 | kernel = dbmodels.CharField(blank=True, max_length=300) |
| 538 | status = dbmodels.CharField(blank=True, max_length=30) |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 539 | |
| 540 | objects = TestViewManager() |
| 541 | |
| 542 | def save(self): |
| 543 | raise NotImplementedError('TestView is read-only') |
| 544 | |
| 545 | |
| 546 | def delete(self): |
| 547 | raise NotImplementedError('TestView is read-only') |
| 548 | |
| 549 | |
| 550 | @classmethod |
showard | 8bfb5cb | 2009-10-07 20:49:15 +0000 | [diff] [blame] | 551 | def query_objects(cls, filter_data, initial_query=None, |
| 552 | apply_presentation=True): |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 553 | if initial_query is None: |
showard | 64aeecd | 2008-09-19 21:32:58 +0000 | [diff] [blame] | 554 | initial_query = cls.objects.get_query_set_with_joins(filter_data) |
showard | 8bfb5cb | 2009-10-07 20:49:15 +0000 | [diff] [blame] | 555 | return super(TestView, cls).query_objects( |
| 556 | filter_data, initial_query=initial_query, |
| 557 | apply_presentation=apply_presentation) |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 558 | |
| 559 | |
| 560 | class Meta: |
| 561 | db_table = 'test_view_2' |