showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 1 | from autotest_lib.frontend.afe import rpc_utils |
mbligh | 48d412c | 2009-07-02 18:55:35 +0000 | [diff] [blame] | 2 | from autotest_lib.client.common_lib import kernel_versions |
showard | 250d84d | 2010-01-12 21:59:48 +0000 | [diff] [blame^] | 3 | from autotest_lib.frontend.tko import models |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 4 | |
| 5 | class TooManyRowsError(Exception): |
| 6 | """ |
| 7 | Raised when a database query returns too many rows. |
| 8 | """ |
| 9 | |
| 10 | |
| 11 | class KernelString(str): |
| 12 | """ |
| 13 | Custom string class that uses correct kernel version comparisons. |
| 14 | """ |
| 15 | def _map(self): |
| 16 | return kernel_versions.version_encode(self) |
| 17 | |
| 18 | |
| 19 | def __hash__(self): |
| 20 | return hash(self._map()) |
| 21 | |
| 22 | |
| 23 | def __eq__(self, other): |
| 24 | return self._map() == other._map() |
| 25 | |
| 26 | |
| 27 | def __ne__(self, other): |
| 28 | return self._map() != other._map() |
| 29 | |
| 30 | |
| 31 | def __lt__(self, other): |
| 32 | return self._map() < other._map() |
| 33 | |
| 34 | |
| 35 | def __lte__(self, other): |
| 36 | return self._map() <= other._map() |
| 37 | |
| 38 | |
| 39 | def __gt__(self, other): |
| 40 | return self._map() > other._map() |
| 41 | |
| 42 | |
| 43 | def __gte__(self, other): |
| 44 | return self._map() >= other._map() |
| 45 | |
| 46 | |
showard | 8a6eb0c | 2008-10-01 11:38:59 +0000 | [diff] [blame] | 47 | # SQL expression to compute passed test count for test groups |
| 48 | _PASS_COUNT_NAME = 'pass_count' |
| 49 | _COMPLETE_COUNT_NAME = 'complete_count' |
| 50 | _INCOMPLETE_COUNT_NAME = 'incomplete_count' |
| 51 | # Using COUNT instead of SUM here ensures the resulting row has the right type |
| 52 | # (i.e. numeric, not string). I don't know why. |
showard | 7c199df | 2008-10-03 10:17:15 +0000 | [diff] [blame] | 53 | _PASS_COUNT_SQL = 'COUNT(IF(status="GOOD", 1, NULL))' |
| 54 | _COMPLETE_COUNT_SQL = ('COUNT(IF(status NOT IN ("TEST_NA", "RUNNING", ' |
| 55 | '"NOSTATUS"), 1, NULL))') |
| 56 | _INCOMPLETE_COUNT_SQL = 'COUNT(IF(status="RUNNING", 1, NULL))' |
| 57 | STATUS_FIELDS = {_PASS_COUNT_NAME : _PASS_COUNT_SQL, |
| 58 | _COMPLETE_COUNT_NAME : _COMPLETE_COUNT_SQL, |
| 59 | _INCOMPLETE_COUNT_NAME : _INCOMPLETE_COUNT_SQL} |
showard | 8a6eb0c | 2008-10-01 11:38:59 +0000 | [diff] [blame] | 60 | _INVALID_STATUSES = ('TEST_NA', 'NOSTATUS') |
| 61 | |
| 62 | |
| 63 | def add_status_counts(group_dict, status): |
| 64 | pass_count = complete_count = incomplete_count = 0 |
| 65 | if status == 'GOOD': |
| 66 | pass_count = complete_count = 1 |
| 67 | elif status == 'RUNNING': |
| 68 | incomplete_count = 1 |
| 69 | else: |
| 70 | complete_count = 1 |
| 71 | group_dict[_PASS_COUNT_NAME] = pass_count |
| 72 | group_dict[_COMPLETE_COUNT_NAME] = complete_count |
| 73 | group_dict[_INCOMPLETE_COUNT_NAME] = incomplete_count |
| 74 | group_dict[models.TestView.objects._GROUP_COUNT_NAME] = 1 |
| 75 | |
| 76 | |
showard | f248952 | 2008-10-23 23:08:00 +0000 | [diff] [blame] | 77 | def _construct_machine_label_header_sql(machine_labels): |
| 78 | """ |
| 79 | Example result for machine_labels=['Index', 'Diskful']: |
| 80 | CONCAT_WS(",", |
showard | eab66ce | 2009-12-23 00:03:56 +0000 | [diff] [blame] | 81 | IF(FIND_IN_SET("Diskful", tko_test_attributes_host_labels.value), |
showard | f248952 | 2008-10-23 23:08:00 +0000 | [diff] [blame] | 82 | "Diskful", NULL), |
showard | eab66ce | 2009-12-23 00:03:56 +0000 | [diff] [blame] | 83 | IF(FIND_IN_SET("Index", tko_test_attributes_host_labels.value), |
showard | f248952 | 2008-10-23 23:08:00 +0000 | [diff] [blame] | 84 | "Index", NULL)) |
| 85 | |
| 86 | This would result in field values "Diskful,Index", "Diskful", "Index", NULL. |
| 87 | """ |
| 88 | machine_labels = sorted(machine_labels) |
| 89 | if_clauses = [] |
| 90 | for label in machine_labels: |
| 91 | if_clauses.append( |
showard | eab66ce | 2009-12-23 00:03:56 +0000 | [diff] [blame] | 92 | 'IF(FIND_IN_SET("%s", tko_test_attributes_host_labels.value), ' |
showard | f248952 | 2008-10-23 23:08:00 +0000 | [diff] [blame] | 93 | '"%s", NULL)' % (label, label)) |
| 94 | return 'CONCAT_WS(",", %s)' % ', '.join(if_clauses) |
| 95 | |
| 96 | |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 97 | class GroupDataProcessor(object): |
mbligh | 444d6e0 | 2009-02-06 22:34:43 +0000 | [diff] [blame] | 98 | _MAX_GROUP_RESULTS = 80000 |
showard | 8a6eb0c | 2008-10-01 11:38:59 +0000 | [diff] [blame] | 99 | |
showard | 8bfb5cb | 2009-10-07 20:49:15 +0000 | [diff] [blame] | 100 | def __init__(self, query, group_by, header_groups, fixed_headers): |
showard | 8a6eb0c | 2008-10-01 11:38:59 +0000 | [diff] [blame] | 101 | self._query = query |
| 102 | self._group_by = self.uniqify(group_by) |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 103 | self._header_groups = header_groups |
showard | 8c9b839 | 2008-09-30 10:38:21 +0000 | [diff] [blame] | 104 | self._fixed_headers = dict((field, set(values)) |
| 105 | for field, values |
| 106 | in fixed_headers.iteritems()) |
showard | 8a6eb0c | 2008-10-01 11:38:59 +0000 | [diff] [blame] | 107 | |
| 108 | self._num_group_fields = len(group_by) |
| 109 | self._header_value_sets = [set() for i |
| 110 | in xrange(len(header_groups))] |
| 111 | self._group_dicts = [] |
| 112 | |
| 113 | |
| 114 | @staticmethod |
| 115 | def uniqify(values): |
| 116 | return list(set(values)) |
| 117 | |
| 118 | |
| 119 | def _restrict_header_values(self): |
| 120 | for header_field, values in self._fixed_headers.iteritems(): |
| 121 | self._query = self._query.filter(**{header_field + '__in' : values}) |
| 122 | |
| 123 | |
| 124 | def _fetch_data(self): |
| 125 | self._restrict_header_values() |
| 126 | self._group_dicts = models.TestView.objects.execute_group_query( |
showard | 8bfb5cb | 2009-10-07 20:49:15 +0000 | [diff] [blame] | 127 | self._query, self._group_by) |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 128 | |
| 129 | |
| 130 | @staticmethod |
| 131 | def _get_field(group_dict, field): |
| 132 | """ |
showard | 8a6eb0c | 2008-10-01 11:38:59 +0000 | [diff] [blame] | 133 | Use special objects for certain fields to achieve custom sorting. |
| 134 | -Wrap kernel versions with a KernelString |
| 135 | -Replace null dates with special values |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 136 | """ |
| 137 | value = group_dict[field] |
| 138 | if field == 'kernel': |
| 139 | return KernelString(value) |
| 140 | if value is None: # handle null dates as later than everything else |
| 141 | if field.startswith('DATE('): |
| 142 | return rpc_utils.NULL_DATE |
| 143 | if field.endswith('_time'): |
| 144 | return rpc_utils.NULL_DATETIME |
| 145 | return value |
| 146 | |
| 147 | |
showard | 8a6eb0c | 2008-10-01 11:38:59 +0000 | [diff] [blame] | 148 | def _process_group_dict(self, group_dict): |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 149 | # compute and aggregate header groups |
| 150 | for i, group in enumerate(self._header_groups): |
| 151 | header = tuple(self._get_field(group_dict, field) |
| 152 | for field in group) |
| 153 | self._header_value_sets[i].add(header) |
| 154 | group_dict.setdefault('header_values', []).append(header) |
| 155 | |
| 156 | # frontend's SelectionManager needs a unique ID |
showard | 8a6eb0c | 2008-10-01 11:38:59 +0000 | [diff] [blame] | 157 | group_values = [group_dict[field] for field in self._group_by] |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 158 | group_dict['id'] = str(group_values) |
| 159 | return group_dict |
| 160 | |
| 161 | |
showard | 8c9b839 | 2008-09-30 10:38:21 +0000 | [diff] [blame] | 162 | def _find_header_value_set(self, field): |
| 163 | for i, group in enumerate(self._header_groups): |
| 164 | if [field] == group: |
| 165 | return self._header_value_sets[i] |
| 166 | raise RuntimeError('Field %s not found in header groups %s' % |
| 167 | (field, self._header_groups)) |
| 168 | |
| 169 | |
| 170 | def _add_fixed_headers(self): |
| 171 | for field, extra_values in self._fixed_headers.iteritems(): |
| 172 | header_value_set = self._find_header_value_set(field) |
| 173 | for value in extra_values: |
| 174 | header_value_set.add((value,)) |
| 175 | |
| 176 | |
showard | 8a6eb0c | 2008-10-01 11:38:59 +0000 | [diff] [blame] | 177 | def _get_sorted_header_values(self): |
showard | 8c9b839 | 2008-09-30 10:38:21 +0000 | [diff] [blame] | 178 | self._add_fixed_headers() |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 179 | sorted_header_values = [sorted(value_set) |
| 180 | for value_set in self._header_value_sets] |
| 181 | # construct dicts mapping headers to their indices, for use in |
| 182 | # replace_headers_with_indices() |
| 183 | self._header_index_maps = [] |
| 184 | for value_list in sorted_header_values: |
| 185 | index_map = dict((value, i) for i, value in enumerate(value_list)) |
| 186 | self._header_index_maps.append(index_map) |
| 187 | |
| 188 | return sorted_header_values |
| 189 | |
| 190 | |
showard | 8a6eb0c | 2008-10-01 11:38:59 +0000 | [diff] [blame] | 191 | def _replace_headers_with_indices(self, group_dict): |
showard | 3544486 | 2008-08-07 22:35:30 +0000 | [diff] [blame] | 192 | group_dict['header_indices'] = [index_map[header_value] |
| 193 | for index_map, header_value |
| 194 | in zip(self._header_index_maps, |
| 195 | group_dict['header_values'])] |
| 196 | for field in self._group_by + ['header_values']: |
| 197 | del group_dict[field] |
showard | 8a6eb0c | 2008-10-01 11:38:59 +0000 | [diff] [blame] | 198 | |
| 199 | |
| 200 | def process_group_dicts(self): |
| 201 | self._fetch_data() |
| 202 | if len(self._group_dicts) > self._MAX_GROUP_RESULTS: |
| 203 | raise TooManyRowsError( |
| 204 | 'Query yielded %d rows, exceeding maximum %d' % ( |
| 205 | len(self._group_dicts), self._MAX_GROUP_RESULTS)) |
| 206 | |
| 207 | for group_dict in self._group_dicts: |
| 208 | self._process_group_dict(group_dict) |
| 209 | self._header_values = self._get_sorted_header_values() |
| 210 | if self._header_groups: |
| 211 | for group_dict in self._group_dicts: |
| 212 | self._replace_headers_with_indices(group_dict) |
| 213 | |
| 214 | |
| 215 | def get_info_dict(self): |
| 216 | return {'groups' : self._group_dicts, |
| 217 | 'header_values' : self._header_values} |