mbligh | 9bb92fe | 2007-09-12 15:54:23 +0000 | [diff] [blame] | 1 | #!/usr/bin/python |
mbligh | 4a370cf | 2008-04-01 19:56:05 +0000 | [diff] [blame] | 2 | import os, re, db, sys, datetime |
mbligh | 5bb5586 | 2008-04-16 23:09:31 +0000 | [diff] [blame] | 3 | MAX_RECORDS = 20000L |
| 4 | MAX_CELLS = 300000L |
mbligh | 9bb92fe | 2007-09-12 15:54:23 +0000 | [diff] [blame] | 5 | |
mbligh | 2ba3e73 | 2008-01-16 01:30:19 +0000 | [diff] [blame] | 6 | tko = os.path.dirname(os.path.realpath(os.path.abspath(__file__))) |
| 7 | client_bin = os.path.abspath(os.path.join(tko, '../client/bin')) |
| 8 | sys.path.insert(0, client_bin) |
| 9 | import kernel_versions |
| 10 | |
mbligh | 2aaeb67 | 2007-10-01 14:54:18 +0000 | [diff] [blame] | 11 | root_url_file = os.path.join(tko, '.root_url') |
| 12 | if os.path.exists(root_url_file): |
mbligh | 4a370cf | 2008-04-01 19:56:05 +0000 | [diff] [blame] | 13 | html_root = open(root_url_file, 'r').readline().rstrip() |
mbligh | 2aaeb67 | 2007-10-01 14:54:18 +0000 | [diff] [blame] | 14 | else: |
mbligh | 4a370cf | 2008-04-01 19:56:05 +0000 | [diff] [blame] | 15 | html_root = '/results/' |
mbligh | 2aaeb67 | 2007-10-01 14:54:18 +0000 | [diff] [blame] | 16 | |
mbligh | 2e4e5df | 2007-11-05 17:22:46 +0000 | [diff] [blame] | 17 | |
mbligh | 2ba3e73 | 2008-01-16 01:30:19 +0000 | [diff] [blame] | 18 | class status_cell: |
| 19 | # One cell in the matrix of status data. |
| 20 | def __init__(self): |
| 21 | # Count is a dictionary: status -> count of tests with status |
| 22 | self.status_count = {} |
mbligh | 3126069 | 2008-04-16 23:12:12 +0000 | [diff] [blame^] | 23 | self.reasons_list = [] |
mbligh | 2ba3e73 | 2008-01-16 01:30:19 +0000 | [diff] [blame] | 24 | self.job_tag = None |
| 25 | self.job_tag_count = 0 |
mbligh | 2e4e5df | 2007-11-05 17:22:46 +0000 | [diff] [blame] | 26 | |
mbligh | 83f63a0 | 2007-12-12 19:13:04 +0000 | [diff] [blame] | 27 | |
mbligh | 3126069 | 2008-04-16 23:12:12 +0000 | [diff] [blame^] | 28 | def add(self, status, count, job_tags, reasons = None): |
mbligh | 2ba3e73 | 2008-01-16 01:30:19 +0000 | [diff] [blame] | 29 | assert count > 0 |
| 30 | |
| 31 | self.job_tag = job_tags |
| 32 | self.job_tag_count += count |
| 33 | if self.job_tag_count > 1: |
| 34 | self.job_tag = None |
| 35 | |
| 36 | self.status_count[status] = count |
mbligh | 3126069 | 2008-04-16 23:12:12 +0000 | [diff] [blame^] | 37 | ### status == 6 means 'GOOD' |
| 38 | if status != 6: |
| 39 | ## None implies sorting problems and extra CRs in a cell |
| 40 | if reasons: |
| 41 | self.reasons_list.append(reasons) |
mbligh | 2ba3e73 | 2008-01-16 01:30:19 +0000 | [diff] [blame] | 42 | |
| 43 | |
| 44 | class status_data: |
mbligh | 3126069 | 2008-04-16 23:12:12 +0000 | [diff] [blame^] | 45 | def __init__(self, sql_rows, x_field, y_field, query_reasons = False): |
mbligh | 2ba3e73 | 2008-01-16 01:30:19 +0000 | [diff] [blame] | 46 | data = {} |
| 47 | y_values = set() |
| 48 | |
| 49 | # Walk through the query, filing all results by x, y info |
mbligh | 3126069 | 2008-04-16 23:12:12 +0000 | [diff] [blame^] | 50 | for row in sql_rows: |
| 51 | if query_reasons: |
| 52 | (x,y, status, count, job_tags, reasons) = row |
| 53 | else: |
| 54 | (x,y, status, count, job_tags) = row |
| 55 | reasons = None |
mbligh | 2ba3e73 | 2008-01-16 01:30:19 +0000 | [diff] [blame] | 56 | if not data.has_key(x): |
| 57 | data[x] = {} |
| 58 | if not data[x].has_key(y): |
| 59 | y_values.add(y) |
| 60 | data[x][y] = status_cell() |
mbligh | 3126069 | 2008-04-16 23:12:12 +0000 | [diff] [blame^] | 61 | data[x][y].add(status, count, job_tags, reasons) |
mbligh | 2ba3e73 | 2008-01-16 01:30:19 +0000 | [diff] [blame] | 62 | |
| 63 | # 2-d hash of data - [x-value][y-value] |
| 64 | self.data = data |
| 65 | # List of possible columns (x-values) |
| 66 | self.x_values = smart_sort(data.keys(), x_field) |
| 67 | # List of rows columns (y-values) |
| 68 | self.y_values = smart_sort(list(y_values), y_field) |
mbligh | 5bb5586 | 2008-04-16 23:09:31 +0000 | [diff] [blame] | 69 | nCells = len(self.y_values)*len(self.x_values) |
| 70 | if nCells > MAX_CELLS: |
mbligh | aea0960 | 2008-04-16 22:59:37 +0000 | [diff] [blame] | 71 | msg = 'Exceeded allowed number of cells in a table' |
| 72 | raise db.MySQLTooManyRows(msg) |
| 73 | |
mbligh | 83f63a0 | 2007-12-12 19:13:04 +0000 | [diff] [blame] | 74 | |
mbligh | 3126069 | 2008-04-16 23:12:12 +0000 | [diff] [blame^] | 75 | def get_matrix_data(db_obj, x_axis, y_axis, where = None, |
| 76 | query_reasons = False): |
mbligh | 83f63a0 | 2007-12-12 19:13:04 +0000 | [diff] [blame] | 77 | # Searches on the test_view table - x_axis and y_axis must both be |
| 78 | # column names in that table. |
mbligh | 2ba3e73 | 2008-01-16 01:30:19 +0000 | [diff] [blame] | 79 | x_field = test_view_field_dict[x_axis] |
| 80 | y_field = test_view_field_dict[y_axis] |
mbligh | 3126069 | 2008-04-16 23:12:12 +0000 | [diff] [blame^] | 81 | query_fields_list = [x_field, y_field, 'status','COUNT(status)'] |
| 82 | query_fields_list.append("LEFT(GROUP_CONCAT(job_tag),100)") |
| 83 | if query_reasons: |
| 84 | query_fields_list.append( |
| 85 | "LEFT(GROUP_CONCAT(DISTINCT reason SEPARATOR '|'),500)" |
| 86 | ) |
| 87 | fields = ','.join(query_fields_list) |
| 88 | |
mbligh | 2ba3e73 | 2008-01-16 01:30:19 +0000 | [diff] [blame] | 89 | group_by = '%s, %s, status' % (x_field, y_field) |
mbligh | aea0960 | 2008-04-16 22:59:37 +0000 | [diff] [blame] | 90 | rows = db_obj.select(fields, 'test_view', |
| 91 | where=where, group_by=group_by, max_rows = MAX_RECORDS) |
mbligh | 3126069 | 2008-04-16 23:12:12 +0000 | [diff] [blame^] | 92 | return status_data(rows, x_field, y_field, query_reasons) |
mbligh | 83f63a0 | 2007-12-12 19:13:04 +0000 | [diff] [blame] | 93 | |
| 94 | |
mbligh | 2ba3e73 | 2008-01-16 01:30:19 +0000 | [diff] [blame] | 95 | # Dictionary used simply for fast lookups from short reference names for users |
| 96 | # to fieldnames in test_view |
| 97 | test_view_field_dict = { |
| 98 | 'kernel' : 'kernel_printable', |
| 99 | 'hostname' : 'machine_hostname', |
| 100 | 'test' : 'test', |
| 101 | 'label' : 'job_label', |
| 102 | 'machine_group' : 'machine_group', |
| 103 | 'reason' : 'reason', |
| 104 | 'tag' : 'job_tag', |
| 105 | 'user' : 'job_username', |
| 106 | 'status' : 'status_word', |
mbligh | 4a370cf | 2008-04-01 19:56:05 +0000 | [diff] [blame] | 107 | 'time' : 'test_finished_time', |
mbligh | 5bb5586 | 2008-04-16 23:09:31 +0000 | [diff] [blame] | 108 | 'time_daily' : 'DATE(test_finished_time)' |
mbligh | 2ba3e73 | 2008-01-16 01:30:19 +0000 | [diff] [blame] | 109 | } |
mbligh | 2b67253 | 2007-11-05 19:24:51 +0000 | [diff] [blame] | 110 | |
mbligh | 3126069 | 2008-04-16 23:12:12 +0000 | [diff] [blame^] | 111 | |
mbligh | 2ba3e73 | 2008-01-16 01:30:19 +0000 | [diff] [blame] | 112 | def smart_sort(list, field): |
| 113 | if field == 'kernel_printable': |
| 114 | def kernel_encode(kernel): |
| 115 | return kernel_versions.version_encode(kernel) |
| 116 | list.sort(key = kernel_encode, reverse = True) |
mbligh | 4a370cf | 2008-04-01 19:56:05 +0000 | [diff] [blame] | 117 | elif field.endswith('time'): |
| 118 | ## old records may contain time=None |
| 119 | ## make None comparable with timestamp as string |
| 120 | def handleNone(date_time): |
| 121 | if date_time==None: |
| 122 | return datetime.datetime(1970,1,1,0,0,0) |
| 123 | else: |
| 124 | return date_time |
| 125 | list = map(handleNone,list) |
| 126 | list.sort() |
mbligh | 2ba3e73 | 2008-01-16 01:30:19 +0000 | [diff] [blame] | 127 | return list |
mbligh | 2e4e5df | 2007-11-05 17:22:46 +0000 | [diff] [blame] | 128 | |
mbligh | 2aaeb67 | 2007-10-01 14:54:18 +0000 | [diff] [blame] | 129 | |
mbligh | cff2d21 | 2007-10-07 00:11:10 +0000 | [diff] [blame] | 130 | class group: |
| 131 | @classmethod |
| 132 | def select(klass, db): |
| 133 | """Return all possible machine groups""" |
| 134 | rows = db.select('distinct machine_group', 'machines', |
| 135 | 'machine_group is not null') |
| 136 | groupnames = sorted([row[0] for row in rows]) |
| 137 | return [klass(db, groupname) for groupname in groupnames] |
mbligh | 83f63a0 | 2007-12-12 19:13:04 +0000 | [diff] [blame] | 138 | |
| 139 | |
mbligh | cff2d21 | 2007-10-07 00:11:10 +0000 | [diff] [blame] | 140 | def __init__(self, db, name): |
| 141 | self.name = name |
| 142 | self.db = db |
| 143 | |
| 144 | |
| 145 | def machines(self): |
| 146 | return machine.select(self.db, { 'machine_group' : self.name }) |
mbligh | 2e4e5df | 2007-11-05 17:22:46 +0000 | [diff] [blame] | 147 | |
mbligh | cff2d21 | 2007-10-07 00:11:10 +0000 | [diff] [blame] | 148 | |
| 149 | def tests(self, where = {}): |
| 150 | values = [self.name] |
mbligh | 3126069 | 2008-04-16 23:12:12 +0000 | [diff] [blame^] | 151 | sql = 't inner join machines m on m.machine_idx=t.machine_idx' |
| 152 | sql += ' where m.machine_group=%s' |
mbligh | cff2d21 | 2007-10-07 00:11:10 +0000 | [diff] [blame] | 153 | for key in where.keys(): |
| 154 | sql += ' and %s=%%s' % key |
| 155 | values.append(where[key]) |
| 156 | return test.select_sql(self.db, sql, values) |
| 157 | |
mbligh | 2e4e5df | 2007-11-05 17:22:46 +0000 | [diff] [blame] | 158 | |
mbligh | 2aaeb67 | 2007-10-01 14:54:18 +0000 | [diff] [blame] | 159 | class machine: |
| 160 | @classmethod |
| 161 | def select(klass, db, where = {}): |
| 162 | fields = ['machine_idx', 'hostname', 'machine_group', 'owner'] |
| 163 | machines = [] |
| 164 | for row in db.select(','.join(fields), 'machines', where): |
| 165 | machines.append(klass(db, *row)) |
| 166 | return machines |
| 167 | |
| 168 | |
| 169 | def __init__(self, db, idx, hostname, group, owner): |
| 170 | self.db = db |
| 171 | self.idx = idx |
| 172 | self.hostname = hostname |
| 173 | self.group = group |
mbligh | f736b33 | 2007-12-18 20:56:51 +0000 | [diff] [blame] | 174 | self.owner = owner |
mbligh | 2e4e5df | 2007-11-05 17:22:46 +0000 | [diff] [blame] | 175 | |
mbligh | 250300e | 2007-09-18 00:50:57 +0000 | [diff] [blame] | 176 | |
mbligh | 9bb92fe | 2007-09-12 15:54:23 +0000 | [diff] [blame] | 177 | class kernel: |
mbligh | 8e1ab17 | 2007-09-13 17:29:56 +0000 | [diff] [blame] | 178 | @classmethod |
| 179 | def select(klass, db, where = {}): |
| 180 | fields = ['kernel_idx', 'kernel_hash', 'base', 'printable'] |
mbligh | 83f63a0 | 2007-12-12 19:13:04 +0000 | [diff] [blame] | 181 | rows = db.select(','.join(fields), 'kernels', where) |
| 182 | return [klass(db, *row) for row in rows] |
mbligh | 9bb92fe | 2007-09-12 15:54:23 +0000 | [diff] [blame] | 183 | |
mbligh | 8e1ab17 | 2007-09-13 17:29:56 +0000 | [diff] [blame] | 184 | |
| 185 | def __init__(self, db, idx, hash, base, printable): |
mbligh | 9bb92fe | 2007-09-12 15:54:23 +0000 | [diff] [blame] | 186 | self.db = db |
mbligh | 8e1ab17 | 2007-09-13 17:29:56 +0000 | [diff] [blame] | 187 | self.idx = idx |
mbligh | 9bb92fe | 2007-09-12 15:54:23 +0000 | [diff] [blame] | 188 | self.hash = hash |
mbligh | 8e1ab17 | 2007-09-13 17:29:56 +0000 | [diff] [blame] | 189 | self.base = base |
| 190 | self.printable = printable |
| 191 | self.patches = [] # THIS SHOULD PULL IN PATCHES! |
mbligh | 9bb92fe | 2007-09-12 15:54:23 +0000 | [diff] [blame] | 192 | |
| 193 | |
| 194 | class test: |
mbligh | 8e1ab17 | 2007-09-13 17:29:56 +0000 | [diff] [blame] | 195 | @classmethod |
mbligh | 85952b4 | 2007-12-07 16:28:33 +0000 | [diff] [blame] | 196 | def select(klass, db, where = {}, wherein = {}, distinct = False): |
mbligh | 8e1ab17 | 2007-09-13 17:29:56 +0000 | [diff] [blame] | 197 | fields = ['test_idx', 'job_idx', 'test', 'subdir', |
mbligh | 2aaeb67 | 2007-10-01 14:54:18 +0000 | [diff] [blame] | 198 | 'kernel_idx', 'status', 'reason', 'machine_idx'] |
mbligh | 8e1ab17 | 2007-09-13 17:29:56 +0000 | [diff] [blame] | 199 | tests = [] |
mbligh | 3126069 | 2008-04-16 23:12:12 +0000 | [diff] [blame^] | 200 | for row in db.select(','.join(fields), 'tests', where, |
| 201 | wherein,distinct): |
mbligh | 8e1ab17 | 2007-09-13 17:29:56 +0000 | [diff] [blame] | 202 | tests.append(klass(db, *row)) |
| 203 | return tests |
| 204 | |
| 205 | |
mbligh | 414c69e | 2007-10-05 15:13:06 +0000 | [diff] [blame] | 206 | @classmethod |
| 207 | def select_sql(klass, db, sql, values): |
| 208 | fields = ['test_idx', 'job_idx', 'test', 'subdir', |
| 209 | 'kernel_idx', 'status', 'reason', 'machine_idx'] |
| 210 | fields = ['t.'+field for field in fields] |
| 211 | rows = db.select_sql(','.join(fields), 'tests', sql, values) |
| 212 | return [klass(db, *row) for row in rows] |
| 213 | |
| 214 | |
mbligh | 3126069 | 2008-04-16 23:12:12 +0000 | [diff] [blame^] | 215 | def __init__(self, db, test_idx, job_idx, testname, subdir, kernel_idx, |
| 216 | status_num, reason, machine_idx): |
mbligh | 8e1ab17 | 2007-09-13 17:29:56 +0000 | [diff] [blame] | 217 | self.idx = test_idx |
mbligh | 250300e | 2007-09-18 00:50:57 +0000 | [diff] [blame] | 218 | self.job = job(db, job_idx) |
mbligh | de7335d | 2007-09-26 16:53:20 +0000 | [diff] [blame] | 219 | self.testname = testname |
mbligh | 8e1ab17 | 2007-09-13 17:29:56 +0000 | [diff] [blame] | 220 | self.subdir = subdir |
mbligh | 50a2525 | 2007-09-27 15:26:17 +0000 | [diff] [blame] | 221 | self.kernel_idx = kernel_idx |
| 222 | self.__kernel = None |
| 223 | self.__iterations = None |
mbligh | 2aaeb67 | 2007-10-01 14:54:18 +0000 | [diff] [blame] | 224 | self.machine_idx = machine_idx |
| 225 | self.__machine = None |
mbligh | 8e1ab17 | 2007-09-13 17:29:56 +0000 | [diff] [blame] | 226 | self.status_num = status_num |
| 227 | self.status_word = db.status_word[status_num] |
mbligh | 9bb92fe | 2007-09-12 15:54:23 +0000 | [diff] [blame] | 228 | self.reason = reason |
mbligh | 50a2525 | 2007-09-27 15:26:17 +0000 | [diff] [blame] | 229 | self.db = db |
mbligh | de7335d | 2007-09-26 16:53:20 +0000 | [diff] [blame] | 230 | if self.subdir: |
| 231 | self.url = html_root + self.job.tag + '/' + self.subdir |
| 232 | else: |
mbligh | 676510c | 2007-09-28 01:28:12 +0000 | [diff] [blame] | 233 | self.url = None |
mbligh | 16ae926 | 2007-09-21 00:53:08 +0000 | [diff] [blame] | 234 | |
mbligh | 50a2525 | 2007-09-27 15:26:17 +0000 | [diff] [blame] | 235 | |
mbligh | 50a2525 | 2007-09-27 15:26:17 +0000 | [diff] [blame] | 236 | def iterations(self): |
| 237 | """ |
| 238 | Caching function for iterations |
| 239 | """ |
| 240 | if not self.__iterations: |
| 241 | self.__iterations = {} |
| 242 | # A dictionary - dict{key} = [value1, value2, ....] |
| 243 | where = {'test_idx' : self.idx} |
| 244 | for i in iteration.select(self.db, where): |
| 245 | if self.__iterations.has_key(i.key): |
| 246 | self.__iterations[i.key].append(i.value) |
| 247 | else: |
| 248 | self.__iterations[i.key] = [i.value] |
| 249 | return self.__iterations |
| 250 | |
| 251 | |
| 252 | def kernel(self): |
| 253 | """ |
| 254 | Caching function for kernels |
| 255 | """ |
| 256 | if not self.__kernel: |
| 257 | where = {'kernel_idx' : self.kernel_idx} |
| 258 | self.__kernel = kernel.select(self.db, where)[0] |
| 259 | return self.__kernel |
| 260 | |
mbligh | 250300e | 2007-09-18 00:50:57 +0000 | [diff] [blame] | 261 | |
mbligh | 2aaeb67 | 2007-10-01 14:54:18 +0000 | [diff] [blame] | 262 | def machine(self): |
| 263 | """ |
| 264 | Caching function for kernels |
| 265 | """ |
| 266 | if not self.__machine: |
| 267 | where = {'machine_idx' : self.machine_idx} |
| 268 | self.__machine = machine.select(self.db, where)[0] |
| 269 | return self.__machine |
| 270 | |
| 271 | |
mbligh | 250300e | 2007-09-18 00:50:57 +0000 | [diff] [blame] | 272 | class job: |
| 273 | def __init__(self, db, job_idx): |
| 274 | where = {'job_idx' : job_idx} |
mbligh | 2aaeb67 | 2007-10-01 14:54:18 +0000 | [diff] [blame] | 275 | rows = db.select('tag, machine_idx', 'jobs', where) |
mbligh | 250300e | 2007-09-18 00:50:57 +0000 | [diff] [blame] | 276 | if not rows: |
| 277 | return None |
mbligh | 2aaeb67 | 2007-10-01 14:54:18 +0000 | [diff] [blame] | 278 | (self.tag, self.machine_idx) = rows[0] |
mbligh | 2b67253 | 2007-11-05 19:24:51 +0000 | [diff] [blame] | 279 | self.job_idx = job_idx |
mbligh | 250300e | 2007-09-18 00:50:57 +0000 | [diff] [blame] | 280 | |
mbligh | 8e1ab17 | 2007-09-13 17:29:56 +0000 | [diff] [blame] | 281 | |
mbligh | 16ae926 | 2007-09-21 00:53:08 +0000 | [diff] [blame] | 282 | class iteration: |
| 283 | @classmethod |
| 284 | def select(klass, db, where): |
| 285 | fields = ['iteration', 'attribute', 'value'] |
| 286 | iterations = [] |
| 287 | rows = db.select(','.join(fields), 'iteration_result', where) |
| 288 | for row in rows: |
| 289 | iterations.append(klass(*row)) |
| 290 | return iterations |
| 291 | |
| 292 | |
| 293 | def __init__(self, iteration, key, value): |
mbligh | 4a370cf | 2008-04-01 19:56:05 +0000 | [diff] [blame] | 294 | self.iteration = iteration |
mbligh | 16ae926 | 2007-09-21 00:53:08 +0000 | [diff] [blame] | 295 | self.key = key |
| 296 | self.value = value |
| 297 | |
mbligh | 8e1ab17 | 2007-09-13 17:29:56 +0000 | [diff] [blame] | 298 | # class patch: |
| 299 | # def __init__(self): |
| 300 | # self.spec = None |