mbligh | 9789445 | 2007-10-05 15:10:33 +0000 | [diff] [blame] | 1 | import MySQLdb, re, os, sys, types |
mbligh | d5c33db | 2006-10-08 21:34:16 +0000 | [diff] [blame] | 2 | |
| 3 | class db: |
mbligh | 432bad4 | 2007-10-09 19:56:07 +0000 | [diff] [blame] | 4 | def __init__(self, debug = False, autocommit=True): |
mbligh | 8e1ab17 | 2007-09-13 17:29:56 +0000 | [diff] [blame] | 5 | self.debug = debug |
mbligh | 432bad4 | 2007-10-09 19:56:07 +0000 | [diff] [blame] | 6 | self.autocommit = autocommit |
mbligh | a218d11 | 2007-10-03 20:15:09 +0000 | [diff] [blame] | 7 | |
| 8 | path = os.path.dirname(os.path.abspath(sys.argv[0])) |
mbligh | b32cd43 | 2007-09-25 18:20:04 +0000 | [diff] [blame] | 9 | try: |
mbligh | a218d11 | 2007-10-03 20:15:09 +0000 | [diff] [blame] | 10 | file = os.path.join(path, '.database') |
mbligh | 432bad4 | 2007-10-09 19:56:07 +0000 | [diff] [blame] | 11 | db_prefs = open(file, 'r') |
mbligh | 728ff0a | 2007-09-27 17:05:12 +0000 | [diff] [blame] | 12 | host = db_prefs.readline().rstrip() |
| 13 | database = db_prefs.readline().rstrip() |
| 14 | except: |
| 15 | host = 'localhost' |
| 16 | database = 'tko' |
| 17 | |
| 18 | try: |
mbligh | a218d11 | 2007-10-03 20:15:09 +0000 | [diff] [blame] | 19 | file = os.path.join(path, '.priv_login') |
| 20 | login = open(file, 'r') |
mbligh | b32cd43 | 2007-09-25 18:20:04 +0000 | [diff] [blame] | 21 | user = login.readline().rstrip() |
| 22 | password = login.readline().rstrip() |
mbligh | 95fca57 | 2007-09-27 17:11:00 +0000 | [diff] [blame] | 23 | except: |
| 24 | try: |
mbligh | a218d11 | 2007-10-03 20:15:09 +0000 | [diff] [blame] | 25 | file = os.path.join(path, '.unpriv_login') |
mbligh | 432bad4 | 2007-10-09 19:56:07 +0000 | [diff] [blame] | 26 | login = open(file, 'r') |
mbligh | 95fca57 | 2007-09-27 17:11:00 +0000 | [diff] [blame] | 27 | user = login.readline().rstrip() |
| 28 | password = login.readline().rstrip() |
| 29 | except: |
| 30 | user = 'nobody' |
| 31 | password = '' |
mbligh | b32cd43 | 2007-09-25 18:20:04 +0000 | [diff] [blame] | 32 | |
| 33 | self.con = MySQLdb.connect(host=host, user=user, |
mbligh | 95fca57 | 2007-09-27 17:11:00 +0000 | [diff] [blame] | 34 | passwd=password, db=database) |
mbligh | d5c33db | 2006-10-08 21:34:16 +0000 | [diff] [blame] | 35 | self.cur = self.con.cursor() |
| 36 | |
mbligh | 8e1ab17 | 2007-09-13 17:29:56 +0000 | [diff] [blame] | 37 | # if not present, insert statuses |
| 38 | self.status_idx = {} |
| 39 | self.status_word = {} |
mbligh | c82f246 | 2007-10-02 19:19:17 +0000 | [diff] [blame] | 40 | status_rows = self.select('status_idx, word', 'status', None) |
| 41 | for s in status_rows: |
mbligh | 9789445 | 2007-10-05 15:10:33 +0000 | [diff] [blame] | 42 | self.status_idx[s[1]] = s[0] |
mbligh | c82f246 | 2007-10-02 19:19:17 +0000 | [diff] [blame] | 43 | self.status_word[s[0]] = s[1] |
mbligh | 048e1c9 | 2007-10-07 00:10:33 +0000 | [diff] [blame] | 44 | |
| 45 | dir = os.path.abspath(os.path.dirname(sys.argv[0])) |
| 46 | machine_map = os.path.join(dir, 'machines') |
| 47 | if os.path.exists(machine_map): |
| 48 | self.machine_map = machine_map |
mbligh | ba9c54c | 2007-10-26 16:41:26 +0000 | [diff] [blame] | 49 | else: |
| 50 | self.machine_map = None |
mbligh | 048e1c9 | 2007-10-07 00:10:33 +0000 | [diff] [blame] | 51 | self.machine_group = {} |
| 52 | |
mbligh | 8e1ab17 | 2007-09-13 17:29:56 +0000 | [diff] [blame] | 53 | |
mbligh | 8e1ab17 | 2007-09-13 17:29:56 +0000 | [diff] [blame] | 54 | def dprint(self, value): |
| 55 | if self.debug: |
| 56 | sys.stderr.write('SQL: ' + str(value) + '\n') |
| 57 | |
mbligh | d5c33db | 2006-10-08 21:34:16 +0000 | [diff] [blame] | 58 | |
mbligh | 432bad4 | 2007-10-09 19:56:07 +0000 | [diff] [blame] | 59 | def commit(self): |
| 60 | self.con.commit() |
| 61 | |
| 62 | |
mbligh | 31d29c4 | 2007-09-27 00:51:33 +0000 | [diff] [blame] | 63 | def select(self, fields, table, where, distinct = False): |
mbligh | 608c325 | 2007-08-31 13:53:00 +0000 | [diff] [blame] | 64 | """\ |
| 65 | select fields from table where {dictionary} |
| 66 | """ |
mbligh | 31d29c4 | 2007-09-27 00:51:33 +0000 | [diff] [blame] | 67 | cmd = ['select'] |
| 68 | if distinct: |
| 69 | cmd.append('distinct') |
| 70 | cmd += [fields, 'from', table] |
mbligh | 608c325 | 2007-08-31 13:53:00 +0000 | [diff] [blame] | 71 | |
mbligh | 31d29c4 | 2007-09-27 00:51:33 +0000 | [diff] [blame] | 72 | values = [] |
mbligh | 414c69e | 2007-10-05 15:13:06 +0000 | [diff] [blame] | 73 | if where and isinstance(where, types.DictionaryType): |
mbligh | 53d1425 | 2007-09-12 16:33:14 +0000 | [diff] [blame] | 74 | keys = [field + '=%s' for field in where.keys()] |
| 75 | values = [where[field] for field in where.keys()] |
| 76 | |
mbligh | 31d29c4 | 2007-09-27 00:51:33 +0000 | [diff] [blame] | 77 | cmd.append(' where ' + ' and '.join(keys)) |
mbligh | 414c69e | 2007-10-05 15:13:06 +0000 | [diff] [blame] | 78 | elif where and isinstance(where, types.StringTypes): |
| 79 | cmd.append(' where ' + where) |
mbligh | 53d1425 | 2007-09-12 16:33:14 +0000 | [diff] [blame] | 80 | |
mbligh | 31d29c4 | 2007-09-27 00:51:33 +0000 | [diff] [blame] | 81 | self.dprint('%s %s' % (' '.join(cmd),values)) |
| 82 | self.cur.execute(' '.join(cmd), values) |
mbligh | d5c33db | 2006-10-08 21:34:16 +0000 | [diff] [blame] | 83 | return self.cur.fetchall() |
| 84 | |
mbligh | 056d0d3 | 2006-10-08 22:31:10 +0000 | [diff] [blame] | 85 | |
mbligh | 414c69e | 2007-10-05 15:13:06 +0000 | [diff] [blame] | 86 | def select_sql(self, fields, table, sql, values): |
| 87 | """\ |
| 88 | select fields from table "sql" |
| 89 | """ |
| 90 | cmd = 'select %s from %s %s' % (fields, table, sql) |
| 91 | self.dprint(cmd) |
| 92 | self.cur.execute(cmd, values) |
| 93 | return self.cur.fetchall() |
| 94 | |
| 95 | |
mbligh | 432bad4 | 2007-10-09 19:56:07 +0000 | [diff] [blame] | 96 | def insert(self, table, data, commit = None): |
mbligh | 608c325 | 2007-08-31 13:53:00 +0000 | [diff] [blame] | 97 | """\ |
| 98 | 'insert into table (keys) values (%s ... %s)', values |
| 99 | |
| 100 | data: |
| 101 | dictionary of fields and data |
| 102 | """ |
mbligh | 432bad4 | 2007-10-09 19:56:07 +0000 | [diff] [blame] | 103 | if commit == None: |
| 104 | commit = self.autocommit |
mbligh | 608c325 | 2007-08-31 13:53:00 +0000 | [diff] [blame] | 105 | fields = data.keys() |
| 106 | refs = ['%s' for field in fields] |
| 107 | values = [data[field] for field in fields] |
| 108 | cmd = 'insert into %s (%s) values (%s)' % \ |
| 109 | (table, ','.join(fields), ','.join(refs)) |
mbligh | 53d1425 | 2007-09-12 16:33:14 +0000 | [diff] [blame] | 110 | |
mbligh | 8e1ab17 | 2007-09-13 17:29:56 +0000 | [diff] [blame] | 111 | self.dprint('%s %s' % (cmd,values)) |
mbligh | 608c325 | 2007-08-31 13:53:00 +0000 | [diff] [blame] | 112 | self.cur.execute(cmd, values) |
mbligh | 432bad4 | 2007-10-09 19:56:07 +0000 | [diff] [blame] | 113 | if commit: |
| 114 | self.con.commit() |
mbligh | 608c325 | 2007-08-31 13:53:00 +0000 | [diff] [blame] | 115 | |
| 116 | |
mbligh | 96b9a5a | 2007-11-24 19:32:20 +0000 | [diff] [blame^] | 117 | def delete(self, table, where, commit = None): |
| 118 | cmd = ['delete from', table] |
| 119 | if commit == None: |
| 120 | commit = self.autocommit |
| 121 | if where and isinstance(where, types.DictionaryType): |
| 122 | keys = [field + '=%s' for field in where.keys()] |
| 123 | values = [where[field] for field in where.keys()] |
| 124 | cmd += ['where', ' and '.join(keys)] |
| 125 | self.dprint('%s %s' % (' '.join(cmd),values)) |
| 126 | self.cur.execute(' '.join(cmd), values) |
| 127 | if commit: |
| 128 | self.con.commit() |
| 129 | |
| 130 | |
mbligh | 432bad4 | 2007-10-09 19:56:07 +0000 | [diff] [blame] | 131 | def update(self, table, data, where, commit = None): |
mbligh | 414c69e | 2007-10-05 15:13:06 +0000 | [diff] [blame] | 132 | """\ |
| 133 | 'update table set data values (%s ... %s) where ...' |
| 134 | |
| 135 | data: |
| 136 | dictionary of fields and data |
| 137 | """ |
mbligh | 432bad4 | 2007-10-09 19:56:07 +0000 | [diff] [blame] | 138 | if commit == None: |
| 139 | commit = self.autocommit |
mbligh | 414c69e | 2007-10-05 15:13:06 +0000 | [diff] [blame] | 140 | cmd = 'update %s ' % table |
| 141 | fields = data.keys() |
| 142 | data_refs = [field + '=%s' for field in fields] |
| 143 | data_values = [data[field] for field in fields] |
| 144 | cmd += ' set ' + ' and '.join(data_refs) |
| 145 | |
| 146 | where_keys = [field + '=%s' for field in where.keys()] |
| 147 | where_values = [where[field] for field in where.keys()] |
| 148 | cmd += ' where ' + ' and '.join(where_keys) |
| 149 | |
| 150 | print '%s %s' % (cmd, data_values + where_values) |
| 151 | self.cur.execute(cmd, data_values + where_values) |
mbligh | 432bad4 | 2007-10-09 19:56:07 +0000 | [diff] [blame] | 152 | if commit: |
| 153 | self.con.commit() |
mbligh | 414c69e | 2007-10-05 15:13:06 +0000 | [diff] [blame] | 154 | |
| 155 | |
mbligh | 96b9a5a | 2007-11-24 19:32:20 +0000 | [diff] [blame^] | 156 | def delete_job(self, tag, commit = None): |
| 157 | job_idx = self.find_job(tag) |
| 158 | for test_idx in self.find_tests(job_idx): |
| 159 | where = {'test_idx' : test_idx} |
| 160 | self.delete('iteration_result', where) |
| 161 | self.delete('test_attributes', where) |
| 162 | where = {'job_idx' : job_idx} |
| 163 | self.delete('tests', where) |
| 164 | self.delete('jobs', where) |
| 165 | |
| 166 | |
mbligh | 432bad4 | 2007-10-09 19:56:07 +0000 | [diff] [blame] | 167 | def insert_job(self, tag, job, commit = None): |
mbligh | 2aaeb67 | 2007-10-01 14:54:18 +0000 | [diff] [blame] | 168 | job.machine_idx = self.lookup_machine(job.machine) |
| 169 | if not job.machine_idx: |
mbligh | 432bad4 | 2007-10-09 19:56:07 +0000 | [diff] [blame] | 170 | job.machine_idx = self.insert_machine(job.machine, |
| 171 | commit=commit) |
mbligh | b10a60f | 2007-10-17 00:03:32 +0000 | [diff] [blame] | 172 | self.insert('jobs', {'tag':tag, |
| 173 | 'label': job.label, |
| 174 | 'user': job.user, |
| 175 | 'machine_idx':job.machine_idx}, |
| 176 | commit=commit) |
mbligh | 608c325 | 2007-08-31 13:53:00 +0000 | [diff] [blame] | 177 | job.index = self.find_job(tag) |
| 178 | for test in job.tests: |
mbligh | 432bad4 | 2007-10-09 19:56:07 +0000 | [diff] [blame] | 179 | self.insert_test(job, test, commit=commit) |
mbligh | 608c325 | 2007-08-31 13:53:00 +0000 | [diff] [blame] | 180 | |
mbligh | 96b9a5a | 2007-11-24 19:32:20 +0000 | [diff] [blame^] | 181 | |
mbligh | 432bad4 | 2007-10-09 19:56:07 +0000 | [diff] [blame] | 182 | def insert_test(self, job, test, commit = None): |
| 183 | kver = self.insert_kernel(test.kernel, commit=commit) |
mbligh | 608c325 | 2007-08-31 13:53:00 +0000 | [diff] [blame] | 184 | data = {'job_idx':job.index, 'test':test.testname, |
mbligh | 2bd4887 | 2007-09-20 18:32:25 +0000 | [diff] [blame] | 185 | 'subdir':test.subdir, 'kernel_idx':kver, |
mbligh | 8e1ab17 | 2007-09-13 17:29:56 +0000 | [diff] [blame] | 186 | 'status':self.status_idx[test.status], |
mbligh | 2aaeb67 | 2007-10-01 14:54:18 +0000 | [diff] [blame] | 187 | 'reason':test.reason, 'machine_idx':job.machine_idx } |
mbligh | 432bad4 | 2007-10-09 19:56:07 +0000 | [diff] [blame] | 188 | self.insert('tests', data, commit=commit) |
mbligh | 2bd4887 | 2007-09-20 18:32:25 +0000 | [diff] [blame] | 189 | test_idx = self.find_test(job.index, test.subdir) |
| 190 | data = { 'test_idx':test_idx } |
| 191 | |
| 192 | for i in test.iterations: |
| 193 | data['iteration'] = i.index |
| 194 | for key in i.keyval: |
| 195 | data['attribute'] = key |
| 196 | data['value'] = i.keyval[key] |
mbligh | 432bad4 | 2007-10-09 19:56:07 +0000 | [diff] [blame] | 197 | self.insert('iteration_result', |
| 198 | data, |
| 199 | commit=commit) |
mbligh | 994a23d | 2007-10-25 15:28:58 +0000 | [diff] [blame] | 200 | data = {'test_idx':test_idx, 'attribute':'version', 'value':test.version} |
| 201 | self.insert('test_attributes', data, commit=commit) |
mbligh | e9cf9d4 | 2007-08-31 08:56:00 +0000 | [diff] [blame] | 202 | |
| 203 | |
mbligh | 048e1c9 | 2007-10-07 00:10:33 +0000 | [diff] [blame] | 204 | def read_machine_map(self): |
| 205 | self.machine_group = {} |
| 206 | for line in open(self.machine_map, 'r').readlines(): |
| 207 | (machine, group) = line.split() |
| 208 | self.machine_group[machine] = group |
| 209 | |
| 210 | |
mbligh | 432bad4 | 2007-10-09 19:56:07 +0000 | [diff] [blame] | 211 | def insert_machine(self, hostname, group = None, commit = None): |
mbligh | 048e1c9 | 2007-10-07 00:10:33 +0000 | [diff] [blame] | 212 | if self.machine_map and not self.machine_group: |
| 213 | self.read_machine_map() |
| 214 | |
| 215 | if not group: |
| 216 | group = self.machine_group.get(hostname, hostname) |
| 217 | |
mbligh | 432bad4 | 2007-10-09 19:56:07 +0000 | [diff] [blame] | 218 | self.insert('machines', |
| 219 | { 'hostname' : hostname , |
| 220 | 'machine_group' : group }, |
| 221 | commit=commit) |
mbligh | 2aaeb67 | 2007-10-01 14:54:18 +0000 | [diff] [blame] | 222 | return self.lookup_machine(hostname) |
| 223 | |
| 224 | |
| 225 | def lookup_machine(self, hostname): |
| 226 | where = { 'hostname' : hostname } |
| 227 | rows = self.select('machine_idx', 'machines', where) |
| 228 | if rows: |
| 229 | return rows[0][0] |
| 230 | else: |
| 231 | return None |
| 232 | |
| 233 | |
mbligh | 9bb92fe | 2007-09-12 15:54:23 +0000 | [diff] [blame] | 234 | def lookup_kernel(self, kernel): |
| 235 | rows = self.select('kernel_idx', 'kernels', |
mbligh | 048e1c9 | 2007-10-07 00:10:33 +0000 | [diff] [blame] | 236 | {'kernel_hash':kernel.kernel_hash}) |
mbligh | 237bed3 | 2007-09-05 13:05:57 +0000 | [diff] [blame] | 237 | if rows: |
| 238 | return rows[0][0] |
| 239 | else: |
| 240 | return None |
mbligh | e9cf9d4 | 2007-08-31 08:56:00 +0000 | [diff] [blame] | 241 | |
| 242 | |
mbligh | 432bad4 | 2007-10-09 19:56:07 +0000 | [diff] [blame] | 243 | def insert_kernel(self, kernel, commit = None): |
mbligh | 9bb92fe | 2007-09-12 15:54:23 +0000 | [diff] [blame] | 244 | kver = self.lookup_kernel(kernel) |
mbligh | 237bed3 | 2007-09-05 13:05:57 +0000 | [diff] [blame] | 245 | if kver: |
| 246 | return kver |
mbligh | 432bad4 | 2007-10-09 19:56:07 +0000 | [diff] [blame] | 247 | self.insert('kernels', |
| 248 | {'base':kernel.base, |
| 249 | 'kernel_hash':kernel.kernel_hash, |
| 250 | 'printable':kernel.base}, |
| 251 | commit=commit) |
mbligh | 9bb92fe | 2007-09-12 15:54:23 +0000 | [diff] [blame] | 252 | # WARNING - incorrectly shoving base into printable here. |
| 253 | kver = self.lookup_kernel(kernel) |
mbligh | 237bed3 | 2007-09-05 13:05:57 +0000 | [diff] [blame] | 254 | for patch in kernel.patches: |
mbligh | 432bad4 | 2007-10-09 19:56:07 +0000 | [diff] [blame] | 255 | self.insert_patch(kver, patch, commit=commit) |
mbligh | 237bed3 | 2007-09-05 13:05:57 +0000 | [diff] [blame] | 256 | return kver |
| 257 | |
| 258 | |
mbligh | 432bad4 | 2007-10-09 19:56:07 +0000 | [diff] [blame] | 259 | def insert_patch(self, kver, patch, commit = None): |
mbligh | 237bed3 | 2007-09-05 13:05:57 +0000 | [diff] [blame] | 260 | print patch.reference |
| 261 | name = os.path.basename(patch.reference)[:80] |
mbligh | 432bad4 | 2007-10-09 19:56:07 +0000 | [diff] [blame] | 262 | self.insert('patches', |
| 263 | {'kernel_idx': kver, |
| 264 | 'name':name, |
| 265 | 'url':patch.reference, |
| 266 | 'hash':patch.hash}, |
| 267 | commit=commit) |
mbligh | 056d0d3 | 2006-10-08 22:31:10 +0000 | [diff] [blame] | 268 | |
mbligh | 048e1c9 | 2007-10-07 00:10:33 +0000 | [diff] [blame] | 269 | |
mbligh | 2bd4887 | 2007-09-20 18:32:25 +0000 | [diff] [blame] | 270 | def find_test(self, job_idx, subdir): |
| 271 | where = { 'job_idx':job_idx , 'subdir':subdir } |
| 272 | rows = self.select('test_idx', 'tests', where) |
| 273 | if rows: |
| 274 | return rows[0][0] |
| 275 | else: |
| 276 | return None |
| 277 | |
mbligh | 056d0d3 | 2006-10-08 22:31:10 +0000 | [diff] [blame] | 278 | |
mbligh | 96b9a5a | 2007-11-24 19:32:20 +0000 | [diff] [blame^] | 279 | def find_tests(self, job_idx): |
| 280 | where = { 'job_idx':job_idx } |
| 281 | rows = self.select('test_idx', 'tests', where) |
| 282 | if rows: |
| 283 | return [row[0] for row in rows] |
| 284 | else: |
| 285 | return None |
| 286 | |
| 287 | |
mbligh | 056d0d3 | 2006-10-08 22:31:10 +0000 | [diff] [blame] | 288 | def find_job(self, tag): |
mbligh | 608c325 | 2007-08-31 13:53:00 +0000 | [diff] [blame] | 289 | rows = self.select('job_idx', 'jobs', {'tag': tag}) |
| 290 | if rows: |
| 291 | return rows[0][0] |
| 292 | else: |
| 293 | return None |