mbligh | c6b01ed | 2006-10-13 17:03:26 +0000 | [diff] [blame] | 1 | #!/usr/bin/python |
mbligh | 74fc046 | 2007-11-05 20:24:17 +0000 | [diff] [blame] | 2 | import os, re, md5, sys, email.Message, smtplib |
mbligh | 532cb27 | 2007-11-26 18:54:20 +0000 | [diff] [blame^] | 3 | client_bin = os.path.join(os.path.dirname(__file__), '../client/bin') |
| 4 | sys.path.insert(0, os.path.abspath(client_bin)) |
| 5 | from autotest_utils import read_keyval |
mbligh | bb7b891 | 2006-10-08 03:59:02 +0000 | [diff] [blame] | 6 | |
mbligh | b10a60f | 2007-10-17 00:03:32 +0000 | [diff] [blame] | 7 | user = re.compile(r'user(\s*)=') |
| 8 | label = re.compile(r'label(\s*)=') |
mbligh | b85e6b0 | 2006-10-08 17:20:56 +0000 | [diff] [blame] | 9 | |
mbligh | 006f230 | 2007-09-13 20:46:46 +0000 | [diff] [blame] | 10 | debug = True |
| 11 | |
mbligh | 74fc046 | 2007-11-05 20:24:17 +0000 | [diff] [blame] | 12 | # XXX: these mail bits came almost verbatim from mirror/mirror and this should |
| 13 | # probably be refactored into another file and used by both. |
| 14 | def mail(from_address, to_addresses, cc_addresses, subject, message_text): |
mbligh | 08415d8 | 2007-11-24 19:23:34 +0000 | [diff] [blame] | 15 | # if passed a string for the to_addresses convert it to a tuple |
| 16 | if type(to_addresses) is str: |
| 17 | to_addresses = (to_addresses,) |
mbligh | 74fc046 | 2007-11-05 20:24:17 +0000 | [diff] [blame] | 18 | |
mbligh | 08415d8 | 2007-11-24 19:23:34 +0000 | [diff] [blame] | 19 | message = email.Message.Message() |
| 20 | message["To"] = ", ".join(to_addresses) |
| 21 | message["Cc"] = ", ".join(cc_addresses) |
| 22 | message["From"] = from_address |
| 23 | message["Subject"] = subject |
| 24 | message.set_payload(message_text) |
mbligh | 74fc046 | 2007-11-05 20:24:17 +0000 | [diff] [blame] | 25 | |
mbligh | 08415d8 | 2007-11-24 19:23:34 +0000 | [diff] [blame] | 26 | try: |
| 27 | sendmail(message.as_string()) |
| 28 | except SendmailException, e: |
| 29 | server = smtplib.SMTP("localhost") |
| 30 | server.sendmail(from_address, to_addresses, cc_addresses, message.as_string()) |
| 31 | server.quit() |
mbligh | 74fc046 | 2007-11-05 20:24:17 +0000 | [diff] [blame] | 32 | |
| 33 | |
| 34 | MAIL = "sendmail" |
| 35 | |
| 36 | class SendmailException(Exception): |
mbligh | 08415d8 | 2007-11-24 19:23:34 +0000 | [diff] [blame] | 37 | pass |
mbligh | 74fc046 | 2007-11-05 20:24:17 +0000 | [diff] [blame] | 38 | |
| 39 | def sendmail(message): |
mbligh | 08415d8 | 2007-11-24 19:23:34 +0000 | [diff] [blame] | 40 | """Send an email using sendmail""" |
| 41 | # open a pipe to the mail program and |
| 42 | # write the data to the pipe |
| 43 | p = os.popen("%s -t" % MAIL, 'w') |
| 44 | p.write(message) |
| 45 | exitcode = p.close() |
| 46 | if exitcode: |
| 47 | raise SendmailException("Exit code: %s" % exitcode) |
mbligh | 74fc046 | 2007-11-05 20:24:17 +0000 | [diff] [blame] | 48 | |
| 49 | # XXX: End of code from mirror/mirror |
| 50 | |
| 51 | |
mbligh | b85e6b0 | 2006-10-08 17:20:56 +0000 | [diff] [blame] | 52 | def shorten_patch(long): |
mbligh | 340c968 | 2006-10-08 18:26:08 +0000 | [diff] [blame] | 53 | short = os.path.basename(long) |
mbligh | b85e6b0 | 2006-10-08 17:20:56 +0000 | [diff] [blame] | 54 | short = re.sub(r'^patch-', '', short) |
| 55 | short = re.sub(r'\.(bz2|gz)$', '', short) |
| 56 | short = re.sub(r'\.patch$', '', short) |
| 57 | short = re.sub(r'\+', '_', short) |
| 58 | return short |
| 59 | |
mbligh | bb7b891 | 2006-10-08 03:59:02 +0000 | [diff] [blame] | 60 | |
mbligh | 006f230 | 2007-09-13 20:46:46 +0000 | [diff] [blame] | 61 | def dprint(info): |
| 62 | if debug: |
| 63 | sys.stderr.write(str(info) + '\n') |
| 64 | |
| 65 | |
mbligh | e9cf9d4 | 2007-08-31 08:56:00 +0000 | [diff] [blame] | 66 | class job: |
mbligh | 532cb27 | 2007-11-26 18:54:20 +0000 | [diff] [blame^] | 67 | def __init__(self, dir): |
mbligh | e9cf9d4 | 2007-08-31 08:56:00 +0000 | [diff] [blame] | 68 | self.dir = dir |
mbligh | e9cf9d4 | 2007-08-31 08:56:00 +0000 | [diff] [blame] | 69 | self.control = os.path.join(dir, "control") |
mbligh | 237bed3 | 2007-09-05 13:05:57 +0000 | [diff] [blame] | 70 | self.status = os.path.join(dir, "status") |
mbligh | bb7b891 | 2006-10-08 03:59:02 +0000 | [diff] [blame] | 71 | self.variables = {} |
mbligh | e9cf9d4 | 2007-08-31 08:56:00 +0000 | [diff] [blame] | 72 | self.tests = [] |
mbligh | bb7b891 | 2006-10-08 03:59:02 +0000 | [diff] [blame] | 73 | self.kernel = None |
| 74 | |
mbligh | b10a60f | 2007-10-17 00:03:32 +0000 | [diff] [blame] | 75 | # Get the user + tag info from the keyval file. |
mbligh | 8e1ab17 | 2007-09-13 17:29:56 +0000 | [diff] [blame] | 76 | try: |
mbligh | 532cb27 | 2007-11-26 18:54:20 +0000 | [diff] [blame^] | 77 | keyval = read_keyval(dir) |
| 78 | print keyval |
mbligh | 8e1ab17 | 2007-09-13 17:29:56 +0000 | [diff] [blame] | 79 | except: |
mbligh | 532cb27 | 2007-11-26 18:54:20 +0000 | [diff] [blame^] | 80 | raise |
| 81 | keyval = {} |
| 82 | self.user = keyval.get('user', None) |
| 83 | self.label = keyval.get('label', None) |
| 84 | self.machine = keyval.get('hostname', None) |
| 85 | |
| 86 | if not self.machine: |
| 87 | self.get_machine() |
| 88 | |
| 89 | print 'MACHINE NAME: ' + self.machine |
| 90 | if not os.path.exists(self.status): |
mbligh | 8e1ab17 | 2007-09-13 17:29:56 +0000 | [diff] [blame] | 91 | return None |
| 92 | |
mbligh | 529f2f3 | 2007-08-30 11:22:50 +0000 | [diff] [blame] | 93 | self.grope_status() |
mbligh | bb7b891 | 2006-10-08 03:59:02 +0000 | [diff] [blame] | 94 | |
| 95 | |
mbligh | 532cb27 | 2007-11-26 18:54:20 +0000 | [diff] [blame^] | 96 | def get_machine(self): |
| 97 | try: |
| 98 | hostname = os.path.join(dir, "sysinfo/hostname") |
| 99 | self.machine = open(hostname, 'r').readline().rstrip() |
| 100 | return |
| 101 | except: |
| 102 | pass |
| 103 | try: |
| 104 | uname = os.path.join(dir, "sysinfo/uname_-a") |
| 105 | self.machine = open(uname, 'r').readline().split()[1] |
| 106 | return |
| 107 | except: |
| 108 | pass |
| 109 | |
| 110 | |
mbligh | d5c33db | 2006-10-08 21:34:16 +0000 | [diff] [blame] | 111 | def grope_status(self): |
mbligh | de7335d | 2007-09-26 16:53:20 +0000 | [diff] [blame] | 112 | dprint('=====================================================') |
| 113 | dprint(self.dir) |
| 114 | dprint('=====================================================') |
mbligh | 049f5e6 | 2007-09-30 02:00:14 +0000 | [diff] [blame] | 115 | self.kernel = kernel(self.dir) |
mbligh | 237bed3 | 2007-09-05 13:05:57 +0000 | [diff] [blame] | 116 | |
mbligh | de7335d | 2007-09-26 16:53:20 +0000 | [diff] [blame] | 117 | # NOTE: currently we don't cope with nested START / END blocks |
| 118 | group_subdir = None |
mbligh | 237bed3 | 2007-09-05 13:05:57 +0000 | [diff] [blame] | 119 | for line in open(self.status, 'r').readlines(): |
mbligh | de7335d | 2007-09-26 16:53:20 +0000 | [diff] [blame] | 120 | dprint('STATUS: ' + line.rstrip()) |
| 121 | if not re.match(r'\t*\S', line): |
| 122 | continue # ignore continuation lines |
| 123 | if re.match(r'\t*START', line): |
| 124 | group_subdir = None |
| 125 | continue # ignore start lines |
| 126 | reason = None |
| 127 | if line.startswith('END'): |
| 128 | elements = line.split(None, 4)[1:] |
| 129 | else: |
| 130 | elements = line.split(None, 3) |
| 131 | elements.append(None) # in case no reason specified |
| 132 | (status, subdir, testname, reason) = elements[0:4] |
mbligh | 59a479f | 2007-11-24 19:24:05 +0000 | [diff] [blame] | 133 | if testname == '----': |
| 134 | # This is a job level event, not a test |
| 135 | continue |
mbligh | de7335d | 2007-09-26 16:53:20 +0000 | [diff] [blame] | 136 | ################################################ |
| 137 | # REMOVE THIS SECTION ONCE OLD FORMAT JOBS ARE GONE |
| 138 | ################################################ |
| 139 | if re.match(r'(GOOD|FAIL|WARN) ', line): |
| 140 | (status, testname, reason) = line.split(None, 2) |
| 141 | if testname.startswith('kernel.'): |
| 142 | subdir = 'build' |
| 143 | else: |
| 144 | subdir = testname |
| 145 | if testname.startswith('completed'): |
| 146 | raise 'testname is crap' |
| 147 | ################################################ |
| 148 | if subdir == '----': |
| 149 | subdir = None |
| 150 | if line.startswith('END'): |
| 151 | subdir = group_subdir |
| 152 | if line.startswith('\t'): # we're in a block group |
| 153 | if subdir: |
| 154 | group_subdir = subdir |
| 155 | continue |
| 156 | debug = str((status, subdir, testname, reason)) |
| 157 | dprint('GROPE_STATUS: ' + debug) |
| 158 | if not re.match(r'(boot$|kernel\.)', testname): |
| 159 | # This is a real test |
| 160 | if subdir and subdir.count('.'): |
| 161 | # eg dbench.ext3 |
| 162 | testname = subdir |
| 163 | self.tests.append(test(subdir, testname, status, reason, self.kernel, self)) |
| 164 | dprint('') |
mbligh | e9cf9d4 | 2007-08-31 08:56:00 +0000 | [diff] [blame] | 165 | |
| 166 | |
| 167 | class kernel: |
mbligh | 049f5e6 | 2007-09-30 02:00:14 +0000 | [diff] [blame] | 168 | def __init__(self, topdir): |
mbligh | e9cf9d4 | 2007-08-31 08:56:00 +0000 | [diff] [blame] | 169 | self.base = None |
| 170 | self.patches = [] |
mbligh | e9cf9d4 | 2007-08-31 08:56:00 +0000 | [diff] [blame] | 171 | patch_hashes = [] |
mbligh | 049f5e6 | 2007-09-30 02:00:14 +0000 | [diff] [blame] | 172 | # HACK. we don't have proper build tags in the status file yet |
| 173 | # so we hardcode build/ and do it at the start of the job |
mbligh | 9c39d15 | 2007-11-24 19:11:58 +0000 | [diff] [blame] | 174 | build_log = os.path.join(topdir, 'build/debug/build_log') |
mbligh | 049f5e6 | 2007-09-30 02:00:14 +0000 | [diff] [blame] | 175 | |
mbligh | 9c39d15 | 2007-11-24 19:11:58 +0000 | [diff] [blame] | 176 | if os.path.exists(build_log): |
| 177 | for line in open(build_log, 'r'): |
mbligh | 049f5e6 | 2007-09-30 02:00:14 +0000 | [diff] [blame] | 178 | print line |
| 179 | (type, rest) = line.split(': ', 1) |
| 180 | words = rest.split() |
| 181 | if type == 'BASE': |
| 182 | self.base = words[0] |
| 183 | if type == 'PATCH': |
| 184 | print words |
| 185 | self.patches.append(patch(*words[0:])) |
| 186 | # patch_hashes.append(words[2]) |
mbligh | 9c39d15 | 2007-11-24 19:11:58 +0000 | [diff] [blame] | 187 | else: |
| 188 | for sysinfo in ['sysinfo/reboot1', 'sysinfo']: |
| 189 | uname_file = os.path.join(topdir, sysinfo, 'uname_-a') |
| 190 | if not os.path.exists(uname_file): |
| 191 | continue |
| 192 | uname = open(uname_file, 'r').readline().split() |
| 193 | self.base = uname[2] |
| 194 | break |
| 195 | print 'kernel.__init__() found kernel version %s' % self.base |
mbligh | 52f9744 | 2007-09-14 17:43:28 +0000 | [diff] [blame] | 196 | if self.base: |
| 197 | self.kernel_hash = self.get_kver_hash(self.base, patch_hashes) |
mbligh | e9cf9d4 | 2007-08-31 08:56:00 +0000 | [diff] [blame] | 198 | |
| 199 | |
mbligh | 237bed3 | 2007-09-05 13:05:57 +0000 | [diff] [blame] | 200 | def get_kver_hash(self, base, patch_hashes): |
mbligh | e9cf9d4 | 2007-08-31 08:56:00 +0000 | [diff] [blame] | 201 | """\ |
| 202 | Calculate a hash representing the unique combination of |
| 203 | the kernel base version plus |
| 204 | """ |
| 205 | key_string = ','.join([base] + patch_hashes) |
| 206 | return md5.new(key_string).hexdigest() |
| 207 | |
| 208 | |
mbligh | 237bed3 | 2007-09-05 13:05:57 +0000 | [diff] [blame] | 209 | class patch: |
| 210 | def __init__(self, spec, reference=None, hash=None): |
| 211 | # NEITHER OF THE ABOVE SHOULD HAVE DEFAULTS!!!! HACK HACK |
| 212 | if not reference: |
| 213 | reference = spec |
| 214 | print 'PATCH::%s %s %s' % (spec, reference, hash) |
| 215 | self.spec = spec |
| 216 | self.reference = reference |
| 217 | self.hash = hash |
| 218 | |
| 219 | |
mbligh | e9cf9d4 | 2007-08-31 08:56:00 +0000 | [diff] [blame] | 220 | class test: |
mbligh | de7335d | 2007-09-26 16:53:20 +0000 | [diff] [blame] | 221 | def __init__(self, subdir, testname, status, reason, kernel, job): |
mbligh | de94094 | 2007-11-05 17:25:56 +0000 | [diff] [blame] | 222 | # NOTE: subdir may be none here for lines that aren't an |
| 223 | # actual test |
mbligh | 2bd4887 | 2007-09-20 18:32:25 +0000 | [diff] [blame] | 224 | self.subdir = subdir |
mbligh | de7335d | 2007-09-26 16:53:20 +0000 | [diff] [blame] | 225 | self.testname = testname |
mbligh | e9cf9d4 | 2007-08-31 08:56:00 +0000 | [diff] [blame] | 226 | self.status = status |
| 227 | self.reason = reason |
mbligh | de94094 | 2007-11-05 17:25:56 +0000 | [diff] [blame] | 228 | self.version = None |
| 229 | self.keyval = None |
mbligh | 994a23d | 2007-10-25 15:28:58 +0000 | [diff] [blame] | 230 | |
mbligh | de7335d | 2007-09-26 16:53:20 +0000 | [diff] [blame] | 231 | if subdir: |
mbligh | de94094 | 2007-11-05 17:25:56 +0000 | [diff] [blame] | 232 | keyval = os.path.join(job.dir, subdir, 'results/keyval') |
| 233 | if os.path.exists(keyval): |
| 234 | self.keyval = keyval |
| 235 | keyval2 = os.path.join(job.dir, subdir, 'keyval') |
| 236 | if os.path.exists(keyval2): |
| 237 | self.version = open(keyval2, 'r').readline().split('=')[1] |
mbligh | de7335d | 2007-09-26 16:53:20 +0000 | [diff] [blame] | 238 | else: |
| 239 | self.keyval = None |
mbligh | e9cf9d4 | 2007-08-31 08:56:00 +0000 | [diff] [blame] | 240 | self.iterations = [] |
mbligh | 237bed3 | 2007-09-05 13:05:57 +0000 | [diff] [blame] | 241 | self.kernel = kernel |
mbligh | 8e1ab17 | 2007-09-13 17:29:56 +0000 | [diff] [blame] | 242 | self.machine = job.machine |
mbligh | e9cf9d4 | 2007-08-31 08:56:00 +0000 | [diff] [blame] | 243 | |
mbligh | de7335d | 2007-09-26 16:53:20 +0000 | [diff] [blame] | 244 | dprint("PARSING TEST %s %s %s" % (subdir, testname, self.keyval)) |
mbligh | 994a23d | 2007-10-25 15:28:58 +0000 | [diff] [blame] | 245 | |
mbligh | de7335d | 2007-09-26 16:53:20 +0000 | [diff] [blame] | 246 | if not self.keyval: |
mbligh | e9cf9d4 | 2007-08-31 08:56:00 +0000 | [diff] [blame] | 247 | return |
| 248 | count = 1 |
| 249 | lines = [] |
| 250 | for line in open(self.keyval, 'r').readlines(): |
mbligh | 2bd4887 | 2007-09-20 18:32:25 +0000 | [diff] [blame] | 251 | if not re.search('\S', line): # blank line |
mbligh | e9cf9d4 | 2007-08-31 08:56:00 +0000 | [diff] [blame] | 252 | self.iterations.append(iteration(count, lines)) |
| 253 | lines = [] |
| 254 | count += 1 |
mbligh | e9cf9d4 | 2007-08-31 08:56:00 +0000 | [diff] [blame] | 255 | else: |
| 256 | lines.append(line) |
| 257 | if lines: |
| 258 | self.iterations.append(iteration(count, lines)) |
| 259 | |
| 260 | |
| 261 | class iteration: |
| 262 | def __init__(self, index, lines): |
| 263 | self.index = index |
| 264 | self.keyval = {} |
| 265 | |
mbligh | 2bd4887 | 2007-09-20 18:32:25 +0000 | [diff] [blame] | 266 | dprint("ADDING ITERATION %d" % index) |
mbligh | e9cf9d4 | 2007-08-31 08:56:00 +0000 | [diff] [blame] | 267 | for line in lines: |
| 268 | (key, value) = line.split('=', 1) |
| 269 | self.keyval[key] = value |