blob: a82ab0abd9792881f3fed69556d6da9cdf73ac59 [file] [log] [blame]
mbligh96cf0512008-04-17 15:25:38 +00001#!/usr/bin/python -u
mblighc2514542008-02-19 15:54:26 +00002
mblighb33e53e2008-06-17 19:41:26 +00003import os, sys, optparse, fcntl, errno, traceback, socket
mblighbb7b8912006-10-08 03:59:02 +00004
mbligh96cf0512008-04-17 15:25:38 +00005import common
jadmanskidb4f9b52008-12-03 22:52:53 +00006from autotest_lib.client.common_lib import mail, pidfile
7from autotest_lib.tko import db as tko_db, utils as tko_utils, status_lib, models
mbligh9e936402009-05-13 20:42:17 +00008from autotest_lib.client.common_lib import utils
mbligh74fc0462007-11-05 20:24:17 +00009
10
mbligh96cf0512008-04-17 15:25:38 +000011def parse_args():
jadmanski0afbb632008-06-06 21:10:57 +000012 # build up our options parser and parse sys.argv
13 parser = optparse.OptionParser()
14 parser.add_option("-m", help="Send mail for FAILED tests",
15 dest="mailit", action="store_true")
16 parser.add_option("-r", help="Reparse the results of a job",
17 dest="reparse", action="store_true")
18 parser.add_option("-o", help="Parse a single results directory",
19 dest="singledir", action="store_true")
20 parser.add_option("-l", help=("Levels of subdirectories to include "
21 "in the job name"),
22 type="int", dest="level", default=1)
23 parser.add_option("-n", help="No blocking on an existing parse",
24 dest="noblock", action="store_true")
25 parser.add_option("-s", help="Database server hostname",
26 dest="db_host", action="store")
27 parser.add_option("-u", help="Database username", dest="db_user",
28 action="store")
29 parser.add_option("-p", help="Database password", dest="db_pass",
30 action="store")
31 parser.add_option("-d", help="Database name", dest="db_name",
32 action="store")
mbligh9e936402009-05-13 20:42:17 +000033 parser.add_option("-P", help="Run site post-processing",
34 dest="site_do_post", action="store_true", default=False)
jadmanskid5ab8c52008-12-03 16:27:07 +000035 parser.add_option("--write-pidfile",
36 help="write pidfile (.parser_execute)",
37 dest="write_pidfile", action="store_true",
38 default=False)
jadmanski0afbb632008-06-06 21:10:57 +000039 options, args = parser.parse_args()
mbligh74fc0462007-11-05 20:24:17 +000040
jadmanski0afbb632008-06-06 21:10:57 +000041 # we need a results directory
42 if len(args) == 0:
43 tko_utils.dprint("ERROR: at least one results directory must "
44 "be provided")
45 parser.print_help()
46 sys.exit(1)
mbligh74fc0462007-11-05 20:24:17 +000047
jadmanski0afbb632008-06-06 21:10:57 +000048 # pass the options back
49 return options, args
mbligh74fc0462007-11-05 20:24:17 +000050
51
mbligh96cf0512008-04-17 15:25:38 +000052def format_failure_message(jobname, kernel, testname, status, reason):
jadmanski0afbb632008-06-06 21:10:57 +000053 format_string = "%-12s %-20s %-12s %-10s %s"
54 return format_string % (jobname, kernel, testname, status, reason)
mblighb85e6b02006-10-08 17:20:56 +000055
mblighbb7b8912006-10-08 03:59:02 +000056
mbligh96cf0512008-04-17 15:25:38 +000057def mailfailure(jobname, job, message):
jadmanski0afbb632008-06-06 21:10:57 +000058 message_lines = [""]
59 message_lines.append("The following tests FAILED for this job")
60 message_lines.append("http://%s/results/%s" %
61 (socket.gethostname(), jobname))
62 message_lines.append("")
63 message_lines.append(format_failure_message("Job name", "Kernel",
64 "Test name", "FAIL/WARN",
65 "Failure reason"))
66 message_lines.append(format_failure_message("=" * 8, "=" * 6, "=" * 8,
67 "=" * 8, "=" * 14))
68 message_header = "\n".join(message_lines)
mbligh96cf0512008-04-17 15:25:38 +000069
jadmanski0afbb632008-06-06 21:10:57 +000070 subject = "AUTOTEST: FAILED tests from job %s" % jobname
71 mail.send("", job.user, "", subject, message_header + message)
mbligh006f2302007-09-13 20:46:46 +000072
73
mbligh96cf0512008-04-17 15:25:38 +000074def parse_one(db, jobname, path, reparse, mail_on_failure):
jadmanski0afbb632008-06-06 21:10:57 +000075 """
76 Parse a single job. Optionally send email on failure.
77 """
78 tko_utils.dprint("\nScanning %s (%s)" % (jobname, path))
jadmanski9b6babf2009-04-21 17:57:40 +000079 old_job_idx = db.find_job(jobname)
showard0fec8a02009-12-04 01:19:54 +000080 # old tests is a dict from tuple (test_name, subdir) to test_idx
81 old_tests = {}
82 if old_job_idx is not None:
83 if not reparse:
84 tko_utils.dprint("! Job is already parsed, done")
85 return
86
87 raw_old_tests = db.select("test_idx,subdir,test", "tests",
88 {"job_idx": old_job_idx})
89 if raw_old_tests:
90 old_tests = dict(((test, subdir), test_idx)
91 for test_idx, subdir, test in raw_old_tests)
mbligh96cf0512008-04-17 15:25:38 +000092
jadmanski0afbb632008-06-06 21:10:57 +000093 # look up the status version
jadmanskidb4f9b52008-12-03 22:52:53 +000094 job_keyval = models.job.read_keyval(path)
95 status_version = job_keyval.get("status_version", 0)
jadmanski6e8bf752008-05-14 00:17:48 +000096
jadmanski0afbb632008-06-06 21:10:57 +000097 # parse out the job
98 parser = status_lib.parser(status_version)
99 job = parser.make_job(path)
100 status_log = os.path.join(path, "status.log")
101 if not os.path.exists(status_log):
102 status_log = os.path.join(path, "status")
103 if not os.path.exists(status_log):
104 tko_utils.dprint("! Unable to parse job, no status file")
105 return
mbligh96cf0512008-04-17 15:25:38 +0000106
jadmanski0afbb632008-06-06 21:10:57 +0000107 # parse the status logs
108 tko_utils.dprint("+ Parsing dir=%s, jobname=%s" % (path, jobname))
109 status_lines = open(status_log).readlines()
110 parser.start(job)
111 tests = parser.end(status_lines)
jadmanski9b6babf2009-04-21 17:57:40 +0000112
113 # parser.end can return the same object multiple times, so filter out dups
114 job.tests = []
115 already_added = set()
116 for test in tests:
117 if test not in already_added:
118 already_added.add(test)
119 job.tests.append(test)
120
showard0fec8a02009-12-04 01:19:54 +0000121 # try and port test_idx over from the old tests, but if old tests stop
jadmanski9b6babf2009-04-21 17:57:40 +0000122 # matching up with new ones just give up
showard0fec8a02009-12-04 01:19:54 +0000123 if reparse and old_job_idx is not None:
124 job.index = old_job_idx
125 for test in job.tests:
126 test_idx = old_tests.pop((test.testname, test.subdir), None)
127 if test_idx is not None:
128 test.test_idx = test_idx
129 else:
130 tko_utils.dprint("! Reparse returned new test "
131 "testname=%r subdir=%r" %
132 (test.testname, test.subdir))
133 for test_idx in old_tests.itervalues():
134 where = {'test_idx' : test_idx}
135 db.delete('iteration_result', where)
136 db.delete('iteration_attributes', where)
137 db.delete('test_attributes', where)
138 db.delete('test_labels_tests', {'test_id': test_idx})
139 db.delete('tests', where)
mbligh96cf0512008-04-17 15:25:38 +0000140
jadmanski0afbb632008-06-06 21:10:57 +0000141 # check for failures
142 message_lines = [""]
143 for test in job.tests:
144 if not test.subdir:
145 continue
146 tko_utils.dprint("* testname, status, reason: %s %s %s"
147 % (test.subdir, test.status, test.reason))
148 if test.status in ("FAIL", "WARN"):
149 message_lines.append(format_failure_message(
150 jobname, test.kernel.base, test.subdir,
151 test.status, test.reason))
152 message = "\n".join(message_lines)
mbligh96cf0512008-04-17 15:25:38 +0000153
jadmanski0afbb632008-06-06 21:10:57 +0000154 # send out a email report of failure
155 if len(message) > 2 and mail_on_failure:
156 tko_utils.dprint("Sending email report of failure on %s to %s"
157 % (jobname, job.user))
158 mailfailure(jobname, job, message)
mbligh96cf0512008-04-17 15:25:38 +0000159
jadmanski0afbb632008-06-06 21:10:57 +0000160 # write the job into the database
161 db.insert_job(jobname, job)
162 db.commit()
mbligh26b992b2008-02-19 15:46:21 +0000163
164
jadmanski8e9c2572008-11-11 00:29:02 +0000165def _get_job_subdirs(path):
166 """
167 Returns a list of job subdirectories at path. Returns None if the test
168 is itself a job directory. Does not recurse into the subdirs.
169 """
170 # if there's a .machines file, use it to get the subdirs
jadmanski0afbb632008-06-06 21:10:57 +0000171 machine_list = os.path.join(path, ".machines")
172 if os.path.exists(machine_list):
jadmanski42fbd072009-01-30 15:07:05 +0000173 subdirs = set(line.strip() for line in file(machine_list))
174 existing_subdirs = set(subdir for subdir in subdirs
175 if os.path.exists(os.path.join(path, subdir)))
176 if len(existing_subdirs) != 0:
177 return existing_subdirs
jadmanski8e9c2572008-11-11 00:29:02 +0000178
179 # if this dir contains ONLY subdirectories, return them
180 contents = set(os.listdir(path))
181 contents.discard(".parse.lock")
182 subdirs = set(sub for sub in contents if
183 os.path.isdir(os.path.join(path, sub)))
184 if len(contents) == len(subdirs) != 0:
185 return subdirs
186
187 # this is a job directory, or something else we don't understand
188 return None
189
190
mbligha48eeb22009-03-11 16:44:43 +0000191def parse_leaf_path(db, path, level, reparse, mail_on_failure):
192 job_elements = path.split("/")[-level:]
193 jobname = "/".join(job_elements)
194 try:
195 db.run_with_retry(parse_one, db, jobname, path, reparse,
196 mail_on_failure)
197 except Exception:
198 traceback.print_exc()
199
200
jadmanski8e9c2572008-11-11 00:29:02 +0000201def parse_path(db, path, level, reparse, mail_on_failure):
202 job_subdirs = _get_job_subdirs(path)
203 if job_subdirs is not None:
mbligha48eeb22009-03-11 16:44:43 +0000204 # parse status.log in current directory, if it exists. multi-machine
205 # synchronous server side tests record output in this directory. without
206 # this check, we do not parse these results.
207 if os.path.exists(os.path.join(path, 'status.log')):
208 parse_leaf_path(db, path, level, reparse, mail_on_failure)
jadmanski0afbb632008-06-06 21:10:57 +0000209 # multi-machine job
jadmanski8e9c2572008-11-11 00:29:02 +0000210 for subdir in job_subdirs:
211 jobpath = os.path.join(path, subdir)
212 parse_path(db, jobpath, level + 1, reparse, mail_on_failure)
jadmanski0afbb632008-06-06 21:10:57 +0000213 else:
214 # single machine job
mbligha48eeb22009-03-11 16:44:43 +0000215 parse_leaf_path(db, path, level, reparse, mail_on_failure)
mblighbb7b8912006-10-08 03:59:02 +0000216
217
mbligh9e936402009-05-13 20:42:17 +0000218def _site_post_parse_job_dummy():
219 return {}
220
221
mbligh96cf0512008-04-17 15:25:38 +0000222def main():
jadmanski0afbb632008-06-06 21:10:57 +0000223 options, args = parse_args()
224 results_dir = os.path.abspath(args[0])
225 assert os.path.exists(results_dir)
mbligh96cf0512008-04-17 15:25:38 +0000226
jadmanskid5ab8c52008-12-03 16:27:07 +0000227 pid_file_manager = pidfile.PidFileManager("parser", results_dir)
mbligh96cf0512008-04-17 15:25:38 +0000228
jadmanskid5ab8c52008-12-03 16:27:07 +0000229 if options.write_pidfile:
230 pid_file_manager.open_file()
mbligh96cf0512008-04-17 15:25:38 +0000231
mbligh9e936402009-05-13 20:42:17 +0000232 site_post_parse_job = utils.import_site_function(__file__,
233 "autotest_lib.tko.site_parse", "site_post_parse_job",
234 _site_post_parse_job_dummy)
235
jadmanskid5ab8c52008-12-03 16:27:07 +0000236 try:
237 # build up the list of job dirs to parse
238 if options.singledir:
239 jobs_list = [results_dir]
240 else:
241 jobs_list = [os.path.join(results_dir, subdir)
242 for subdir in os.listdir(results_dir)]
243
244 # build up the database
245 db = tko_db.db(autocommit=False, host=options.db_host,
246 user=options.db_user, password=options.db_pass,
247 database=options.db_name)
248
249 # parse all the jobs
250 for path in jobs_list:
251 lockfile = open(os.path.join(path, ".parse.lock"), "w")
252 flags = fcntl.LOCK_EX
253 if options.noblock:
mblighdb18b0e2009-01-30 00:34:32 +0000254 flags |= fcntl.LOCK_NB
jadmanskid5ab8c52008-12-03 16:27:07 +0000255 try:
256 fcntl.flock(lockfile, flags)
257 except IOError, e:
mblighdb18b0e2009-01-30 00:34:32 +0000258 # lock is not available and nonblock has been requested
jadmanskid5ab8c52008-12-03 16:27:07 +0000259 if e.errno == errno.EWOULDBLOCK:
260 lockfile.close()
261 continue
262 else:
263 raise # something unexpected happened
264 try:
265 parse_path(db, path, options.level, options.reparse,
266 options.mailit)
mbligh9e936402009-05-13 20:42:17 +0000267
jadmanskid5ab8c52008-12-03 16:27:07 +0000268 finally:
269 fcntl.flock(lockfile, fcntl.LOCK_UN)
jadmanski0afbb632008-06-06 21:10:57 +0000270 lockfile.close()
mblighe97e0e62009-05-21 01:41:58 +0000271
272 if options.site_do_post is True:
273 site_post_parse_job(results_dir)
274
jadmanskid5ab8c52008-12-03 16:27:07 +0000275 except:
276 pid_file_manager.close_file(1)
277 raise
278 else:
279 pid_file_manager.close_file(0)
mbligh71d340d2008-03-05 15:51:16 +0000280
mbligh532cb272007-11-26 18:54:20 +0000281
mbligh96cf0512008-04-17 15:25:38 +0000282if __name__ == "__main__":
jadmanski0afbb632008-06-06 21:10:57 +0000283 main()