mbligh | 96cf051 | 2008-04-17 15:25:38 +0000 | [diff] [blame^] | 1 | #!/usr/bin/python -u |
mbligh | c251454 | 2008-02-19 15:54:26 +0000 | [diff] [blame] | 2 | |
mbligh | 96cf051 | 2008-04-17 15:25:38 +0000 | [diff] [blame^] | 3 | import os, sys, optparse, fcntl, errno, traceback |
mbligh | bb7b891 | 2006-10-08 03:59:02 +0000 | [diff] [blame] | 4 | |
mbligh | 96cf051 | 2008-04-17 15:25:38 +0000 | [diff] [blame^] | 5 | import common |
| 6 | from autotest_lib.client.common_lib import mail as common_mail |
| 7 | from autotest_lib.tko import db as tko_db, utils, status_lib |
mbligh | 74fc046 | 2007-11-05 20:24:17 +0000 | [diff] [blame] | 8 | |
| 9 | |
mbligh | 96cf051 | 2008-04-17 15:25:38 +0000 | [diff] [blame^] | 10 | def parse_args(): |
| 11 | # build up our options parser and parse sys.argv |
| 12 | parser = optparse.OptionParser() |
| 13 | parser.add_option("-m", help="Send mail for FAILED tests", |
| 14 | dest="mailit", action="store_true") |
| 15 | parser.add_option("-r", help="Reparse the results of a job", |
| 16 | dest="reparse", action="store_true") |
| 17 | parser.add_option("-o", help="Parse a single results directory", |
| 18 | dest="singledir", action="store_true") |
| 19 | parser.add_option("-l", help=("Levels of subdirectories to include " |
| 20 | "in the job name"), |
| 21 | type="int", dest="level", default=1) |
| 22 | parser.add_option("-n", help="No blocking on an existing parse", |
| 23 | dest="noblock", action="store_true") |
| 24 | parser.add_option("-s", help="Database server hostname", |
| 25 | dest="db_host", action="store") |
| 26 | parser.add_option("-u", help="Database username", dest="db_user", |
| 27 | action="store") |
| 28 | parser.add_option("-p", help="Database password", dest="db_pass", |
| 29 | action="store") |
| 30 | parser.add_option("-d", help="Database name", dest="db_name", |
| 31 | action="store") |
| 32 | options, args = parser.parse_args() |
mbligh | 74fc046 | 2007-11-05 20:24:17 +0000 | [diff] [blame] | 33 | |
mbligh | 96cf051 | 2008-04-17 15:25:38 +0000 | [diff] [blame^] | 34 | # we need a results directory |
| 35 | if len(args) == 0: |
| 36 | utils.dprint("ERROR: at least one results directory must " |
| 37 | "be provided") |
| 38 | parser.print_help() |
| 39 | sys.exit(1) |
mbligh | 74fc046 | 2007-11-05 20:24:17 +0000 | [diff] [blame] | 40 | |
mbligh | 96cf051 | 2008-04-17 15:25:38 +0000 | [diff] [blame^] | 41 | # pass the options back |
| 42 | return options, args |
mbligh | 74fc046 | 2007-11-05 20:24:17 +0000 | [diff] [blame] | 43 | |
| 44 | |
mbligh | 96cf051 | 2008-04-17 15:25:38 +0000 | [diff] [blame^] | 45 | def format_failure_message(jobname, kernel, testname, status, reason): |
| 46 | format_string = "%-12s %-20s %-12s %-10s %s" |
| 47 | return format_string % (jobname, kernel, testname, status, reason) |
mbligh | b85e6b0 | 2006-10-08 17:20:56 +0000 | [diff] [blame] | 48 | |
mbligh | bb7b891 | 2006-10-08 03:59:02 +0000 | [diff] [blame] | 49 | |
mbligh | 96cf051 | 2008-04-17 15:25:38 +0000 | [diff] [blame^] | 50 | def mailfailure(jobname, job, message): |
| 51 | message_lines = [""] |
| 52 | message_lines.append("The following tests FAILED for this job") |
| 53 | message_lines.append("http://%s/results/%s" % |
| 54 | (socket.gethostname(), jobname)) |
| 55 | message_lines.append("") |
| 56 | message_lines.append(format_failure_message("Job name", "Kernel", |
| 57 | "Test name", "FAIL/WARN", |
| 58 | "Failure reason")) |
| 59 | message_lines.append(format_failure_message("=" * 8, "=" * 6, "=" * 8, |
| 60 | "=" * 8, "=" * 14)) |
| 61 | message_header = "\n".join(message_lines) |
| 62 | |
| 63 | subject = "AUTOTEST: FAILED tests from job %s" % jobname |
| 64 | common_mail.send("", job.user, "", subject, message_header + message) |
mbligh | 006f230 | 2007-09-13 20:46:46 +0000 | [diff] [blame] | 65 | |
| 66 | |
mbligh | 96cf051 | 2008-04-17 15:25:38 +0000 | [diff] [blame^] | 67 | def parse_one(db, jobname, path, reparse, mail_on_failure): |
| 68 | """ |
| 69 | Parse a single job. Optionally send email on failure. |
| 70 | """ |
| 71 | utils.dprint("\nScanning %s (%s)" % (jobname, path)) |
| 72 | if reparse and db.find_job(jobname): |
| 73 | utils.dprint("! Deleting old copy of job results to " |
| 74 | "reparse it") |
| 75 | db.delete_job(jobname) |
| 76 | if db.find_job(jobname): |
| 77 | utils.dprint("! Job is already parsed, done") |
| 78 | return |
| 79 | |
| 80 | # parse out the job |
| 81 | parser = status_lib.parser(0) |
| 82 | job = parser.make_job(path) |
| 83 | status_log = os.path.join(path, "status.log") |
| 84 | if not os.path.exists(status_log): |
| 85 | status_log = os.path.join(path, "status") |
| 86 | if not os.path.exists(status_log): |
| 87 | utils.dprint("! Unable to parse job, no status file") |
| 88 | return |
| 89 | |
| 90 | # parse the status logs |
| 91 | utils.dprint("+ Parsing dir=%s, jobname=%s" % (path, jobname)) |
| 92 | status_lines = open(status_log).readlines() |
| 93 | parser.start(job) |
| 94 | tests = parser.end(status_lines) |
| 95 | job.tests = tests |
| 96 | |
| 97 | # check for failures |
| 98 | message_lines = [""] |
| 99 | for test in job.tests: |
| 100 | if not test.subdir: |
| 101 | continue |
| 102 | utils.dprint("* testname, status, reason: %s %s %s" |
| 103 | % (test.subdir, test.status, test.reason)) |
| 104 | if test.status in ("FAIL", "WARN"): |
| 105 | message_lines.append(format_failure_message( |
| 106 | jobname, test.kernel.base, test.subdir, |
| 107 | test.status, test.reason)) |
| 108 | message = "\n".join(message_lines) |
| 109 | |
| 110 | # send out a email report of failure |
| 111 | if len(message) > 2 and mail_on_failure: |
| 112 | utils.dprint("Sending email report of failure on %s to %s" |
| 113 | % (jobname, job.user)) |
| 114 | mailfailure(jobname, job, message) |
| 115 | |
| 116 | # write the job into the database |
| 117 | db.insert_job(jobname, job) |
| 118 | db.commit() |
mbligh | 26b992b | 2008-02-19 15:46:21 +0000 | [diff] [blame] | 119 | |
| 120 | |
mbligh | 96cf051 | 2008-04-17 15:25:38 +0000 | [diff] [blame^] | 121 | def parse_path(db, path, level, reparse, mail_on_failure): |
| 122 | machine_list = os.path.join(path, ".machines") |
| 123 | if os.path.exists(machine_list): |
| 124 | # multi-machine job |
| 125 | for m in file(machine_list): |
| 126 | machine = m.rstrip() |
| 127 | if not machine: |
| 128 | continue |
| 129 | jobpath = os.path.join(path, machine) |
| 130 | jobname = "%s/%s" % (os.path.basename(path), machine) |
| 131 | try: |
| 132 | parse_one(db, jobname, jobpath, reparse, |
| 133 | mail_on_failure) |
| 134 | except Exception: |
| 135 | traceback.print_exc() |
| 136 | continue |
| 137 | else: |
| 138 | # single machine job |
| 139 | job_elements = path.split("/")[-level:] |
| 140 | jobname = "/".join(job_elements) |
mbligh | 8e1ab17 | 2007-09-13 17:29:56 +0000 | [diff] [blame] | 141 | try: |
mbligh | 96cf051 | 2008-04-17 15:25:38 +0000 | [diff] [blame^] | 142 | parse_one(db, jobname, path, reparse, mail_on_failure) |
| 143 | except Exception: |
| 144 | traceback.print_exc() |
mbligh | bb7b891 | 2006-10-08 03:59:02 +0000 | [diff] [blame] | 145 | |
| 146 | |
mbligh | 96cf051 | 2008-04-17 15:25:38 +0000 | [diff] [blame^] | 147 | def main(): |
| 148 | options, args = parse_args() |
| 149 | results_dir = os.path.abspath(args[0]) |
| 150 | assert os.path.exists(results_dir) |
| 151 | |
| 152 | # build up the list of job dirs to parse |
| 153 | if options.singledir: |
| 154 | jobs_list = [results_dir] |
| 155 | else: |
| 156 | jobs_list = [os.path.join(results_dir, subdir) |
| 157 | for subdir in os.listdir(results_dir)] |
| 158 | |
| 159 | # build up the database |
| 160 | db = tko_db.db(autocommit=False, host=options.db_host, |
| 161 | user=options.db_user, password=options.db_pass, |
| 162 | database=options.db_name) |
| 163 | |
| 164 | # parse all the jobs |
| 165 | for path in jobs_list: |
| 166 | lockfile = open(os.path.join(path, ".parse.lock"), "w") |
| 167 | flags = fcntl.LOCK_EX |
| 168 | if options.noblock: |
| 169 | flags != fcntl.LOCK_NB |
mbligh | 532cb27 | 2007-11-26 18:54:20 +0000 | [diff] [blame] | 170 | try: |
mbligh | 96cf051 | 2008-04-17 15:25:38 +0000 | [diff] [blame^] | 171 | fcntl.flock(lockfile, flags) |
| 172 | except IOError, e: |
| 173 | # was this because the lock is unavailable? |
| 174 | if e.errno == errno.EWOULDBLOCK: |
| 175 | lockfile.close() |
| 176 | continue |
| 177 | else: |
| 178 | raise # something unexpected happened |
mbligh | 532cb27 | 2007-11-26 18:54:20 +0000 | [diff] [blame] | 179 | try: |
mbligh | 96cf051 | 2008-04-17 15:25:38 +0000 | [diff] [blame^] | 180 | parse_path(db, path, options.level, options.reparse, |
| 181 | options.mailit) |
| 182 | finally: |
| 183 | fcntl.flock(lockfile, fcntl.LOCK_UN) |
| 184 | lockfile.close() |
mbligh | 71d340d | 2008-03-05 15:51:16 +0000 | [diff] [blame] | 185 | |
mbligh | 532cb27 | 2007-11-26 18:54:20 +0000 | [diff] [blame] | 186 | |
mbligh | 96cf051 | 2008-04-17 15:25:38 +0000 | [diff] [blame^] | 187 | if __name__ == "__main__": |
| 188 | main() |