tko/parse

#!/usr/bin/python
import os, re, parse, frontend, db, sys, socket, getopt
from traceback import format_exception

usage = """\
usage: parse
	[-m]                           # Send mail for FAILED tests
	[-r]                           # Reparse the results of a job
	[-o directory]                 # Specify results directory directly
	<top level results directory>  # Specify top level results directory
"""

def format_error():
	t, o, tb = sys.exc_info()
	trace = format_exception(t, o, tb)
	# Clear the backtrace to prevent a circular reference
	# in the heap -- as per tutorial
	tb = ''

	return ''.join(trace)


try:
	opts, args = getopt.getopt(sys.argv[1:], "hrmo:l:", ["help"])
except getopt.GetoptError:
	# print help information and exit:
	print usage,
	sys.exit(2)

if len(sys.argv) < 2:
	print usage
	sys.exit(2)

singledir = None
mailit = False
reparse = False
level = 1
for name, value in opts:
	if name in ("-h", "--help"):
		usage()
		sys.exit()
	if name == "-m":
		mailit = True
	if name == "-r":
		reparse = True
	if name in ("-o"):
		singledir = value
	if name in ("-l"):
		level = int(value)

if singledir:
	dir = os.path.abspath(singledir)
	assert os.path.exists(dir)
	jobs_list = [dir]
else:
	topdir = os.path.abspath(args[0])
	assert os.path.exists(topdir)
	jobs_list = [os.path.join(topdir, dir) for dir in os.listdir(topdir)]

debug = True

failcc = ""
# The user to notify on job failures - TOOD, pull into the config file
notify_user = None

db = db.db(autocommit=False)  # do commits transactionally


def mailfailure(jobname, job, mesgtxt):
	# XXX: Need to insert URL here too (frontend.test.url?)
	link = "http://" + socket.gethostname() + "/results/" + jobname

	# This looks pretty good on fixed-width-font email reader.
	message_header = "\n%s\n%s\n\n%-12s %-20s %-12s %-10s %s\n" % \
				("The following tests FAILED for this job:",
				link, "Job name", "Kernel", "Test name",
				"FAIL/WARN", "Failure Reason")
	message_header += "%-12s %-20s %-12s %-10s %s\n" % \
					("========", "======", "=========",
					 "=========", "==============")

	subject = "AUTOTEST: FAILED tests from " + " job " + jobname
	parse.mail(notify_user, job.user, failcc, subject,
						message_header + mesgtxt)


def dprint(string):
	if debug:
		print string


def do_parse(jobname, path):
	"""
	Parse a single job. Optionally send email on failure, etc.
	"""
	dprint('\nScanning %s (%s)' % (jobname, path))
	if reparse and db.find_job(jobname):
		dprint('! Deleting old copy of job results, to reparse it')
		db.delete_job(jobname)
	if db.find_job(jobname):		# Job has already been parsed
		dprint('! Already processed')
		return
	job = parse.job(path)
	if not job:
		dprint('! Failed to parse job (no status file?)')
		return
	if not job.kernel:
		dprint('! Failed to find kernel for job')
		return
	print '+ Parsing ' + path
	print '* jobname, kernel version: %s %s' % (jobname, job.kernel.base)
	mesgtxt = "\n"
	for test in job.tests:
		if not test.subdir:
			continue
		print "* testname, status, reason: %s %s %s" % \
					(test.subdir, test.status, test.reason)
		if re.match(r'(FAIL|WARN)',test.status):
			mesgtxt += "%-12s %-20s %-12s %-10s %s" % \
					(jobname, job.kernel.base, test.subdir,
					 test.status, test.reason)

	if len(mesgtxt) > 2 and mailit:
		print "Sending email report of FAILURES on %s to %s" % \
							(jobname, job.user)
		mailfailure(jobname, job, mesgtxt)
	db.insert_job(jobname, job)
	print "COMMITING"
	db.commit()



for path in jobs_list:
	job_elements = path.split('/')[-level:] # last 'level' elements of path
	jobname = '/'.join(job_elements)
	machine_list = os.path.join(path, '.machines')
	if os.path.exists(machine_list):
		# This is a multi-machine job
		for m in open(machine_list, 'r').readlines():
			machine = m.rstrip()
			if not machine:
				continue
			jobpath = os.path.join(path, machine)
			jobname = os.path.join(os.path.basename(path), machine)
			try:
				do_parse(jobname, jobpath)
			except:
				print format_error()
				continue
	else:
		# This is a single-machine job
		try:
			do_parse(jobname, path)
		except:
			print format_error()
			continue


# Generate vertical text pngs for pretty display in tables.
#
# rows = db.select('distinct hostname', 'machines', {})
# machines = [row[0] for row in rows]
# 
# for machine in machines:
# 	dir = os.path.dirname(os.path.abspath(sys.argv[0]))
# 	vertical_text = os.path.join(dir, 'vertical_text.py')
# 	os.system(vertical_text + ' ' + machine)