tko/parse

#!/usr/bin/python -u
import os, re, parse, frontend, db, sys, socket, fcntl
from optparse import OptionParser
from traceback import format_exception

def format_error():
	t, o, tb = sys.exc_info()
	trace = format_exception(t, o, tb)
	# Clear the backtrace to prevent a circular reference
	# in the heap -- as per tutorial
	tb = ''

	return ''.join(trace)

parser = OptionParser()
parser.add_option('-m', help='Send mail for FAILED tests', dest='mailit',
							action='store_true')
parser.add_option('-r', help='Reparse the results of a job', dest='reparse',
							action='store_true')
parser.add_option('-o', help='one: parse a single results directory',
					dest='singledir', action='store_true')
parser.add_option('-l', help='levels of subdirectories to include in job name',
					type='int', dest='level', default=1)
parser.add_option('-n', help='no blocking on an existing parse',
		  dest='noblock', action='store_true')
parser.add_option('-s', help='Database server hostname',
					dest='db_host', action='store')
parser.add_option('-u', help='Database username',
					dest='db_user', action='store')
parser.add_option('-p', help='Database password',
					dest='db_pass', action='store')
parser.add_option('-d', help='Database name',
					dest='db_name', action='store')
(options, args) = parser.parse_args()

if len(args) == 0:
	print "ERROR: You need to at least provide a directory to parse\n"
	parser.print_help()
	sys.exit(1)

dir = os.path.abspath(args[0])
assert os.path.exists(dir)

if options.singledir:
	jobs_list = [dir]
else:
	jobs_list = [os.path.join(dir, subdir) for subdir in os.listdir(dir)]

debug = True

failcc = ""
# The user to notify on job failures - TOOD, pull into the config file
notify_user = None
# do commits transactionally
db = db.db(autocommit=False, host=options.db_host,
           user=options.db_user, password=options.db_pass,
           database=options.db_name)


def mailfailure(jobname, job, mesgtxt):
	# XXX: Need to insert URL here too (frontend.test.url?)
	link = "http://" + socket.gethostname() + "/results/" + jobname

	# This looks pretty good on fixed-width-font email reader.
	message_header = "\n%s\n%s\n\n%-12s %-20s %-12s %-10s %s\n" % \
				("The following tests FAILED for this job:",
				link, "Job name", "Kernel", "Test name",
				"FAIL/WARN", "Failure Reason")
	message_header += "%-12s %-20s %-12s %-10s %s\n" % \
					("========", "======", "=========",
					 "=========", "==============")

	subject = "AUTOTEST: FAILED tests from " + " job " + jobname
	parse.mail(notify_user, job.user, failcc, subject,
						message_header + mesgtxt)


def dprint(string):
	if debug:
		print string


def do_parse(jobname, path):
	"""
	Parse a single job. Optionally send email on failure, etc.
	"""
	dprint('\nScanning %s (%s)' % (jobname, path))
	if options.reparse and db.find_job(jobname):
		dprint('! Deleting old copy of job results, to reparse it')
		db.delete_job(jobname)
	if db.find_job(jobname):		# Job has already been parsed
		dprint('! Already processed')
		return
	job = parse.job(path)
	if not job:
		dprint('! Failed to parse job (no status file?)')
		return
	if not job.kernel:
		dprint('! Failed to find kernel for job')
		return
	print '+ Parsing ' + path
	print '* jobname, kernel version: %s %s' % (jobname, job.kernel.base)
	mesgtxt = "\n"
	for test in job.tests:
		if not test.subdir:
			continue
		print "* testname, status, reason: %s %s %s" % \
					(test.subdir, test.status, test.reason)
		if re.match(r'(FAIL|WARN)',test.status):
			mesgtxt += "%-12s %-20s %-12s %-10s %s" % \
					(jobname, job.kernel.base, test.subdir,
					 test.status, test.reason)

	if len(mesgtxt) > 2 and options.mailit:
		print "Sending email report of FAILURES on %s to %s" % \
							(jobname, job.user)
		mailfailure(jobname, job, mesgtxt)
	db.insert_job(jobname, job)
	print "COMMITING"
	db.commit()


def parse_path(path):
	job_elements = path.split('/')[-options.level:]
					# last 'level' elements of path
	jobname = '/'.join(job_elements)
	machine_list = os.path.join(path, '.machines')
	if os.path.exists(machine_list):
		# This is a multi-machine job
		for m in open(machine_list):
			machine = m.rstrip()
			if not machine:
				continue
			jobpath = os.path.join(path, machine)
			jobname = os.path.join(os.path.basename(path), machine)
			try:
				do_parse(jobname, jobpath)
			except:
				print format_error()
				continue
	else:
		# This is a single-machine job
		try:
			do_parse(jobname, path)
		except:
			print format_error()


for path in jobs_list:
	lockfile = open(os.path.join(path, ".parse.lock"), "w")
	try:
		flags = fcntl.LOCK_EX
		if options.noblock:
			flags |= fcntl.LOCK_NB
		fcntl.flock(lockfile, flags)
	except IOError:
		# only happens when non-blocking if the lock is already held
		lockfile.close()
		continue   # a parse is already running here, skip it
	try:
		parse_path(path)
	finally:
		fcntl.flock(lockfile, fcntl.LOCK_UN)
		lockfile.close()