scheduler/monitor_queue

#!/usr/bin/python -u
# monitor_queue <client> <spool_directory> <resultsdir> [<conmux_server>]
import os, time, sys
from subprocess import *
import tempfile

if (len(sys.argv) < 4):
	print "Usage: monitor_queue <client> <spool_directory> <resultsdir> [<conmux_server>]"
	sys.exit(1)
(client, spooldir, resultsdir) = sys.argv[1:4]
if len(sys.argv) == 5:
	console = sys.argv[4]
else:
	console = None
if not os.path.exists(spooldir):
	print "spooldir %s does not exist" % spooldir
	sys.exit(1)
if not os.path.exists(resultsdir):
	print "resultsdir %s does not exist" % resultsdir
	sys.exit(1)


def pick_job(jobs):
	"""Pick the next job to run. Currently we just pick the oldest job
	However, this would be the place to put prioritizations."""
	if not jobs:
		return None
	return sorted(jobs, key=lambda x:os.stat(x).st_mtime, reverse=True)[0]


def __create_autoserv_wrapper(control_path, results):
	"""Create an autoserv file that runs an autotest file at
	control_path on client and outputs the results in results."""
	# Create an autoserv control file to run this autotest control file
	tmpfd, tmpname = tempfile.mkstemp()
	tmp = os.fdopen(tmpfd, 'w')
	if console:
		print >> tmp, "host = hosts.ConmuxSSHHost('%s', \
		server='%s')" % (client, console)
	else:
		print >> tmp, "host = hosts.ConmuxSSHHost('%s')" % client
	print >> tmp, "at = autotest.Autotest()"
	# TODO(stutsman): at.install(host) here?
	print >> tmp, "at.run('%s', '%s', host)" % (control_path, results)
	tmp.close()
	return tmpname
	

def run_job(control):
	"""Runs a control file from the spooldir.
	Args:
	  control: A path to a control file.  It is assumed to be an
	    Autotest control file in which case it will automatically
	    be wrapped with autoserv control commands and run with
	    autoserv.  If the file name ends with .srv the wrapping
	    procedure will be skipped and the autoserv file will be
	    run directly.

	Return:
	  The return code from the autoserv process.
	"""
	# Make sure all the output directories are all setup
	results = os.path.join(resultsdir, control)
	if os.path.exists(results):
		print "Resultsdir %s already present, " % results,
		results = "%s.%d" % (results, int(time.time()))
		print "changing to %s" % results
	os.mkdir(results)
	debug = os.path.join(results, 'debug')
	os.mkdir(debug)

	# If this is an autoserv file then don't create the wrapper control
	is_autoserv_ctl = control.endswith('.srv')
	control_path = os.path.abspath(os.path.join(spooldir, control))
	# Otherwise create a tmp autoserv file just to launch the AT ctl file
	if not is_autoserv_ctl:
		control_path = __create_autoserv_wrapper(control_path, results)

	# Now run the job
	exedir = os.path.abspath(os.path.dirname(sys.argv[0]))
	autoserv_exe = os.path.abspath(os.path.join(exedir,
						    '..',
						    'server',
						    'autoserv'))
	autoserv_cmd = ' '.join([autoserv_exe, control_path])
	
	autoserv_log = open(os.path.join(debug, 'server.log'), 'w');
	p = Popen(autoserv_cmd, shell=True, stdout=autoserv_log, stderr=STDOUT)
	(pid, ret) = os.waitpid(p.pid, 0)
	autoserv_log.close()
	
	# If this was a tempfile then clean it up
	if not is_autoserv_ctl:
		os.unlink(control_path)
	print "Completed job: %s (%d) " % (control, ret)
	
	return ret
	

dir = os.path.abspath(os.path.dirname(sys.argv[0]))
runjob = os.path.join(dir, 'runjob')
os.chdir(spooldir)
print "monitoring spool directory: " + spooldir
while True:
	jobs = [j for j in os.listdir(spooldir) if not j.startswith('.')]
	next_job = pick_job(jobs)
	if not next_job:
		time.sleep(10)
		continue
	ret = run_job(next_job)
	if ret == 1:          # Ooops. the machine is stuffed.
		print "WARNING: client %s failed verify_machine" % client
		sleep(600)
		continue
	os.remove(next_job)