client/bin/job.py

"""The main job wrapper

This is the core infrastructure.
"""

__author__ = """Copyright Andy Whitcroft, Martin J. Bligh 2006"""

# standard stuff
import os, sys, re, pickle, shutil, time, traceback, types, copy

# autotest stuff
from autotest_lib.client.bin import autotest_utils
from autotest_lib.client.common_lib import error, barrier, logging

import parallel, kernel, xen, test, profilers, filesystem, fd_stack, boottool
import harness, config, sysinfo, cpuset



JOB_PREAMBLE = """
from common.error import *
from autotest_utils import *
"""


class StepError(error.AutotestError):
	pass


class base_job:
	"""The actual job against which we do everything.

	Properties:
		autodir
			The top level autotest directory (/usr/local/autotest).
			Comes from os.environ['AUTODIR'].
		bindir
			<autodir>/bin/
		libdir
			<autodir>/lib/
		testdir
			<autodir>/tests/
		site_testdir
			<autodir>/site_tests/
		profdir
			<autodir>/profilers/
		tmpdir
			<autodir>/tmp/
		resultdir
			<autodir>/results/<jobtag>
		stdout
			fd_stack object for stdout
		stderr
			fd_stack object for stderr
		profilers
			the profilers object for this job
		harness
			the server harness object for this job
		config
			the job configuration for this job
	"""

	DEFAULT_LOG_FILENAME = "status"

	def __init__(self, control, jobtag, cont, harness_type=None,
			use_external_logging = False):
		"""
			control
				The control file (pathname of)
			jobtag
				The job tag string (eg "default")
			cont
				If this is the continuation of this job
			harness_type
				An alternative server harness
		"""
		self.autodir = os.environ['AUTODIR']
		self.bindir = os.path.join(self.autodir, 'bin')
		self.libdir = os.path.join(self.autodir, 'lib')
		self.testdir = os.path.join(self.autodir, 'tests')
		self.site_testdir = os.path.join(self.autodir, 'site_tests')
		self.profdir = os.path.join(self.autodir, 'profilers')
		self.tmpdir = os.path.join(self.autodir, 'tmp')
		self.resultdir = os.path.join(self.autodir, 'results', jobtag)
		self.sysinfodir = os.path.join(self.resultdir, 'sysinfo')
		self.control = os.path.abspath(control)
		self.state_file = self.control + '.state'
		self.__load_state()

		if not cont:
			if os.path.exists(self.tmpdir):
				cmd = ('umount -f %s > /dev/null 2> /dev/null'
					 	% (self.tmpdir))
				autotest_utils.system(cmd, ignore_status=True)
				autotest_utils.system('rm -rf ' + self.tmpdir)
			os.mkdir(self.tmpdir)

			results = os.path.join(self.autodir, 'results')
			if not os.path.exists(results):
				os.mkdir(results)
				
			download = os.path.join(self.testdir, 'download')
			if os.path.exists(download):
				autotest_utils.system('rm -rf ' + download)
			os.mkdir(download)
				
			if os.path.exists(self.resultdir):
				autotest_utils.system('rm -rf ' 
							+ self.resultdir)
			os.mkdir(self.resultdir)
			os.mkdir(self.sysinfodir)

			os.mkdir(os.path.join(self.resultdir, 'debug'))
			os.mkdir(os.path.join(self.resultdir, 'analysis'))

			shutil.copyfile(self.control,
					os.path.join(self.resultdir, 'control'))


		self.control = control
		self.jobtag = jobtag
		self.log_filename = self.DEFAULT_LOG_FILENAME
		self.container = None

		self.stdout = fd_stack.fd_stack(1, sys.stdout)
		self.stderr = fd_stack.fd_stack(2, sys.stderr)

		self._init_group_level()

		self.config = config.config(self)

		self.harness = harness.select(harness_type, self)

		self.profilers = profilers.profilers(self)

		try:
			tool = self.config_get('boottool.executable')
			self.bootloader = boottool.boottool(tool)
		except:
			pass

		sysinfo.log_per_reboot_data(self.sysinfodir)

		if not cont:
			self.record('START', None, None)
			self._increment_group_level()

		self.harness.run_start()
		
		if use_external_logging:
			self.enable_external_logging()

		# load the max disk usage rate - default to no monitoring
		self.max_disk_usage_rate = self.get_state('__monitor_disk',
							  default=0.0)


	def monitor_disk_usage(self, max_rate):
		"""\
		Signal that the job should monitor disk space usage on /
		and generate a warning if a test uses up disk space at a
		rate exceeding 'max_rate'.

		Parameters:
		     max_rate - the maximium allowed rate of disk consumption
		                during a test, in MB/hour, or 0 to indicate
				no limit.
		"""
		self.set_state('__monitor_disk', max_rate)
		self.max_disk_usage_rate = max_rate


	def relative_path(self, path):
		"""\
		Return a patch relative to the job results directory
		"""
		head = len(self.resultdir) + 1     # remove the / inbetween
		return path[head:]


	def control_get(self):
		return self.control


	def control_set(self, control):
		self.control = os.path.abspath(control)


	def harness_select(self, which):
		self.harness = harness.select(which, self)


	def config_set(self, name, value):
		self.config.set(name, value)


	def config_get(self, name):
		return self.config.get(name)

	def setup_dirs(self, results_dir, tmp_dir):
		if not tmp_dir:
			tmp_dir = os.path.join(self.tmpdir, 'build')
		if not os.path.exists(tmp_dir):
			os.mkdir(tmp_dir)
		if not os.path.isdir(tmp_dir):
			e_msg = "Temp dir (%s) is not a dir - args backwards?" % self.tmpdir
			raise ValueError(e_msg)

		# We label the first build "build" and then subsequent ones 
		# as "build.2", "build.3", etc. Whilst this is a little bit 
		# inconsistent, 99.9% of jobs will only have one build 
		# (that's not done as kernbench, sparse, or buildtest),
		# so it works out much cleaner. One of life's comprimises.
		if not results_dir:
			results_dir = os.path.join(self.resultdir, 'build')
			i = 2
			while os.path.exists(results_dir):
				results_dir = os.path.join(self.resultdir, 'build.%d' % i)
				i += 1
		if not os.path.exists(results_dir):
			os.mkdir(results_dir)

		return (results_dir, tmp_dir)


	def xen(self, base_tree, results_dir = '', tmp_dir = '', leave = False, \
				kjob = None ):
		"""Summon a xen object"""
		(results_dir, tmp_dir) = self.setup_dirs(results_dir, tmp_dir)
		build_dir = 'xen'
		return xen.xen(self, base_tree, results_dir, tmp_dir, build_dir, leave, kjob)


	def kernel(self, base_tree, results_dir = '', tmp_dir = '', leave = False):
		"""Summon a kernel object"""
		(results_dir, tmp_dir) = self.setup_dirs(results_dir, tmp_dir)
		build_dir = 'linux'
		return kernel.auto_kernel(self, base_tree, results_dir,
					  tmp_dir, build_dir, leave)


	def barrier(self, *args, **kwds):
		"""Create a barrier object"""
		return barrier.barrier(*args, **kwds)


	def setup_dep(self, deps): 
		"""Set up the dependencies for this test.
		
		deps is a list of libraries required for this test.
		"""
		for dep in deps: 
			try: 
				os.chdir(os.path.join(self.autodir, 'deps', dep))
				autotest_utils.system('./' + dep + '.py')
			except: 
				err = "setting up dependency " + dep + "\n"
				raise error.UnhandledError(err)


	def __runtest(self, url, tag, args, dargs):
		try:
			l = lambda : test.runtest(self, url, tag, args, dargs)
			pid = parallel.fork_start(self.resultdir, l)
			parallel.fork_waitfor(self.resultdir, pid)
		except error.AutotestError:
			raise
		except:
			raise error.UnhandledError('running test ' + \
				self.__class__.__name__ + "\n")


	def run_test(self, url, *args, **dargs):
		"""Summon a test object and run it.
		
		tag
			tag to add to testname
		url
			url of the test to run
		"""

		if not url:
			raise TypeError("Test name is invalid. "
			                "Switched arguments?")
		(group, testname) = test.testname(url)
		tag = dargs.pop('tag', None)
		container = dargs.pop('container', None)
		subdir = testname
		if tag:
			subdir += '.' + tag

		if container:
			cname = container.get('name', None)
			if not cname:   # get old name
				cname = container.get('container_name', None)
			mbytes = container.get('mbytes', None)
			if not mbytes:  # get old name
				mbytes = container.get('mem', None) 
			cpus  = container.get('cpus', None)
			if not cpus:    # get old name
				cpus  = container.get('cpu', None)
			root  = container.get('root', None)
			self.new_container(mbytes=mbytes, cpus=cpus, 
					root=root, name=cname)
			# We are running in a container now...

		def log_warning(reason):
			self.record("WARN", subdir, testname, reason)
		@disk_usage_monitor.watch(log_warning, "/",
					  self.max_disk_usage_rate)
		def group_func():
			try:
				self.__runtest(url, tag, args, dargs)
			except error.TestNAError, detail:
				self.record('TEST_NA', subdir, testname,
					    str(detail))
				raise
			except Exception, detail:
				self.record('FAIL', subdir, testname,
					    str(detail))
				raise
			else:
				self.record('GOOD', subdir, testname,
					    'completed successfully')

		result, exc_info = self.__rungroup(subdir, group_func)
		if container:
			self.release_container()
		if exc_info and isinstance(exc_info[1], error.TestError):
			return False
		elif exc_info:
			raise exc_info[0], exc_info[1], exc_info[2]
		else:
			return True


	def __rungroup(self, name, function, *args, **dargs):
		"""\
		name:
		        name of the group
		function:
			subroutine to run
		*args:
			arguments for the function

		Returns a 2-tuple (result, exc_info) where result
		is the return value of function, and exc_info is
		the sys.exc_info() of the exception thrown by the
		function (which may be None).
		"""

		result, exc_info = None, None
		try:
			self.record('START', None, name)
			self._increment_group_level()
			result = function(*args, **dargs)
			self._decrement_group_level()
			self.record('END GOOD', None, name)
		except error.TestNAError, e:
			self._decrement_group_level()
			self.record('END TEST_NA', None, name, str(e))
		except Exception, e:
			exc_info = sys.exc_info()
			self._decrement_group_level()
			err_msg = str(e) + '\n' + traceback.format_exc()
			self.record('END FAIL', None, name, err_msg)

		return result, exc_info


	def run_group(self, function, *args, **dargs):
		"""\
		function:
			subroutine to run
		*args:
			arguments for the function
		"""

		# Allow the tag for the group to be specified
		name = function.__name__
		tag = dargs.pop('tag', None)
		if tag:
			name = tag

		result, exc_info = self.__rungroup(name, function,
						   *args, **dargs)

		# if there was a non-TestError exception, raise it
		if exc_info and not isinstance(exc_info[1], error.TestError):
			err = ''.join(traceback.format_exception(*exc_info))
			raise error.TestError(name + ' failed\n' + err)

		# pass back the actual return value from the function
		return result


	def new_container(self, mbytes=None, cpus=None, root=None, name=None):
		if not autotest_utils.grep('cpuset', '/proc/filesystems'):
			print "Containers not enabled by latest reboot"
			return  # containers weren't enabled in this kernel boot
		pid = os.getpid()
		if not name:
			name = 'test%d' % pid  # make arbitrary unique name
		self.container = cpuset.cpuset(name, job_size=mbytes, 
			job_pid=pid, cpus=cpus, root=root)
		# This job's python shell is now running in the new container
		# and all forked test processes will inherit that container


	def release_container(self):
		if self.container:
			self.container.release()
			self.container = None


	def cpu_count(self):
		if self.container:
			return len(self.container.cpus)
		return autotest_utils.count_cpus()  # use total system count


	# Check the passed kernel identifier against the command line
	# and the running kernel, abort the job on missmatch.
	def kernel_check_ident(self, expected_when, expected_id, subdir,
			       type = 'src', patches=[]):
		print (("POST BOOT: checking booted kernel " +
			"mark=%d identity='%s' type='%s'") %
		       (expected_when, expected_id, type))

		running_id = autotest_utils.running_os_ident()

		cmdline = autotest_utils.read_one_line("/proc/cmdline")

		find_sum = re.compile(r'.*IDENT=(\d+)')
		m = find_sum.match(cmdline)
		cmdline_when = -1
		if m:
			cmdline_when = int(m.groups()[0])

		# We have all the facts, see if they indicate we
		# booted the requested kernel or not.
		bad = False
		if (type == 'src' and expected_id != running_id or
		    type == 'rpm' and
		    not running_id.startswith(expected_id + '::')):
			print "check_kernel_ident: kernel identifier mismatch"
			bad = True
		if expected_when != cmdline_when:
			print "check_kernel_ident: kernel command line mismatch"
			bad = True

		if bad:
			print "   Expected Ident: " + expected_id
			print "    Running Ident: " + running_id
			print "    Expected Mark: %d" % (expected_when)
			print "Command Line Mark: %d" % (cmdline_when)
			print "     Command Line: " + cmdline

			raise error.JobError("boot failure", "reboot.verify")

		kernel_info = {'kernel': expected_id}
		for i, patch in enumerate(patches):
			kernel_info["patch%d" % i] = patch
		self.record('GOOD', subdir, 'reboot.verify', expected_id)
		self._decrement_group_level()
		self.record('END GOOD', subdir, 'reboot',
			    optional_fields=kernel_info)


	def filesystem(self, device, mountpoint = None, loop_size = 0):
		if not mountpoint:
			mountpoint = self.tmpdir
		return filesystem.filesystem(self, device, mountpoint,loop_size)

	
	def enable_external_logging(self):
		pass


	def disable_external_logging(self):
		pass
	

	def reboot_setup(self):
		pass


	def reboot(self, tag='autotest'):
		self.reboot_setup()
		self.record('START', None, 'reboot')
		self._increment_group_level()
		self.record('GOOD', None, 'reboot.start')
		self.harness.run_reboot()
		default = self.config_get('boot.set_default')
		if default:
			self.bootloader.set_default(tag)
		else:
			self.bootloader.boot_once(tag)
		cmd = "(sleep 5; reboot) </dev/null >/dev/null 2>&1 &"
		autotest_utils.system(cmd)
		self.quit()


	def noop(self, text):
		print "job: noop: " + text


	def parallel(self, *tasklist):
		"""Run tasks in parallel"""

		pids = []
		old_log_filename = self.log_filename
		for i, task in enumerate(tasklist):
			self.log_filename = old_log_filename + (".%d" % i)
			task_func = lambda: task[0](*task[1:])
			pids.append(parallel.fork_start(self.resultdir, 
							task_func))

		old_log_path = os.path.join(self.resultdir, old_log_filename)
		old_log = open(old_log_path, "a")
		exceptions = []
		for i, pid in enumerate(pids):
			# wait for the task to finish
			try:
				parallel.fork_waitfor(self.resultdir, pid)
			except Exception, e:
				exceptions.append(e)
			# copy the logs from the subtask into the main log
			new_log_path = old_log_path + (".%d" % i)
			if os.path.exists(new_log_path):
				new_log = open(new_log_path)
				old_log.write(new_log.read())
				new_log.close()
				old_log.flush()
				os.remove(new_log_path)
		old_log.close()

		self.log_filename = old_log_filename

		# handle any exceptions raised by the parallel tasks
		if exceptions:
			msg = "%d task(s) failed" % len(exceptions)
			raise error.JobError(msg, str(exceptions), exceptions)


	def quit(self):
		# XXX: should have a better name.
		self.harness.run_pause()
		raise error.JobContinue("more to come")


	def complete(self, status):
		"""Clean up and exit"""
		# We are about to exit 'complete' so clean up the control file.
		try:
			os.unlink(self.state_file)
		except:
			pass

		self.harness.run_complete()
		self.disable_external_logging()
		sys.exit(status)


	def set_state(self, var, val):
		# Deep copies make sure that the state can't be altered
		# without it being re-written.  Perf wise, deep copies
		# are overshadowed by pickling/loading.
		self.state[var] = copy.deepcopy(val)
		pickle.dump(self.state, open(self.state_file, 'w'))


	def __load_state(self):
		assert not hasattr(self, "state")
		try:
			self.state = pickle.load(open(self.state_file, 'r'))
			self.state_existed = True
		except Exception:
			print "Initializing the state engine."
			self.state = {}
			self.set_state('__steps', []) # writes pickle file
			self.state_existed = False


	def get_state(self, var, default=None):
		if var in self.state or default == None:
			val = self.state[var]
		else:
			val = default
		return copy.deepcopy(val)


	def __create_step_tuple(self, fn, args, dargs):
		# Legacy code passes in an array where the first arg is
		# the function or its name.
		if isinstance(fn, list):
			assert(len(args) == 0)
			assert(len(dargs) == 0)
			args = fn[1:]
			fn = fn[0]
		# Pickling actual functions is harry, thus we have to call
		# them by name.  Unfortunately, this means only functions
		# defined globally can be used as a next step.
		if isinstance(fn, types.FunctionType):
			fn = fn.__name__
		if not isinstance(fn, types.StringTypes):
			raise StepError("Next steps must be functions or "
			                "strings containing the function name")
		return (fn, args, dargs)


	def next_step(self, fn, *args, **dargs):
		"""Define the next step"""
		steps = self.get_state('__steps')
		steps.append(self.__create_step_tuple(fn, args, dargs))
		self.set_state('__steps', steps)


	def next_step_prepend(self, fn, *args, **dargs):
		"""Insert a new step, executing first"""
		steps = self.get_state('__steps')
		steps.insert(0, self.__create_step_tuple(fn, args, dargs))
		self.set_state('__steps', steps)


	def step_engine(self):
		"""the stepping engine -- if the control file defines
		step_init we will be using this engine to drive multiple runs.
		"""
		"""Do the next step"""

		# Set up the environment and then interpret the control file.
		# Some control files will have code outside of functions,
		# which means we need to have our state engine initialized
		# before reading in the file.
		lcl = {'job': self}
		exec(JOB_PREAMBLE, lcl, lcl)
		execfile(self.control, lcl, lcl)

		# If we loaded in a mid-job state file, then we presumably
		# know what steps we have yet to run.
		if not self.state_existed:
			if lcl.has_key('step_init'):
				self.next_step([lcl['step_init']])

		# Iterate through the steps.  If we reboot, we'll simply
		# continue iterating on the next step.
		while len(self.get_state('__steps')) > 0:
			steps = self.get_state('__steps')
			(fn, args, dargs) = steps.pop(0)
			self.set_state('__steps', steps)

			lcl['__args'] = args
			lcl['__dargs'] = dargs
			exec(fn + "(*__args, **__dargs)", lcl, lcl)


	def _init_group_level(self):
		self.group_level = self.get_state("__group_level", default=0)


	def _increment_group_level(self):
		self.group_level += 1
		self.set_state("__group_level", self.group_level)


	def _decrement_group_level(self):
		self.group_level -= 1
		self.set_state("__group_level", self.group_level)


	def record(self, status_code, subdir, operation, status = '',
		   optional_fields=None):
		"""
		Record job-level status

		The intent is to make this file both machine parseable and
		human readable. That involves a little more complexity, but
		really isn't all that bad ;-)

		Format is <status code>\t<subdir>\t<operation>\t<status>

		status code: (GOOD|WARN|FAIL|ABORT)
			or   START
			or   END (GOOD|WARN|FAIL|ABORT)

		subdir: MUST be a relevant subdirectory in the results,
		or None, which will be represented as '----'

		operation: description of what you ran (e.g. "dbench", or
						"mkfs -t foobar /dev/sda9")

		status: error message or "completed sucessfully"

		------------------------------------------------------------

		Initial tabs indicate indent levels for grouping, and is
		governed by self.group_level

		multiline messages have secondary lines prefaced by a double
		space ('  ')
		"""

		if subdir:
			if re.match(r'[\n\t]', subdir):
				raise ValueError("Invalid character in "
						 "subdir string")
			substr = subdir
		else:
			substr = '----'
		
		if not logging.is_valid_status(status_code):
			raise ValueError("Invalid status code supplied: %s" %
					 status_code)
		if not operation:
			operation = '----'

		if re.match(r'[\n\t]', operation):
			raise ValueError("Invalid character in "
					 "operation string")
		operation = operation.rstrip()

		if not optional_fields:
			optional_fields = {}

		status = status.rstrip()
		status = re.sub(r"\t", "  ", status)
		# Ensure any continuation lines are marked so we can
		# detect them in the status file to ensure it is parsable.
		status = re.sub(r"\n", "\n" + "\t" * self.group_level + "  ",
				status)

		# Generate timestamps for inclusion in the logs
		epoch_time = int(time.time())  # seconds since epoch, in UTC
		local_time = time.localtime(epoch_time)
		optional_fields["timestamp"] = str(epoch_time)
		optional_fields["localtime"] = time.strftime("%b %d %H:%M:%S",
							     local_time)

		fields = [status_code, substr, operation]
		fields += ["%s=%s" % x for x in optional_fields.iteritems()]
		fields.append(status)

		msg = '\t'.join(str(x) for x in fields)
		msg = '\t' * self.group_level + msg

		msg_tag = ""
		if "." in self.log_filename:
			msg_tag = self.log_filename.split(".", 1)[1]

		self.harness.test_status_detail(status_code, substr,
						operation, status, msg_tag)
		self.harness.test_status(msg, msg_tag)

		# log to stdout (if enabled)
		#if self.log_filename == self.DEFAULT_LOG_FILENAME:
		print msg

		# log to the "root" status log
		status_file = os.path.join(self.resultdir, self.log_filename)
		open(status_file, "a").write(msg + "\n")

		# log to the subdir status log (if subdir is set)
		if subdir:
			dir = os.path.join(self.resultdir, subdir)
			if not os.path.exists(dir):
				os.mkdir(dir)

			status_file = os.path.join(dir,
						   self.DEFAULT_LOG_FILENAME)
			open(status_file, "a").write(msg + "\n")


class disk_usage_monitor:
	def __init__(self, logging_func, device, max_mb_per_hour):
		self.func = logging_func
		self.device = device
		self.max_mb_per_hour = max_mb_per_hour


	def start(self):
		self.initial_space = autotest_utils.freespace(self.device)
		self.start_time = time.time()


	def stop(self):
		# if no maximum usage rate was set, we don't need to
		# generate any warnings
		if not self.max_mb_per_hour:
			return

		final_space = autotest_utils.freespace(self.device)
		used_space = self.initial_space - final_space
		stop_time = time.time()
		total_time = stop_time - self.start_time
		# round up the time to one minute, to keep extremely short
		# tests from generating false positives due to short, badly
		# timed bursts of activity
		total_time = max(total_time, 60.0)

		# determine the usage rate
		bytes_per_sec = used_space / total_time
		mb_per_sec = bytes_per_sec / 1024**2
		mb_per_hour = mb_per_sec * 60 * 60

		if mb_per_hour > self.max_mb_per_hour:
			msg = ("disk space on %s was consumed at a rate of "
			       "%.2f MB/hour")
			msg %= (self.device, mb_per_hour)
			self.func(msg)


	@classmethod
	def watch(cls, *monitor_args, **monitor_dargs):
		""" Generic decorator to wrap a function call with the
		standard create-monitor -> start -> call -> stop idiom."""
		def decorator(func):
			def watched_func(*args, **dargs):
				monitor = cls(*monitor_args, **monitor_dargs)
				monitor.start()
				try:
					func(*args, **dargs)
				finally:
					monitor.stop()
			return watched_func
		return decorator


def runjob(control, cont = False, tag = "default", harness_type = '',
	   use_external_logging = False):
	"""The main interface to this module

	control
		The control file to use for this job.
	cont
		Whether this is the continuation of a previously started job
	"""
	control = os.path.abspath(control)
	state = control + '.state'

	# instantiate the job object ready for the control file.
	myjob = None
	try:
		# Check that the control file is valid
		if not os.path.exists(control):
			raise error.JobError(control + 
						": control file not found")

		# When continuing, the job is complete when there is no
		# state file, ensure we don't try and continue.
		if cont and not os.path.exists(state):
			raise error.JobComplete("all done")
		if cont == False and os.path.exists(state):
			os.unlink(state)

		myjob = job(control, tag, cont, harness_type,
			    use_external_logging)

		# Load in the users control file, may do any one of:
		#  1) execute in toto
		#  2) define steps, and select the first via next_step()
		myjob.step_engine()

	except error.JobContinue:
		sys.exit(5)

	except error.JobComplete:
		sys.exit(1)

	except error.JobError, instance:
		print "JOB ERROR: " + instance.args[0]
		if myjob:
			command = None
			if len(instance.args) > 1:
				command = instance.args[1]
			myjob.record('ABORT', None, command, instance.args[0])
			myjob._decrement_group_level()
			myjob.record('END ABORT', None, None)
			assert(myjob.group_level == 0)
			myjob.complete(1)
		else:
			sys.exit(1)

	except Exception, e:
		msg = str(e) + '\n' + traceback.format_exc()
		print "JOB ERROR: " + msg
		if myjob:
			myjob.record('ABORT', None, None, msg)
			myjob._decrement_group_level()
			myjob.record('END ABORT', None, None)
			assert(myjob.group_level == 0)
			myjob.complete(1)
		else:
			sys.exit(1)

	# If we get here, then we assume the job is complete and good.
	myjob._decrement_group_level()
	myjob.record('END GOOD', None, None)
	assert(myjob.group_level == 0)

	myjob.complete(0)


# site_job.py may be non-existant or empty, make sure that an appropriate
# site_job class is created nevertheless
try:
	from site_job import site_job
except ImportError:
	class site_job(base_job):
		pass

class job(site_job):
	pass