Make sysinfo more configurable. This adds some methods to the job
classes, job.add_sysinfo_command and job.add_sysinfo_logfile, that
can be used to add more commands and/or logfiles to the standard
sysinfo collection for the job.
Risk: High
Visiblity: You can add code to control files to customize sysinfo
collection.
Signed-off-by: John Admanski <jadmanski@google.com>
git-svn-id: http://test.kernel.org/svn/autotest/trunk@2289 592f7852-d20e-0410-864c-8624ca9c26a4
diff --git a/client/bin/base_sysinfo.py b/client/bin/base_sysinfo.py
index f7490f9..5547aae 100644
--- a/client/bin/base_sysinfo.py
+++ b/client/bin/base_sysinfo.py
@@ -3,8 +3,70 @@
from autotest_lib.client.common_lib import utils, log
-class command(object):
- def __init__(self, cmd, logfile=None):
+_DEFAULT_COMMANDS_TO_LOG_PER_TEST = []
+_DEFAULT_COMMANDS_TO_LOG_PER_BOOT = [
+ "lspci -vvn", "gcc --version", "ld --version", "mount", "hostname",
+ ]
+
+_DEFAULT_FILES_TO_LOG_PER_TEST = []
+_DEFAULT_FILES_TO_LOG_PER_BOOT = [
+ "/proc/pci", "/proc/meminfo", "/proc/slabinfo", "/proc/version",
+ "/proc/cpuinfo", "/proc/modules", "/proc/interrupts",
+ ]
+
+
+class loggable(object):
+ """ Abstract class for representing all things "loggable" by sysinfo. """
+ def __init__(self, log_in_keyval):
+ self.log_in_keyval = log_in_keyval
+
+
+ def readline(self, logdir):
+ path = os.path.join(logdir, self.logfile)
+ if os.path.exists(path):
+ return utils.read_one_line(path)
+ else:
+ return ""
+
+
+class logfile(loggable):
+ def __init__(self, path, log_in_keyval=False):
+ super(logfile, self).__init__(log_in_keyval)
+ self.path = path
+ self.logfile = os.path.basename(self.path)
+
+
+ def __repr__(self):
+ return "sysinfo.logfile(%r, %r)" % (self.path, self.log_in_keyval)
+
+
+ def __eq__(self, other):
+ if isinstance(other, logfile):
+ return self.path == other.path
+ elif isinstance(other, loggable):
+ return False
+ return NotImplemented
+
+
+ def __ne__(self, other):
+ result = self.__eq__(other)
+ if result is NotImplemented:
+ return result
+ return not result
+
+
+ def __hash__(self):
+ return hash(self.path)
+
+
+ def run(self, logdir):
+ if os.path.exists(self.path):
+ shutil.copy(self.path, logdir)
+
+
+class command(loggable):
+ def __init__(self, cmd, logfile=None, log_in_keyval=False):
+ super(command, self).__init__(log_in_keyval)
self.cmd = cmd
if logfile:
self.logfile = logfile
@@ -12,9 +74,17 @@
self.logfile = cmd.replace(" ", "_")
+ def __repr__(self):
+ r = "sysinfo.command(%r, %r, %r)"
+ r %= (self.cmd, self.logfile, self.log_in_keyval)
+ return r
+
+
def __eq__(self, other):
if isinstance(other, command):
return (self.cmd, self.logfile) == (other.cmd, other.logfile)
+ elif isinstance(other, loggable):
+ return False
return NotImplemented
@@ -46,26 +116,36 @@
def __init__(self, job_resultsdir):
self.sysinfodir = self._get_sysinfodir(job_resultsdir)
+ # pull in the post-test logs to collect
+ self.test_loggables = set()
+ for cmd in _DEFAULT_COMMANDS_TO_LOG_PER_TEST:
+ self.test_loggables.add(command(cmd))
+ for filename in _DEFAULT_FILES_TO_LOG_PER_TEST:
+ self.test_loggables.add(logfile(filename))
- @classmethod
- def get_postboot_log_files(cls):
- return set(["/proc/pci", "/proc/meminfo", "/proc/slabinfo",
- "/proc/version", "/proc/cpuinfo", "/proc/cmdline",
- "/proc/modules", "/proc/interrupts"])
+ # pull in the EXTRA post-boot logs to collect
+ self.boot_loggables = set()
+ for cmd in _DEFAULT_COMMANDS_TO_LOG_PER_BOOT:
+ self.boot_loggables.add(command(cmd))
+ for filename in _DEFAULT_FILES_TO_LOG_PER_BOOT:
+ self.boot_loggables.add(logfile(filename))
+
+ # add in a couple of extra files and commands we want to grab
+ self.test_loggables.add(command("df -mP", logfile="df"))
+ self.test_loggables.add(command("dmesg -c", logfile="dmesg"))
+ self.boot_loggables.add(logfile("/proc/cmdline",
+ log_in_keyval=True))
+ self.boot_loggables.add(command("uname -a", logfile="uname",
+ log_in_keyval=True))
- @classmethod
- def get_posttest_log_commands(cls):
- return set([command("dmesg -c", "dmesg"), command("df -mP", "df")])
+ def serialize(self):
+ return {"boot": self.boot_loggables, "test": self.test_loggables}
- @classmethod
- def get_postboot_log_commands(cls):
- commands = cls.get_posttest_log_commands()
- commands |= set([command("uname -a"), command("lspci -vvn"),
- command("gcc --version"), command("ld --version"),
- command("mount"), command("hostname")])
- return commands
+ def deserialize(self, serialized):
+ self.boot_loggables = serialized["boot"]
+ self.test_loggables = serialized["test"]
@staticmethod
@@ -96,23 +176,19 @@
@log.log_and_ignore_errors("post-reboot sysinfo error:")
def log_per_reboot_data(self):
- """ Log this data when the job starts, and again after any reboot. """
+ """ Logging hook called whenever a job starts, and again after
+ any reboot. """
logdir = self._get_boot_subdir(next=True)
if not os.path.exists(logdir):
os.mkdir(logdir)
- # run all the standard logging commands
- for cmd in self.get_postboot_log_commands():
- cmd.run(logdir)
-
- # grab all the standard logging files
- for filename in self.get_postboot_log_files():
- if os.path.exists(filename):
- shutil.copy(filename, logdir)
+ for log in (self.test_loggables | self.boot_loggables):
+ log.run(logdir)
@log.log_and_ignore_errors("pre-test sysinfo error:")
def log_before_each_test(self, test):
+ """ Logging hook called before a test starts. """
if os.path.exists("/var/log/messages"):
stat = os.stat("/var/log/messages")
self._messages_size = stat.st_size
@@ -121,6 +197,7 @@
@log.log_and_ignore_errors("post-test sysinfo error:")
def log_after_each_test(self, test):
+ """ Logging hook called after a test finishs. """
test_sysinfodir = self._get_sysinfodir(test.outputdir)
# create a symlink in the test sysinfo dir to the current boot
@@ -130,14 +207,15 @@
os.symlink(reboot_dir, symlink_dest)
# run all the standard logging commands
- for cmd in self.get_posttest_log_commands():
- cmd.run(test_sysinfodir)
+ for log in self.test_loggables:
+ log.run(test_sysinfodir)
# grab any new data from /var/log/messages
self._log_messages(test_sysinfodir)
# log some sysinfo data into the test keyval file
- self._log_test_keyvals(test)
+ keyval = self.log_test_keyvals(test_sysinfodir)
+ test.write_test_keyval(keyval)
def _log_messages(self, logdir):
@@ -159,17 +237,26 @@
print "/var/log/messages collection failed with %s" % e
- def _log_test_keyvals(self, test):
+ @staticmethod
+ def _read_sysinfo_keyvals(loggables, logdir):
keyval = {}
- test_sysinfodir = self._get_sysinfodir(test.outputdir)
+ for log in loggables:
+ if log.log_in_keyval:
+ keyval["sysinfo-" + log.logfile] = log.readline(logdir)
+ return keyval
- # grab a bunch of single line files and turn them into keyvals
- files_to_log = ["cmdline", "uname_-a"]
- keyval_fields = ["cmdline", "uname"]
- for filename, field in zip(files_to_log, keyval_fields):
- path = os.path.join(test_sysinfodir, "reboot_current", filename)
- if os.path.exists(path):
- keyval["sysinfo-%s" % field] = utils.read_one_line(path)
+
+ def log_test_keyvals(self, test_sysinfodir):
+ """ Logging hook called by log_after_each_test to collect keyval
+ entries to be written in the test keyval. """
+ keyval = {}
+
+ # grab any loggables that should be in the keyval
+ keyval.update(self._read_sysinfo_keyvals(
+ self.test_loggables, test_sysinfodir))
+ keyval.update(self._read_sysinfo_keyvals(
+ self.boot_loggables,
+ os.path.join(test_sysinfodir, "reboot_current")))
# grab the total memory
path = os.path.join(test_sysinfodir, "reboot_current", "meminfo")
@@ -180,5 +267,5 @@
if match:
keyval["sysinfo-memtotal-in-kb"] = match.group(1)
- # write out the data to the test keyval file
- test.write_test_keyval(keyval)
+ # return what we collected
+ return keyval
diff --git a/client/bin/job.py b/client/bin/job.py
index 460fc51..bab0c2e 100755
--- a/client/bin/job.py
+++ b/client/bin/job.py
@@ -112,7 +112,7 @@
self.tmpdir = os.path.join(self.autodir, 'tmp')
self.toolsdir = os.path.join(self.autodir, 'tools')
self.resultdir = os.path.join(self.autodir, 'results', jobtag)
- self.sysinfo = sysinfo.sysinfo(self.resultdir)
+
self.control = os.path.abspath(control)
self.state_file = self.control + '.state'
self.current_step_ancestry = []
@@ -124,6 +124,10 @@
self.pkgdir = os.path.join(self.autodir, 'packages')
self.run_test_cleanup = self.get_state("__run_test_cleanup",
default=True)
+
+ self.sysinfo = sysinfo.sysinfo(self.resultdir)
+ self._load_sysinfo_state()
+
self.last_boot_tag = self.get_state("__last_boot_tag", default=None)
self.job_log = debug.get_logger(module='client')
self._is_continuation = cont
@@ -769,12 +773,14 @@
assert not hasattr(self, "state")
try:
self.state = pickle.load(open(self.state_file, 'r'))
- self.state_existed = True
+ initialize = "__steps" not in self.state
except Exception:
- print "Initializing the state engine."
self.state = {}
+ initialize = True
+
+ if initialize:
+ print "Initializing the state engine."
self.set_state('__steps', []) # writes pickle file
- self.state_existed = False
def get_state(self, var, default=NO_DEFAULT):
@@ -907,7 +913,7 @@
# If we loaded in a mid-job state file, then we presumably
# know what steps we have yet to run.
- if not self.state_existed:
+ if not self._is_continuation:
if global_control_vars.has_key('step_init'):
self.next_step(global_control_vars['step_init'])
@@ -925,6 +931,34 @@
self._add_step_init(local_vars, fn_name)
+ def add_sysinfo_command(self, command, logfile=None, on_every_test=False):
+ self._add_sysinfo_loggable(sysinfo.command(command, logfile),
+ on_every_test)
+
+
+ def add_sysinfo_logfile(self, file, on_every_test=False):
+ self._add_sysinfo_loggable(sysinfo.logfile(file), on_every_test)
+
+
+ def _add_sysinfo_loggable(self, loggable, on_every_test):
+ if on_every_test:
+ self.sysinfo.test_loggables.add(loggable)
+ else:
+ self.sysinfo.boot_loggables.add(loggable)
+ self._save_sysinfo_state()
+
+
+ def _load_sysinfo_state(self):
+ state = self.get_state("__sysinfo", None)
+ if state:
+ self.sysinfo.deserialize(state)
+
+
+ def _save_sysinfo_state(self):
+ state = self.sysinfo.serialize()
+ self.set_state("__sysinfo", state)
+
+
def _init_group_level(self):
self.group_level = self.get_state("__group_level", default=0)
@@ -1111,8 +1145,6 @@
# state file, ensure we don't try and continue.
if cont and not os.path.exists(state):
raise error.JobComplete("all done")
- if cont == False and os.path.exists(state):
- os.unlink(state)
myjob = job(control, tag, cont, harness_type, use_external_logging)
diff --git a/client/bin/job_unittest.py b/client/bin/job_unittest.py
index 99b13bf..14b724e 100644
--- a/client/bin/job_unittest.py
+++ b/client/bin/job_unittest.py
@@ -70,13 +70,15 @@
results = os.path.join(self.autodir, 'results')
download = os.path.join(self.autodir, 'tests', 'download')
resultdir = os.path.join(self.autodir, 'results', self.jobtag)
- job_sysinfo = sysinfo.sysinfo.expect_new(resultdir)
pkgdir = os.path.join(self.autodir, 'packages')
# record
self.job._load_state.expect_call()
self.job.get_state.expect_call("__run_test_cleanup",
default=True).and_return(True)
+ job_sysinfo = sysinfo.sysinfo.expect_new(resultdir)
+ self.job.get_state.expect_call("__sysinfo",
+ None).and_return(None)
self.job.get_state.expect_call("__last_boot_tag",
default=None).and_return(None)
if not cont:
diff --git a/client/bin/sysinfo.py b/client/bin/sysinfo.py
index 7efd1a0..5a92431 100755
--- a/client/bin/sysinfo.py
+++ b/client/bin/sysinfo.py
@@ -9,3 +9,7 @@
# otherwise, use the site version (should inherit from the base)
class sysinfo(site_sysinfo.site_sysinfo):
pass
+
+# pull in some data stucture stubs from base_sysinfo, for convenience
+logfile = base_sysinfo.logfile
+command = base_sysinfo.command