Refactor all the sysinfo collection code into something a bit less
ad-hoc than the original code. In particular:
- move all the code into base_sysinfo and site_sysinfo classes, and
use inheritance to hook the site-specific code rather that having
to add explicit hooks to the base code
- get rid of all the magic state created in client/bin/test.py so
that we can pass data between the sysinfo code run before and
after each test; instead, just have a single sysinfo instance
associated with a job and store state there
- change all the code to work with paths taken from job and test
objects (i.e. job.resultdir and test.outputdir) instead of just
magically changing the working directory and implicitly doing all
logging in place
- change the reboot in the job sysinfo dir to call them boot.0,
boot.1, etc. to be more in line with the names we use in TKO
- replace some code that used shell commands with functions from
the python standard library (e.g. os.symlink instead of "ln -s")
Risk: Medium
Visibility: reboot.N -> boot.(N-1) in sysinfo; all other changes
should not be user visible
Signed-off-by: John Admanski <jadmanski@google.com>
git-svn-id: http://test.kernel.org/svn/autotest/trunk@2257 592f7852-d20e-0410-864c-8624ca9c26a4
diff --git a/client/bin/base_sysinfo.py b/client/bin/base_sysinfo.py
new file mode 100644
index 0000000..f7490f9
--- /dev/null
+++ b/client/bin/base_sysinfo.py
@@ -0,0 +1,184 @@
+import os, shutil, re, glob, subprocess
+
+from autotest_lib.client.common_lib import utils, log
+
+
+class command(object):
+ def __init__(self, cmd, logfile=None):
+ self.cmd = cmd
+ if logfile:
+ self.logfile = logfile
+ else:
+ self.logfile = cmd.replace(" ", "_")
+
+
+ def __eq__(self, other):
+ if isinstance(other, command):
+ return (self.cmd, self.logfile) == (other.cmd, other.logfile)
+ return NotImplemented
+
+
+ def __ne__(self, other):
+ result = self.__eq__(other)
+ if result is NotImplemented:
+ return result
+ return not result
+
+
+ def __hash__(self):
+ return hash((self.cmd, self.logfile))
+
+
+ def run(self, logdir):
+ stdin = open(os.devnull, "r")
+ stdout = open(os.path.join(logdir, self.logfile), "w")
+ stderr = open(os.devnull, "w")
+ env = os.environ.copy()
+ if "PATH" not in env:
+ env["PATH"] = "/usr/bin:/bin"
+ subprocess.call(self.cmd, stdin=stdin, stdout=stdout, stderr=stderr,
+ shell=True, env=env)
+ for f in (stdin, stdout, stderr):
+ f.close()
+
+
+class base_sysinfo(object):
+ def __init__(self, job_resultsdir):
+ self.sysinfodir = self._get_sysinfodir(job_resultsdir)
+
+
+ @classmethod
+ def get_postboot_log_files(cls):
+ return set(["/proc/pci", "/proc/meminfo", "/proc/slabinfo",
+ "/proc/version", "/proc/cpuinfo", "/proc/cmdline",
+ "/proc/modules", "/proc/interrupts"])
+
+
+ @classmethod
+ def get_posttest_log_commands(cls):
+ return set([command("dmesg -c", "dmesg"), command("df -mP", "df")])
+
+
+ @classmethod
+ def get_postboot_log_commands(cls):
+ commands = cls.get_posttest_log_commands()
+ commands |= set([command("uname -a"), command("lspci -vvn"),
+ command("gcc --version"), command("ld --version"),
+ command("mount"), command("hostname")])
+ return commands
+
+
+ @staticmethod
+ def _get_sysinfodir(resultsdir):
+ sysinfodir = os.path.join(resultsdir, "sysinfo")
+ if not os.path.exists(sysinfodir):
+ os.makedirs(sysinfodir)
+ return sysinfodir
+
+
+ def _get_reboot_count(self):
+ if not glob.glob(os.path.join(self.sysinfodir, "*")):
+ return -1
+ else:
+ return len(glob.glob(os.path.join(self.sysinfodir, "boot.*")))
+
+
+ def _get_boot_subdir(self, next=False):
+ reboot_count = self._get_reboot_count()
+ if next:
+ reboot_count += 1
+ if reboot_count < 1:
+ return self.sysinfodir
+ else:
+ boot_dir = "boot.%d" % (reboot_count - 1)
+ return os.path.join(self.sysinfodir, boot_dir)
+
+
+ @log.log_and_ignore_errors("post-reboot sysinfo error:")
+ def log_per_reboot_data(self):
+ """ Log this data when the job starts, and again after any reboot. """
+ logdir = self._get_boot_subdir(next=True)
+ if not os.path.exists(logdir):
+ os.mkdir(logdir)
+
+ # run all the standard logging commands
+ for cmd in self.get_postboot_log_commands():
+ cmd.run(logdir)
+
+ # grab all the standard logging files
+ for filename in self.get_postboot_log_files():
+ if os.path.exists(filename):
+ shutil.copy(filename, logdir)
+
+
+ @log.log_and_ignore_errors("pre-test sysinfo error:")
+ def log_before_each_test(self, test):
+ if os.path.exists("/var/log/messages"):
+ stat = os.stat("/var/log/messages")
+ self._messages_size = stat.st_size
+ self._messages_inode = stat.st_ino
+
+
+ @log.log_and_ignore_errors("post-test sysinfo error:")
+ def log_after_each_test(self, test):
+ test_sysinfodir = self._get_sysinfodir(test.outputdir)
+
+ # create a symlink in the test sysinfo dir to the current boot
+ reboot_dir = self._get_boot_subdir()
+ assert os.path.exists(reboot_dir)
+ symlink_dest = os.path.join(test_sysinfodir, "reboot_current")
+ os.symlink(reboot_dir, symlink_dest)
+
+ # run all the standard logging commands
+ for cmd in self.get_posttest_log_commands():
+ cmd.run(test_sysinfodir)
+
+ # grab any new data from /var/log/messages
+ self._log_messages(test_sysinfodir)
+
+ # log some sysinfo data into the test keyval file
+ self._log_test_keyvals(test)
+
+
+ def _log_messages(self, logdir):
+ """ Log all of the new data in /var/log/messages. """
+ try:
+ # log all of the new data in /var/log/messages
+ bytes_to_skip = 0
+ if hasattr(self, "_messages_size"):
+ current_inode = os.stat("/var/log/messages").st_ino
+ if current_inode == self._messages_inode:
+ bytes_to_skip = self._messages_size
+ in_messages = open("/var/log/messages")
+ in_messages.seek(bytes_to_skip)
+ out_messages = open(os.path.join(logdir, "messages"), "w")
+ out_messages.write(in_messages.read())
+ in_messages.close()
+ out_messages.close()
+ except Exception, e:
+ print "/var/log/messages collection failed with %s" % e
+
+
+ def _log_test_keyvals(self, test):
+ keyval = {}
+ test_sysinfodir = self._get_sysinfodir(test.outputdir)
+
+ # grab a bunch of single line files and turn them into keyvals
+ files_to_log = ["cmdline", "uname_-a"]
+ keyval_fields = ["cmdline", "uname"]
+ for filename, field in zip(files_to_log, keyval_fields):
+ path = os.path.join(test_sysinfodir, "reboot_current", filename)
+ if os.path.exists(path):
+ keyval["sysinfo-%s" % field] = utils.read_one_line(path)
+
+ # grab the total memory
+ path = os.path.join(test_sysinfodir, "reboot_current", "meminfo")
+ if os.path.exists(path):
+ mem_data = open(path).read()
+ match = re.search(r"^MemTotal:\s+(\d+) kB$", mem_data,
+ re.MULTILINE)
+ if match:
+ keyval["sysinfo-memtotal-in-kb"] = match.group(1)
+
+ # write out the data to the test keyval file
+ test.write_test_keyval(keyval)
diff --git a/client/bin/job.py b/client/bin/job.py
index 03f7a79..07b1bb2 100755
--- a/client/bin/job.py
+++ b/client/bin/job.py
@@ -105,7 +105,7 @@
self.tmpdir = os.path.join(self.autodir, 'tmp')
self.toolsdir = os.path.join(self.autodir, 'tools')
self.resultdir = os.path.join(self.autodir, 'results', jobtag)
- self.sysinfodir = os.path.join(self.resultdir, 'sysinfo')
+ self.sysinfo = sysinfo.sysinfo(self.resultdir)
self.control = os.path.abspath(control)
self.state_file = self.control + '.state'
self.current_step_ancestry = []
@@ -143,7 +143,6 @@
if os.path.exists(self.resultdir):
utils.system('rm -rf ' + self.resultdir)
os.mkdir(self.resultdir)
- os.mkdir(self.sysinfodir)
os.mkdir(os.path.join(self.resultdir, 'debug'))
os.mkdir(os.path.join(self.resultdir, 'analysis'))
@@ -172,7 +171,7 @@
except:
pass
- sysinfo.log_per_reboot_data(self.sysinfodir)
+ self.sysinfo.log_per_reboot_data()
if not cont:
self.record('START', None, None)
diff --git a/client/bin/job_unittest.py b/client/bin/job_unittest.py
index 9cfefb5..a32e5b3 100644
--- a/client/bin/job_unittest.py
+++ b/client/bin/job_unittest.py
@@ -45,6 +45,7 @@
self.god.stub_class(config, 'config')
self.god.stub_class(boottool, 'boottool')
+ self.god.stub_class(sysinfo, 'sysinfo')
def tearDown(self):
@@ -69,7 +70,7 @@
results = os.path.join(self.autodir, 'results')
download = os.path.join(self.autodir, 'tests', 'download')
resultdir = os.path.join(self.autodir, 'results', self.jobtag)
- sysinfodir = os.path.join(resultdir, 'sysinfo')
+ job_sysinfo = sysinfo.sysinfo.expect_new(resultdir)
pkgdir = os.path.join(self.autodir, 'packages')
# record
@@ -88,7 +89,6 @@
os.path.exists.expect_call(resultdir).and_return(True)
utils.system.expect_call('rm -rf ' + resultdir)
os.mkdir.expect_call(resultdir)
- os.mkdir.expect_call(sysinfodir)
os.mkdir.expect_call(os.path.join(resultdir, 'debug'))
os.mkdir.expect_call(os.path.join(resultdir,
'analysis'))
@@ -104,7 +104,7 @@
self.job.config_get.expect_call(
'boottool.executable').and_return(None)
bootloader = boottool.boottool.expect_new(None)
- sysinfo.log_per_reboot_data.expect_call(sysinfodir)
+ job_sysinfo.log_per_reboot_data.expect_call()
if not cont:
self.job.record.expect_call('START', None, None)
self.job._increment_group_level.expect_call()
diff --git a/client/bin/sysinfo.py b/client/bin/sysinfo.py
index 5530f27..7efd1a0 100755
--- a/client/bin/sysinfo.py
+++ b/client/bin/sysinfo.py
@@ -1,164 +1,11 @@
-#!/usr/bin/python
-
-import common
-
-import os, shutil, re, glob
-from autotest_lib.client.common_lib import utils
-
+from autotest_lib.client.bin import base_sysinfo
try:
from autotest_lib.client.bin import site_sysinfo
- local = True
except ImportError:
- local = False
-
-# stuff to log per reboot
-files = ['/proc/pci', '/proc/meminfo', '/proc/slabinfo', '/proc/version',
- '/proc/cpuinfo', '/proc/cmdline', '/proc/modules', '/proc/interrupts']
-# commands = ['lshw'] # this causes problems triggering CDROM drives
-commands = ['uname -a', 'lspci -vvn', 'gcc --version', 'ld --version',
- 'mount', 'hostname']
-path = ['/usr/bin', '/bin']
-
-
-def run_command(command, output):
- parts = command.split(None, 1)
- cmd = parts[0]
- if len(parts) > 1:
- args = parts[1]
- else:
- args = ''
- for dir in path:
- pathname = dir + '/' + cmd
- if not os.path.exists(pathname):
- continue
- tmp_cmd = "%s %s > %s 2> /dev/null" % (pathname, args, output)
- utils.system(tmp_cmd)
-
-
-def reboot_count():
- if not glob.glob('*'):
- return -1 # No reboots, initial data not logged
- else:
- return len(glob.glob('reboot*'))
-
-
-def boot_subdir(reboot_count):
- """subdir of job sysinfo"""
- if reboot_count == 0:
- return '.'
- else:
- return 'reboot%d' % reboot_count
-
-
-def log_per_reboot_data(sysinfo_dir):
- """we log this data when the job starts, and again after any reboot"""
- pwd = os.getcwd()
- try:
- os.chdir(sysinfo_dir)
- subdir = boot_subdir(reboot_count() + 1)
- if not os.path.exists(subdir):
- os.mkdir(subdir)
- os.chdir(os.path.join(sysinfo_dir, subdir))
- _log_per_reboot_data()
- finally:
- os.chdir(pwd)
-
-
-def _log_per_reboot_data():
- """system info to log before each step of the job"""
- for command in commands:
- run_command(command, re.sub(r'\s', '_', command))
-
- for file in files:
- if (os.path.exists(file)):
- shutil.copyfile(file, os.path.basename(file))
-
- utils.system('dmesg -c > dmesg', ignore_status=True)
- utils.system('df -mP > df', ignore_status=True)
- if local:
- site_sysinfo.log_per_reboot_data()
-
-
-def log_before_each_test(state_dict, job_sysinfo_dir, test_sysinfo_dir):
- if os.path.exists("/var/log/messages"):
- stat = os.stat("/var/log/messages")
- state_dict["messages_size"] = stat.st_size
- state_dict["messages_inode"] = stat.st_ino
-
-
-def _log_messages(state_dict):
- """ Log all of the new data in /var/log/messages. """
- try:
- # log all of the new data in /var/log/messages
- bytes_to_skip = 0
- if "messages_size" in state_dict and "messages_inode" in state_dict:
- current_inode = os.stat("/var/log/messages").st_ino
- if current_inode == state_dict["messages_inode"]:
- bytes_to_skip = state_dict["messages_size"]
- in_messages = open("/var/log/messages")
- in_messages.seek(bytes_to_skip)
- out_messages = open("messages", "w")
- out_messages.write(in_messages.read())
- in_messages.close()
- out_messages.close()
- except Exception, e:
- print "/var/log/messages collection failed with %s" % e
-
-
-def log_after_each_test(state_dict, job_sysinfo_dir, test_sysinfo_dir):
- """log things that change after each test (called from test.py)"""
- pwd = os.getcwd()
- try:
- os.chdir(job_sysinfo_dir)
- reboot_subdir = boot_subdir(reboot_count())
- reboot_dir = os.path.join(job_sysinfo_dir, reboot_subdir)
- assert os.path.exists(reboot_dir)
-
- os.makedirs(test_sysinfo_dir)
- os.chdir(test_sysinfo_dir)
- utils.system('ln -s %s reboot_current' % reboot_dir)
-
- utils.system('dmesg -c > dmesg', ignore_status=True)
- utils.system('df -mP > df', ignore_status=True)
-
- _log_messages(state_dict)
-
- if local:
- site_sysinfo.log_after_each_test()
- finally:
- os.chdir(pwd)
-
-
-def log_test_keyvals(test, test_sysinfo_dir):
- """
- Extract some useful data from the sysinfo and write it out into
- the test keyval.
- """
- keyval = {}
-
- # grab a bunch of single line files and turn them into keyvals
- files_to_log = ["cmdline", "uname_-a"]
- keyval_fields = ["cmdline", "uname"]
- for filename, field in zip(files_to_log, keyval_fields):
- path = os.path.join(test_sysinfo_dir, "reboot_current", filename)
- if os.path.exists(path):
- keyval["sysinfo-%s" % field] = utils.read_one_line(path)
-
- # grab the total memory
- path = os.path.join(test_sysinfo_dir, "reboot_current", "meminfo")
- if os.path.exists(path):
- mem_data = open(path).read()
- match = re.search(r"^MemTotal:\s+(\d+) kB$", mem_data, re.MULTILINE)
- if match:
- keyval["sysinfo-memtotal-in-kb"] = match.group(1)
-
- # write out the data to the test keyval file
- test.write_test_keyval(keyval)
-
- # call the site-specific version of this function
- if local:
- site_sysinfo.log_test_keyvals(test, test_sysinfo_dir)
-
-
-if __name__ == '__main__':
- log_per_reboot_data()
+ # no site_sysinfo, just make a class using the base version
+ class sysinfo(base_sysinfo.base_sysinfo):
+ pass
+else:
+ # otherwise, use the site version (should inherit from the base)
+ class sysinfo(site_sysinfo.site_sysinfo):
+ pass
diff --git a/client/bin/test.py b/client/bin/test.py
index 14f6ed7..acae8e3 100755
--- a/client/bin/test.py
+++ b/client/bin/test.py
@@ -29,37 +29,7 @@
pass
-def _get_sysinfo_dirs(mytest):
- """ Returns (job_sysinfo_dir, test_sysinfo_dir) for a given test """
- job_dir = mytest.job.sysinfodir
- test_dir = os.path.join(mytest.outputdir, "sysinfo")
- return job_dir, test_dir
-
-
-def _prepare_sysinfo(state, mytest):
- try:
- job_dir, test_dir = _get_sysinfo_dirs(mytest)
- sysinfo.log_before_each_test(state, job_dir, test_dir)
- except:
- print "before-test error:"
- traceback.print_exc(file=sys.stdout)
-
-
-def _grab_sysinfo(state, mytest):
- try:
- job_dir, test_dir = _get_sysinfo_dirs(mytest)
- sysinfo.log_after_each_test(state, job_dir, test_dir)
- sysinfo.log_test_keyvals(mytest, test_dir)
- if os.path.exists(mytest.tmpdir):
- shutil.rmtree(mytest.tmpdir, ignore_errors=True)
- except:
- print "after-test error:"
- traceback.print_exc(file=sys.stdout)
-
-
def runtest(job, url, tag, args, dargs):
- state_dict = {}
- before_hook = lambda t: _prepare_sysinfo(state_dict, t)
- after_hook = lambda t: _grab_sysinfo(state_dict, t)
- common_test.runtest(job, url, tag, args, dargs,
- locals(), globals(), before_hook, after_hook)
+ common_test.runtest(job, url, tag, args, dargs, locals(), globals(),
+ job.sysinfo.log_before_each_test,
+ job.sysinfo.log_after_each_test)