Refactor of dmesg to more general monitoring of any log file.
Signed-off-by: Cary Hull <chull@google.com>
git-svn-id: http://test.kernel.org/svn/autotest/trunk@2395 592f7852-d20e-0410-864c-8624ca9c26a4
diff --git a/server/hosts/logfile_monitor.py b/server/hosts/logfile_monitor.py
new file mode 100644
index 0000000..fbb70e2
--- /dev/null
+++ b/server/hosts/logfile_monitor.py
@@ -0,0 +1,253 @@
+import os, sys, subprocess, tempfile, traceback
+
+from autotest_lib.client.common_lib import debug, utils
+from autotest_lib.server import utils as server_utils
+from autotest_lib.server.hosts import abstract_ssh, monitors
+
+MONITORDIR = monitors.__path__[0]
+
+
+class Error(Exception):
+ pass
+
+
+class InvalidPatternsPathError(Error):
+ """An invalid patterns_path was specified."""
+
+
+class InvalidConfigurationError(Error):
+ """An invalid configuration was specified."""
+
+
+def run_cmd_on_host(hostname, cmd, stdin, stdout, stderr):
+ base_cmd = abstract_ssh.make_ssh_command()
+ full_cmd = "%s %s \"%s\"" % (base_cmd, hostname,
+ server_utils.sh_escape(cmd))
+
+ return subprocess.Popen(full_cmd, stdin=stdin, stdout=stdout,
+ stderr=stderr, shell=True)
+
+
+def copy_monitordir(host):
+ """Copy over monitordir to a tmpdir on the remote host."""
+ tmp_dir = host.get_tmp_dir()
+ host.send_file(MONITORDIR, tmp_dir)
+ return os.path.join(tmp_dir, 'monitors')
+
+
+def launch_remote_followfiles(host, lastlines_dirpath, follow_paths):
+ """Launch followfiles.py remotely on follow_paths."""
+ remote_monitordir = copy_monitordir(host)
+ remote_script_path = os.path.join(
+ remote_monitordir, 'followfiles.py')
+
+ followfiles_cmd = '%s %s --lastlines_dirpath=%s %s' % (
+ sys.executable, remote_script_path,
+ lastlines_dirpath, ' '.join(follow_paths))
+
+ devnull_r = open(os.devnull, 'r')
+ devnull_w = open(os.devnull, 'w')
+ remote_followfiles_proc = run_cmd_on_host(
+ host.hostname, followfiles_cmd, stdout=subprocess.PIPE,
+ stdin=devnull_r, stderr=devnull_w)
+ return remote_followfiles_proc
+
+
+def resolve_patterns_path(patterns_path):
+ """Resolve patterns_path to existing absolute local path or raise.
+
+ As a convenience we allow users to specify a non-absolute patterns_path.
+ However these need to be resolved before allowing them to be passed down
+ to console.py.
+
+ For now we expect non-absolute ones to be in self.monitordir.
+ """
+ if os.path.isabs(patterns_path):
+ if os.path.exists(patterns_path):
+ return patterns_path
+ else:
+ raise InvalidPatternsPathError('Absolute path does not exist.')
+ else:
+ patterns_path = os.path.join(MONITORDIR, patterns_path)
+ if os.path.exists(patterns_path):
+ return patterns_path
+ else:
+ raise InvalidPatternsPathError('Relative path does not exist.')
+
+
+def launch_local_console(
+ input_stream, console_log_path, pattern_paths=None):
+ """Launch console.py locally.
+
+ This will process the output from followfiles and
+ fire warning messages per configuration in pattern_paths.
+ """
+ r, w = os.pipe()
+ local_script_path = os.path.join(MONITORDIR, 'console.py')
+ console_cmd = [sys.executable, local_script_path]
+ if pattern_paths:
+ console_cmd.append('--pattern_paths=%s' % ','.join(pattern_paths))
+
+ console_cmd += [console_log_path, str(w)]
+
+ # Setup warning stream before we actually launch
+ warning_stream = os.fdopen(r, 'r', 0)
+
+ devnull_r = open(os.devnull, 'r')
+ devnull_w = open(os.devnull, 'w')
+ # Launch console.py locally
+ console_proc = subprocess.Popen(
+ console_cmd, stdin=input_stream,
+ stdout=devnull_w, stderr=devnull_w)
+ os.close(w)
+ return console_proc, warning_stream
+
+
+def _log_and_ignore_exceptions(f):
+ """Decorator: automatically log exception during a method call.
+ """
+ def wrapped(self, *args, **dargs):
+ try:
+ return f(self, *args, **dargs)
+ except Exception, e:
+ print "LogfileMonitor.%s failed with exception %s" % (f.__name__, e)
+ print "Exception ignored:"
+ traceback.print_exc(file=sys.stdout)
+ wrapped.__name__ = f.__name__
+ wrapped.__doc__ = f.__doc__
+ wrapped.__dict__.update(f.__dict__)
+ return wrapped
+
+
+class LogfileMonitorMixin(abstract_ssh.AbstractSSHHost):
+ """This can monitor one or more remote files using tail.
+
+ This class and its counterpart script, monitors/followfiles.py,
+ add most functionality one would need to launch and monitor
+ remote tail processes on self.hostname.
+
+ This can be used by subclassing normally or by calling
+ NewLogfileMonitorMixin (below)
+
+ It is configured via two class attributes:
+ follow_paths: Remote paths to monitor
+ pattern_paths: Local paths to alert pattern definition files.
+ """
+ follow_paths = ()
+ pattern_paths = ()
+
+ def _initialize(self, console_log=None, *args, **dargs):
+ super(LogfileMonitorMixin, self)._initialize(*args, **dargs)
+
+ self._lastlines_dirpath = None
+ self._console_proc = None
+ self._console_log = console_log or 'logfile_monitor.log'
+ self.logfile_monitor_log = debug.get_logger()
+
+
+ def reboot_followup(self, *args, **dargs):
+ super(LogfileMonitorMixin, self).reboot_followup(*args, **dargs)
+ self.__stop_loggers()
+ self.__start_loggers()
+
+
+ def start_loggers(self):
+ super(LogfileMonitorMixin, self).start_loggers()
+ self.__start_loggers()
+
+
+ def remote_path_exists(self, remote_path):
+ """Return True if remote_path exists, False otherwise."""
+ return not self.run(
+ 'ls %s' % remote_path, ignore_status=True).exit_status
+
+
+ def check_remote_paths(self, remote_paths):
+ """Return list of remote_paths that currently exist."""
+ return [
+ path for path in remote_paths if self.remote_path_exists(path)]
+
+
+ @_log_and_ignore_exceptions
+ def __start_loggers(self):
+ """Start multifile monitoring logger.
+
+ Launch monitors/followfiles.py on the target and hook its output
+ to monitors/console.py locally.
+ """
+ # Check if follow_paths exist, in the case that one doesn't
+ # emit a warning and proceed.
+ follow_paths_set = set(self.follow_paths)
+ existing = self.check_remote_paths(follow_paths_set)
+ missing = follow_paths_set.difference(existing)
+ if missing:
+ # Log warning that we are missing expected remote paths.
+ self.logfile_monitor_log.warn(
+ 'Target(%s) missing expected remote paths: %s',
+ self.hostname, ', '.join(missing))
+
+ # If none of them exist just return (for now).
+ if not existing:
+ return
+
+ # Create a new lastlines_dirpath on the remote host if not already set.
+ if not self._lastlines_dirpath:
+ self._lastlines_dirpath = self.get_tmp_dir(parent='/var/log')
+
+ # Launch followfiles on target
+ self._followfiles_proc = launch_remote_followfiles(
+ self, self._lastlines_dirpath, existing)
+
+ # Ensure we have sane pattern_paths before launching console.py
+ sane_pattern_paths = []
+ for patterns_path in set(self.pattern_paths):
+ try:
+ patterns_path = resolve_patterns_path(patterns_path)
+ except InvalidPatternsPathError, e:
+ self.logfile_monitor_log.warn(
+ 'Specified patterns_path is invalid: %s, %s',
+ patterns_path, str(e))
+ else:
+ sane_pattern_paths.append(patterns_path)
+
+ # Launch console.py locally, pass in output stream from followfiles.
+ self._console_proc, self._logfile_warning_stream = \
+ launch_local_console(
+ self._followfiles_proc.stdout, self._console_log,
+ sane_pattern_paths)
+
+ if self.job:
+ self.job.warning_loggers.add(self._logfile_warning_stream)
+
+
+ def stop_loggers(self):
+ super(LogfileMonitorMixin, self).stop_loggers()
+ self.__stop_loggers()
+
+
+ @_log_and_ignore_exceptions
+ def __stop_loggers(self):
+ if self._console_proc:
+ utils.nuke_subprocess(self._console_proc)
+ utils.nuke_subprocess(self._followfiles_proc)
+ self._console_proc = self._followfile_proc = None
+ if self.job:
+ self.job.warning_loggers.discard(self._logfile_warning_stream)
+ self._logfile_warning_stream.close()
+
+
+def NewLogfileMonitorMixin(follow_paths, pattern_paths=None):
+ """Create a custom in-memory subclass of LogfileMonitorMixin.
+
+ Args:
+ follow_paths: list; Remote paths to tail.
+ pattern_paths: list; Local alert pattern definition files.
+ """
+ if not follow_paths or (pattern_paths and not follow_paths):
+ raise InvalidConfigurationError
+
+ return type(
+ 'LogfileMonitorMixin%d' % id(follow_paths),
+ (LogfileMonitorMixin,),
+ {'follow_paths': follow_paths,
+ 'pattern_paths': pattern_paths or ()})