Add autotest support in server side
From: poirier@google.com
This patch adds some functionality to autotest in autoserv. Credit goes
to Ryan for some of this.
Autotest support in autoserv:
* automatically hardreset the machine if it never comes back online
* autotest now uses get() interface
* remove autotest results directory if it is there before run (to avoid
mixing things up)
utils has unarchive code (to extract tar bz2 gz)
DEBKernel uses get() interface
git-svn-id: http://test.kernel.org/svn/autotest/trunk@570 592f7852-d20e-0410-864c-8624ca9c26a4
diff --git a/server/autoserv b/server/autoserv
index ae81de3..8ff6f92 100755
--- a/server/autoserv
+++ b/server/autoserv
@@ -2,13 +2,20 @@
#
# Copyright 2007 Google Inc. Released under the GPL v2
-"""Run an autoserv control file
+"""
+Run an autoserv control file
TODO(poirier): add a singleton logger
TODO(poirier): maybe change the name "get_file" to "receive_file" ?
+TODO(poirier): change get(), send_file(), get_file() to consistantly recognize
+ paths that start with '~' as refering to the home directory
"""
-__author__ = "poirier@google.com (Benjamin Poirier)"
+__author__ = """
+mbligh@google.com (Martin J. Bligh),
+poirier@google.com (Benjamin Poirier),
+stutsman@google.com (Ryan Stutsman)
+"""
import sys
diff --git a/server/autotest.py b/server/autotest.py
index c1c4f4a..b05da6a 100644
--- a/server/autotest.py
+++ b/server/autotest.py
@@ -49,36 +49,12 @@
"""
def __init__(self):
super(Autotest, self).__init__()
-
- def get_from_file(self, filename):
- """Specify a tarball on the local filesystem from which
- autotest will be installed.
-
- Args:
- filename: a str specifying the path to the tarball
-
- Raises:
- AutoservError: if filename does not exist
- """
- if os.path.exists(filename):
- self.__filename = filename
- else:
- raise errors.AutoservError('%s not found' % filename)
-
- def get_from_url(self, url):
- """Specify a url to a tarball from which autotest will be
- installed.
-
- Args:
- url: a str specifying the url to the tarball
- """
- self.__filename = utils.get(url)
+
def install(self, host):
- """Install autotest from the specified tarball (either
- via get_from_file or get_from_url). If neither were
- called an attempt will be made to install from the
- autotest svn repository.
+ """Install autotest. If get() was not called previously, an
+ attempt will be made to install from the autotest svn
+ repository.
Args:
host: a Host instance on which autotest will be
@@ -87,23 +63,28 @@
Raises:
AutoservError: if a tarball was not specified and
the target host does not have svn installed in its path
+
+ TODO(poirier): check dependencies
+ autotest needs:
+ bzcat
+ liboptdev (oprofile)
+ binutils-dev (oprofile)
+ make
"""
# try to install from file or directory
- try:
- if os.path.isdir(self.__filename):
+ if self.source_material:
+ if os.path.isdir(self.source_material):
# Copy autotest recursively
autodir = _get_autodir(host)
host.run('mkdir -p %s' %
utils.scp_remote_escape(autodir))
- host.send_file(self.__filename + '/',
+ host.send_file(self.source_material,
autodir)
else:
# Copy autotest via tarball
raise "Not yet implemented!"
return
- except AttributeError, e:
- pass
-
+
# if that fails try to install using svn
if utils.run('which svn').exit_status:
raise AutoservError('svn not found in path on \
@@ -119,7 +100,7 @@
def run(self, control_file, results_dir, host):
"""
Run an autotest job on the remote machine.
-
+
Args:
control_file: an open file-like-obj of the control file
results_dir: a str path where the results should be stored
@@ -133,21 +114,22 @@
"""
atrun = _Run(host, results_dir)
atrun.verify_machine()
+ if os.path.isdir(results_dir):
+ shutil.rmtree(results_dir)
debug = os.path.join(results_dir, 'debug')
- if not os.path.exists(debug):
- os.makedirs(debug)
-
+ os.makedirs(debug)
+
# Ready .... Aim ....
host.run('rm -f ' + atrun.remote_control_file)
host.run('rm -f ' + atrun.remote_control_file + '.state')
-
+
# Copy control_file to remote_control_file on the host
tmppath = utils.get(control_file)
host.send_file(tmppath, atrun.remote_control_file)
os.remove(tmppath)
-
+
atrun.execute_control()
-
+
# retrive results
results = os.path.join(atrun.autodir, 'results', 'default')
# Copy all dirs in default to results_dir
@@ -165,16 +147,14 @@
def __init__(self, host, results_dir):
self.host = host
self.results_dir = results_dir
-
+
self.autodir = _get_autodir(self.host)
self.remote_control_file = os.path.join(self.autodir, 'control')
-
-
+
def verify_machine(self):
binary = os.path.join(self.autodir, 'bin/autotest')
self.host.run('ls ' + binary)
-
-
+
def __execute_section(self, section):
print "Executing %s/bin/autotest %s/control phase %d" % \
(self.autodir, self.autodir,
@@ -201,8 +181,7 @@
raise AutotestRunError("execute_section: %s '%s' \
failed to return anything" % (ssh, cmd))
return line
-
-
+
def execute_control(self):
section = 0
while True:
@@ -217,7 +196,7 @@
if not self.host.wait_down(HALT_TIME):
raise AutotestRunError("%s \
failed to shutdown after %ds" %
- (self.host.hostname,
+ (self.host.hostname,
HALT_TIME))
print "Client down, waiting for restart"
if not self.host.wait_up(BOOT_TIME):
@@ -225,21 +204,22 @@
# hardreset the machine once if possible
# before failing this control file
if hasattr(self.host, 'hardreset'):
- print "Hardresetting %s" % self.hostname
+ print "Hardresetting %s" % (
+ self.hostname,)
self.host.hardreset()
- raise AutotestRunError("%s \
- failed to boot after %ds" %
- (self.host.hostname,
- BOOT_TIME))
+ raise AutotestRunError("%s failed to "
+ "boot after %ds" % (
+ self.host.hostname,
+ BOOT_TIME,))
continue
- raise AutotestRunError("Aborting - unknown \
- return code: %s\n" % last)
+ raise AutotestRunError("Aborting - unknown "
+ "return code: %s\n" % last)
def _get_autodir(host):
try:
atdir = host.run(
- 'grep autodir= /etc/autotest.conf').stdout.strip(" \n")
+ 'grep "autodir *=" /etc/autotest.conf').stdout.strip()
if atdir:
m = re.search(r'autodir *= *[\'"]?([^\'"]*)[\'"]?',
atdir)
diff --git a/server/deb_kernel.py b/server/deb_kernel.py
index 07d0cf4..a8c1471 100644
--- a/server/deb_kernel.py
+++ b/server/deb_kernel.py
@@ -35,26 +35,12 @@
super(DEBKernel, self).__init__()
- def get_from_file(self, filename):
- if os.path.exists(filename):
- self.__filename = filename
- else:
- raise errors.AutoservError('%s not found' % filename)
-
-
- def get_from_url(self, url):
- tmpdir = utils.get_tmp_dir()
- tmpfile = os.path.join(tmpdir, os.path.basename(url))
- urllib.urlretrieve(url, tmpfile)
- self.__filename = tmpfile
-
-
def install(self, host):
# this directory will get cleaned up for us automatically
remote_tmpdir = host.get_tmp_dir()
- basename = os.path.basename(self.__filename)
+ basename = os.path.basename(self.source_material)
remote_filename = os.path.join(remote_tmpdir, basename)
- host.send_file(self.__filename, remote_filename)
+ host.send_file(self.source_material, remote_filename)
try:
result = host.run('dpkg -i %s'
% remote_filename)
diff --git a/server/tests/autotest_test.py b/server/tests/autotest_test.py
index 46507dd..39dfe2a 100644
--- a/server/tests/autotest_test.py
+++ b/server/tests/autotest_test.py
@@ -73,12 +73,13 @@
host = MockInstallHost()
tmpdir = utils.get_tmp_dir()
- self.autotest.get_from_file(tmpdir)
+ self.autotest.get(tmpdir)
self.autotest.install(host)
- self.assertEqual(host.commands,
- ['mkdir -p /usr/local/autotest',
- 'send_file: %s/ %s' % (tmpdir,
- '/usr/local/autotest')])
+ self.assertEqual(host.commands[0],
+ 'mkdir -p /usr/local/autotest')
+ self.assertTrue(host.commands[1].startswith('send_file: /tmp/'))
+ self.assertTrue(host.commands[1].endswith(
+ '/ /usr/local/autotest'))
diff --git a/server/utils.py b/server/utils.py
index 9ed6218..ea860f5 100644
--- a/server/utils.py
+++ b/server/utils.py
@@ -32,7 +32,7 @@
def sh_escape(command):
"""Escape special characters from a command so that it can be passed
- as a double quoted (" ") string.
+ as a double quoted (" ") string in a (ba)sh command.
Args:
command: the command string to escape.
@@ -92,7 +92,8 @@
Returns:
The location of the file or directory where the requested
content was saved. This will be contained in a temporary
- directory on the local host.
+ directory on the local host. If the material to get was a
+ directory, the location will contain a trailing '/'
"""
tmpdir = get_tmp_dir()
@@ -246,3 +247,43 @@
for dir in __tmp_dirs:
shutil.rmtree(dir)
__tmp_dirs= []
+
+
+def unarchive(host, source_material):
+ """Uncompress and untar an archive on a host.
+
+ If the "source_material" is compresses (according to the file
+ extension) it will be uncompressed. Supported compression formats
+ are gzip and bzip2. Afterwards, if the source_material is a tar
+ archive, it will be untarred.
+
+ Args:
+ host: the host object on which the archive is located
+ source_material: the path of the archive on the host
+
+ Returns:
+ The file or directory name of the unarchived source material.
+ If the material is a tar archive, it will be extracted in the
+ directory where it is and the path returned will be the first
+ entry in the archive, assuming it is the topmost directory.
+ If the material is not an archive, nothing will be done so this
+ function is "harmless" when it is "useless".
+ """
+ # uncompress
+ if (source_material.endswith(".gz") or
+ source_material.endswith(".gzip")):
+ host.run('gunzip "%s"' % (sh_escape(source_material)))
+ source_material= ".".join(source_material.split(".")[:-1])
+ elif source_material.endswith("bz2"):
+ host.run('bunzip2 "%s"' % (sh_escape(source_material)))
+ source_material= ".".join(source_material.split(".")[:-1])
+
+ # untar
+ if source_material.endswith(".tar"):
+ retval= host.run('tar -C "%s" -xvf "%s"' % (
+ sh_escape(os.path.dirname(source_material)),
+ sh_escape(source_material),))
+ source_material= os.path.join(os.path.dirname(source_material),
+ retval.stdout.split()[0])
+
+ return source_material