Convert all python code to use four-space indents instead of eight-space tabs.
Signed-off-by: John Admanski <jadmanski@google.com>
git-svn-id: http://test.kernel.org/svn/autotest/trunk@1658 592f7852-d20e-0410-864c-8624ca9c26a4
diff --git a/client/bin/autotest b/client/bin/autotest
index 9d3a968..34c1e26 100755
--- a/client/bin/autotest
+++ b/client/bin/autotest
@@ -28,27 +28,27 @@
parser = OptionParser()
parser.add_option("-c", "--continue", dest="cont", action="store_true",
- default=False, help="continue previously started job")
+ default=False, help="continue previously started job")
parser.add_option("-t", "--tag", dest="tag", type="string", default="default",
- help="set the job tag")
+ help="set the job tag")
parser.add_option("-H", "--harness", dest="harness", type="string", default='',
- help="set the harness type")
+ help="set the harness type")
parser.add_option("-l", "--external_logging", dest="log", action="store_true",
- default=False, help="enable external logging")
+ default=False, help="enable external logging")
def usage():
- parser.print_help()
- sys.exit(1)
+ parser.print_help()
+ sys.exit(1)
options, args = parser.parse_args()
# Check for a control file.
if len(args) != 1:
- usage()
+ usage()
# JOB: run the specified job control file.
job.runjob(os.path.abspath(args[0]), options.cont, options.tag, options.harness,
- options.log)
+ options.log)
diff --git a/client/bin/autotest.py b/client/bin/autotest.py
index 9d4faa7..7695892 100755
--- a/client/bin/autotest.py
+++ b/client/bin/autotest.py
@@ -1,17 +1,17 @@
import os, sys
class system:
- def __init__(self):
- self.autodir = os.environ['AUTODIR']
- self.resultdir = self.autodir + '/results'
- self.tmpdir = self.autodir + '/tmp'
+ def __init__(self):
+ self.autodir = os.environ['AUTODIR']
+ self.resultdir = self.autodir + '/results'
+ self.tmpdir = self.autodir + '/tmp'
- if not os.path.isdir(self.resultdir):
- os.mkdir(self.resultdir)
- if not os.path.isdir(self.tmpdir):
- os.mkdir(self.tmpdir)
- return None
+ if not os.path.isdir(self.resultdir):
+ os.mkdir(self.resultdir)
+ if not os.path.isdir(self.tmpdir):
+ os.mkdir(self.tmpdir)
+ return None
- def boot(self, tag=None):
- print "I OUGHT TO REBOOT NOW!"
+ def boot(self, tag=None):
+ print "I OUGHT TO REBOOT NOW!"
diff --git a/client/bin/autotest_client b/client/bin/autotest_client
index 762d0cb..1e275a4 100755
--- a/client/bin/autotest_client
+++ b/client/bin/autotest_client
@@ -17,11 +17,11 @@
# If we're using cpusets, run inside the root one by default
if os.path.exists("/dev/cpuset/tasks") and getpass.getuser() == "root":
- utils.write_one_line("/dev/cpuset/tasks", str(os.getpid()))
+ utils.write_one_line("/dev/cpuset/tasks", str(os.getpid()))
autodir = os.path.dirname(sys.argv[0])
autotest = os.path.join(autodir, 'autotest')
cmd = ' '.join([autotest, '-H simple'] + sys.argv[1:])
exit_code = subprocess.call(cmd, shell=True, stderr=subprocess.STDOUT,
- close_fds=False)
+ close_fds=False)
sys.exit(exit_code) # pass on the exit status from autotest
diff --git a/client/bin/autotest_utils.py b/client/bin/autotest_utils.py
index 1e6c725..f6eb86e 100755
--- a/client/bin/autotest_utils.py
+++ b/client/bin/autotest_utils.py
@@ -8,593 +8,593 @@
def grep(pattern, file):
- """
- This is mainly to fix the return code inversion from grep
- Also handles compressed files.
+ """
+ This is mainly to fix the return code inversion from grep
+ Also handles compressed files.
- returns 1 if the pattern is present in the file, 0 if not.
- """
- command = 'grep "%s" > /dev/null' % pattern
- ret = cat_file_to_cmd(file, command, ignore_status=True)
- return not ret
+ returns 1 if the pattern is present in the file, 0 if not.
+ """
+ command = 'grep "%s" > /dev/null' % pattern
+ ret = cat_file_to_cmd(file, command, ignore_status=True)
+ return not ret
def difflist(list1, list2):
- """returns items in list2 that are not in list1"""
- diff = [];
- for x in list2:
- if x not in list1:
- diff.append(x)
- return diff
+ """returns items in list2 that are not in list1"""
+ diff = [];
+ for x in list2:
+ if x not in list1:
+ diff.append(x)
+ return diff
def cat_file_to_cmd(file, command, ignore_status=0, return_output=False):
- """
- equivalent to 'cat file | command' but knows to use
- zcat or bzcat if appropriate
- """
- if return_output:
- run_cmd = utils.system_output
- else:
- run_cmd = utils.system
+ """
+ equivalent to 'cat file | command' but knows to use
+ zcat or bzcat if appropriate
+ """
+ if return_output:
+ run_cmd = utils.system_output
+ else:
+ run_cmd = utils.system
- if not os.path.isfile(file):
- raise NameError('invalid file %s to cat to command %s'
- % (file, command))
- if file.endswith('.bz2'):
- return run_cmd('bzcat ' + file + ' | ' + command, ignore_status)
- elif (file.endswith('.gz') or file.endswith('.tgz')):
- return run_cmd('zcat ' + file + ' | ' + command, ignore_status)
- else:
- return run_cmd('cat ' + file + ' | ' + command, ignore_status)
+ if not os.path.isfile(file):
+ raise NameError('invalid file %s to cat to command %s'
+ % (file, command))
+ if file.endswith('.bz2'):
+ return run_cmd('bzcat ' + file + ' | ' + command, ignore_status)
+ elif (file.endswith('.gz') or file.endswith('.tgz')):
+ return run_cmd('zcat ' + file + ' | ' + command, ignore_status)
+ else:
+ return run_cmd('cat ' + file + ' | ' + command, ignore_status)
def extract_tarball_to_dir(tarball, dir):
- """
- Extract a tarball to a specified directory name instead of whatever
- the top level of a tarball is - useful for versioned directory names, etc
- """
- if os.path.exists(dir):
- raise NameError, 'target %s already exists' % dir
- pwd = os.getcwd()
- os.chdir(os.path.dirname(os.path.abspath(dir)))
- newdir = extract_tarball(tarball)
- os.rename(newdir, dir)
- os.chdir(pwd)
+ """
+ Extract a tarball to a specified directory name instead of whatever
+ the top level of a tarball is - useful for versioned directory names, etc
+ """
+ if os.path.exists(dir):
+ raise NameError, 'target %s already exists' % dir
+ pwd = os.getcwd()
+ os.chdir(os.path.dirname(os.path.abspath(dir)))
+ newdir = extract_tarball(tarball)
+ os.rename(newdir, dir)
+ os.chdir(pwd)
def extract_tarball(tarball):
- """Returns the directory extracted by the tarball."""
- extracted = cat_file_to_cmd(tarball, 'tar xvf - 2>/dev/null',
- return_output=True).splitlines()
+ """Returns the directory extracted by the tarball."""
+ extracted = cat_file_to_cmd(tarball, 'tar xvf - 2>/dev/null',
+ return_output=True).splitlines()
- dir = None
+ dir = None
- for line in extracted:
- line = re.sub(r'^./', '', line)
- if not line or line == '.':
- continue
- topdir = line.split('/')[0]
- if os.path.isdir(topdir):
- if dir:
- assert(dir == topdir)
- else:
- dir = topdir
- if dir:
- return dir
- else:
- raise NameError('extracting tarball produced no dir')
+ for line in extracted:
+ line = re.sub(r'^./', '', line)
+ if not line or line == '.':
+ continue
+ topdir = line.split('/')[0]
+ if os.path.isdir(topdir):
+ if dir:
+ assert(dir == topdir)
+ else:
+ dir = topdir
+ if dir:
+ return dir
+ else:
+ raise NameError('extracting tarball produced no dir')
def get_md5sum(file_path):
- """Gets the md5sum of a file. You must provide a valid path to the file"""
- if not os.path.isfile(file_path):
- raise ValueError, 'invalid file %s to verify' % file_path
- return utils.system_output("md5sum " + file_path + " | awk '{print $1}'")
+ """Gets the md5sum of a file. You must provide a valid path to the file"""
+ if not os.path.isfile(file_path):
+ raise ValueError, 'invalid file %s to verify' % file_path
+ return utils.system_output("md5sum " + file_path + " | awk '{print $1}'")
def unmap_url_cache(cachedir, url, expected_md5):
- """\
- Downloads a file from a URL to a cache directory. If the file is already
- at the expected position and has the expected md5 number, let's not
- download it again.
- """
- # Let's convert cachedir to a canonical path, if it's not already
- cachedir = os.path.realpath(cachedir)
- if not os.path.isdir(cachedir):
- try:
- system('mkdir -p ' + cachedir)
- except:
- raise ValueError('Could not create cache directory %s' % cachedir)
- file_from_url = os.path.basename(url)
- file_local_path = os.path.join(cachedir, file_from_url)
- if os.path.isfile(file_local_path):
- file_md5 = get_md5sum(file_local_path)
- if file_md5 == expected_md5:
- # File is already at the expected position and ready to go
- src = file_from_url
- else:
- # Let's download the package again, it's corrupted...
- src = url
- else:
- # File is not there, let's download it
- src = url
- return utils.unmap_url(cachedir, src, cachedir)
+ """\
+ Downloads a file from a URL to a cache directory. If the file is already
+ at the expected position and has the expected md5 number, let's not
+ download it again.
+ """
+ # Let's convert cachedir to a canonical path, if it's not already
+ cachedir = os.path.realpath(cachedir)
+ if not os.path.isdir(cachedir):
+ try:
+ system('mkdir -p ' + cachedir)
+ except:
+ raise ValueError('Could not create cache directory %s' % cachedir)
+ file_from_url = os.path.basename(url)
+ file_local_path = os.path.join(cachedir, file_from_url)
+ if os.path.isfile(file_local_path):
+ file_md5 = get_md5sum(file_local_path)
+ if file_md5 == expected_md5:
+ # File is already at the expected position and ready to go
+ src = file_from_url
+ else:
+ # Let's download the package again, it's corrupted...
+ src = url
+ else:
+ # File is not there, let's download it
+ src = url
+ return utils.unmap_url(cachedir, src, cachedir)
def basename(path):
- i = path.rfind('/');
- return path[i+1:]
+ i = path.rfind('/');
+ return path[i+1:]
def force_copy(src, dest):
- """Replace dest with a new copy of src, even if it exists"""
- if os.path.isfile(dest):
- os.remove(dest)
- if os.path.isdir(dest):
- dest = os.path.join(dest, os.path.basename(src))
- shutil.copyfile(src, dest)
- return dest
+ """Replace dest with a new copy of src, even if it exists"""
+ if os.path.isfile(dest):
+ os.remove(dest)
+ if os.path.isdir(dest):
+ dest = os.path.join(dest, os.path.basename(src))
+ shutil.copyfile(src, dest)
+ return dest
def force_link(src, dest):
- """Link src to dest, overwriting it if it exists"""
- return utils.system("ln -sf %s %s" % (src, dest))
+ """Link src to dest, overwriting it if it exists"""
+ return utils.system("ln -sf %s %s" % (src, dest))
def file_contains_pattern(file, pattern):
- """Return true if file contains the specified egrep pattern"""
- if not os.path.isfile(file):
- raise NameError('file %s does not exist' % file)
- return not utils.system('egrep -q "' + pattern + '" ' + file, ignore_status=True)
+ """Return true if file contains the specified egrep pattern"""
+ if not os.path.isfile(file):
+ raise NameError('file %s does not exist' % file)
+ return not utils.system('egrep -q "' + pattern + '" ' + file, ignore_status=True)
def list_grep(list, pattern):
- """True if any item in list matches the specified pattern."""
- compiled = re.compile(pattern)
- for line in list:
- match = compiled.search(line)
- if (match):
- return 1
- return 0
+ """True if any item in list matches the specified pattern."""
+ compiled = re.compile(pattern)
+ for line in list:
+ match = compiled.search(line)
+ if (match):
+ return 1
+ return 0
def get_os_vendor():
- """Try to guess what's the os vendor
- """
- issue = '/etc/issue'
+ """Try to guess what's the os vendor
+ """
+ issue = '/etc/issue'
- if not os.path.isfile(issue):
- return 'Unknown'
+ if not os.path.isfile(issue):
+ return 'Unknown'
- if file_contains_pattern(issue, 'Red Hat'):
- return 'Red Hat'
- elif file_contains_pattern(issue, 'Fedora Core'):
- return 'Fedora Core'
- elif file_contains_pattern(issue, 'SUSE'):
- return 'SUSE'
- elif file_contains_pattern(issue, 'Ubuntu'):
- return 'Ubuntu'
- elif file_contains_pattern(issue, 'Debian'):
- return 'Debian'
- else:
- return 'Unknown'
+ if file_contains_pattern(issue, 'Red Hat'):
+ return 'Red Hat'
+ elif file_contains_pattern(issue, 'Fedora Core'):
+ return 'Fedora Core'
+ elif file_contains_pattern(issue, 'SUSE'):
+ return 'SUSE'
+ elif file_contains_pattern(issue, 'Ubuntu'):
+ return 'Ubuntu'
+ elif file_contains_pattern(issue, 'Debian'):
+ return 'Debian'
+ else:
+ return 'Unknown'
def get_vmlinux():
- """Return the full path to vmlinux
+ """Return the full path to vmlinux
- Ahem. This is crap. Pray harder. Bad Martin.
- """
- vmlinux = '/boot/vmlinux-%s' % utils.system_output('uname -r')
- if os.path.isfile(vmlinux):
- return vmlinux
- vmlinux = '/lib/modules/%s/build/vmlinux' % utils.system_output('uname -r')
- if os.path.isfile(vmlinux):
- return vmlinux
- return None
+ Ahem. This is crap. Pray harder. Bad Martin.
+ """
+ vmlinux = '/boot/vmlinux-%s' % utils.system_output('uname -r')
+ if os.path.isfile(vmlinux):
+ return vmlinux
+ vmlinux = '/lib/modules/%s/build/vmlinux' % utils.system_output('uname -r')
+ if os.path.isfile(vmlinux):
+ return vmlinux
+ return None
def get_systemmap():
- """Return the full path to System.map
+ """Return the full path to System.map
- Ahem. This is crap. Pray harder. Bad Martin.
- """
- map = '/boot/System.map-%s' % utils.system_output('uname -r')
- if os.path.isfile(map):
- return map
- map = '/lib/modules/%s/build/System.map' % utils.system_output('uname -r')
- if os.path.isfile(map):
- return map
- return None
+ Ahem. This is crap. Pray harder. Bad Martin.
+ """
+ map = '/boot/System.map-%s' % utils.system_output('uname -r')
+ if os.path.isfile(map):
+ return map
+ map = '/lib/modules/%s/build/System.map' % utils.system_output('uname -r')
+ if os.path.isfile(map):
+ return map
+ return None
def get_modules_dir():
- """Return the modules dir for the running kernel version"""
- kernel_version = utils.system_output('uname -r')
- return '/lib/modules/%s/kernel' % kernel_version
+ """Return the modules dir for the running kernel version"""
+ kernel_version = utils.system_output('uname -r')
+ return '/lib/modules/%s/kernel' % kernel_version
def get_cpu_arch():
- """Work out which CPU architecture we're running on"""
- f = open('/proc/cpuinfo', 'r')
- cpuinfo = f.readlines()
- f.close()
- if list_grep(cpuinfo, '^cpu.*(RS64|POWER3|Broadband Engine)'):
- return 'power'
- elif list_grep(cpuinfo, '^cpu.*POWER4'):
- return 'power4'
- elif list_grep(cpuinfo, '^cpu.*POWER5'):
- return 'power5'
- elif list_grep(cpuinfo, '^cpu.*POWER6'):
- return 'power6'
- elif list_grep(cpuinfo, '^cpu.*PPC970'):
- return 'power970'
- elif list_grep(cpuinfo, 'Opteron'):
- return 'x86_64'
- elif list_grep(cpuinfo, 'GenuineIntel') and list_grep(cpuinfo, '48 bits virtual'):
- return 'x86_64'
- else:
- return 'i386'
+ """Work out which CPU architecture we're running on"""
+ f = open('/proc/cpuinfo', 'r')
+ cpuinfo = f.readlines()
+ f.close()
+ if list_grep(cpuinfo, '^cpu.*(RS64|POWER3|Broadband Engine)'):
+ return 'power'
+ elif list_grep(cpuinfo, '^cpu.*POWER4'):
+ return 'power4'
+ elif list_grep(cpuinfo, '^cpu.*POWER5'):
+ return 'power5'
+ elif list_grep(cpuinfo, '^cpu.*POWER6'):
+ return 'power6'
+ elif list_grep(cpuinfo, '^cpu.*PPC970'):
+ return 'power970'
+ elif list_grep(cpuinfo, 'Opteron'):
+ return 'x86_64'
+ elif list_grep(cpuinfo, 'GenuineIntel') and list_grep(cpuinfo, '48 bits virtual'):
+ return 'x86_64'
+ else:
+ return 'i386'
def get_current_kernel_arch():
- """Get the machine architecture, now just a wrap of 'uname -m'."""
- return os.popen('uname -m').read().rstrip()
+ """Get the machine architecture, now just a wrap of 'uname -m'."""
+ return os.popen('uname -m').read().rstrip()
def get_file_arch(filename):
- # -L means follow symlinks
- file_data = utils.system_output('file -L ' + filename)
- if file_data.count('80386'):
- return 'i386'
- return None
+ # -L means follow symlinks
+ file_data = utils.system_output('file -L ' + filename)
+ if file_data.count('80386'):
+ return 'i386'
+ return None
def count_cpus():
- """number of CPUs in the local machine according to /proc/cpuinfo"""
- f = file('/proc/cpuinfo', 'r')
- cpus = 0
- for line in f.readlines():
- if line.startswith('processor'):
- cpus += 1
- return cpus
+ """number of CPUs in the local machine according to /proc/cpuinfo"""
+ f = file('/proc/cpuinfo', 'r')
+ cpus = 0
+ for line in f.readlines():
+ if line.startswith('processor'):
+ cpus += 1
+ return cpus
# Returns total memory in kb
def read_from_meminfo(key):
- meminfo = utils.system_output('grep %s /proc/meminfo' % key)
- return int(re.search(r'\d+', meminfo).group(0))
+ meminfo = utils.system_output('grep %s /proc/meminfo' % key)
+ return int(re.search(r'\d+', meminfo).group(0))
def memtotal():
- return read_from_meminfo('MemTotal')
+ return read_from_meminfo('MemTotal')
def freememtotal():
- return read_from_meminfo('MemFree')
+ return read_from_meminfo('MemFree')
def sysctl_kernel(key, value=None):
- """(Very) partial implementation of sysctl, for kernel params"""
- if value:
- # write
- utils.write_one_line('/proc/sys/kernel/%s' % key, str(value))
- else:
- # read
- out = utils.read_one_line('/proc/sys/kernel/%s' % key)
- return int(re.search(r'\d+', out).group(0))
+ """(Very) partial implementation of sysctl, for kernel params"""
+ if value:
+ # write
+ utils.write_one_line('/proc/sys/kernel/%s' % key, str(value))
+ else:
+ # read
+ out = utils.read_one_line('/proc/sys/kernel/%s' % key)
+ return int(re.search(r'\d+', out).group(0))
def _convert_exit_status(sts):
- if os.WIFSIGNALED(sts):
- return -os.WTERMSIG(sts)
- elif os.WIFEXITED(sts):
- return os.WEXITSTATUS(sts)
- else:
- # impossible?
- raise RuntimeError("Unknown exit status %d!" % sts)
+ if os.WIFSIGNALED(sts):
+ return -os.WTERMSIG(sts)
+ elif os.WIFEXITED(sts):
+ return os.WEXITSTATUS(sts)
+ else:
+ # impossible?
+ raise RuntimeError("Unknown exit status %d!" % sts)
def where_art_thy_filehandles():
- """Dump the current list of filehandles"""
- os.system("ls -l /proc/%d/fd >> /dev/tty" % os.getpid())
+ """Dump the current list of filehandles"""
+ os.system("ls -l /proc/%d/fd >> /dev/tty" % os.getpid())
def print_to_tty(string):
- """Output string straight to the tty"""
- open('/dev/tty', 'w').write(string + '\n')
+ """Output string straight to the tty"""
+ open('/dev/tty', 'w').write(string + '\n')
def dump_object(object):
- """Dump an object's attributes and methods
+ """Dump an object's attributes and methods
- kind of like dir()
- """
- for item in object.__dict__.iteritems():
- print item
- try:
- (key,value) = item
- dump_object(value)
- except:
- continue
+ kind of like dir()
+ """
+ for item in object.__dict__.iteritems():
+ print item
+ try:
+ (key,value) = item
+ dump_object(value)
+ except:
+ continue
def environ(env_key):
- """return the requested environment variable, or '' if unset"""
- if (os.environ.has_key(env_key)):
- return os.environ[env_key]
- else:
- return ''
+ """return the requested environment variable, or '' if unset"""
+ if (os.environ.has_key(env_key)):
+ return os.environ[env_key]
+ else:
+ return ''
def prepend_path(newpath, oldpath):
- """prepend newpath to oldpath"""
- if (oldpath):
- return newpath + ':' + oldpath
- else:
- return newpath
+ """prepend newpath to oldpath"""
+ if (oldpath):
+ return newpath + ':' + oldpath
+ else:
+ return newpath
def append_path(oldpath, newpath):
- """append newpath to oldpath"""
- if (oldpath):
- return oldpath + ':' + newpath
- else:
- return newpath
+ """append newpath to oldpath"""
+ if (oldpath):
+ return oldpath + ':' + newpath
+ else:
+ return newpath
def avgtime_print(dir):
- """ Calculate some benchmarking statistics.
- Input is a directory containing a file called 'time'.
- File contains one-per-line results of /usr/bin/time.
- Output is average Elapsed, User, and System time in seconds,
- and average CPU percentage.
- """
- f = open(dir + "/time")
- user = system = elapsed = cpu = count = 0
- r = re.compile('([\d\.]*)user ([\d\.]*)system (\d*):([\d\.]*)elapsed (\d*)%CPU')
- for line in f.readlines():
- try:
- s = r.match(line);
- user += float(s.group(1))
- system += float(s.group(2))
- elapsed += (float(s.group(3)) * 60) + float(s.group(4))
- cpu += float(s.group(5))
- count += 1
- except:
- raise ValueError("badly formatted times")
+ """ Calculate some benchmarking statistics.
+ Input is a directory containing a file called 'time'.
+ File contains one-per-line results of /usr/bin/time.
+ Output is average Elapsed, User, and System time in seconds,
+ and average CPU percentage.
+ """
+ f = open(dir + "/time")
+ user = system = elapsed = cpu = count = 0
+ r = re.compile('([\d\.]*)user ([\d\.]*)system (\d*):([\d\.]*)elapsed (\d*)%CPU')
+ for line in f.readlines():
+ try:
+ s = r.match(line);
+ user += float(s.group(1))
+ system += float(s.group(2))
+ elapsed += (float(s.group(3)) * 60) + float(s.group(4))
+ cpu += float(s.group(5))
+ count += 1
+ except:
+ raise ValueError("badly formatted times")
- f.close()
- return "Elapsed: %0.2fs User: %0.2fs System: %0.2fs CPU: %0.0f%%" % \
- (elapsed/count, user/count, system/count, cpu/count)
+ f.close()
+ return "Elapsed: %0.2fs User: %0.2fs System: %0.2fs CPU: %0.0f%%" % \
+ (elapsed/count, user/count, system/count, cpu/count)
def running_config():
- """
- Return path of config file of the currently running kernel
- """
- version = utils.system_output('uname -r')
- for config in ('/proc/config.gz', \
- '/boot/config-%s' % version,
- '/lib/modules/%s/build/.config' % version):
- if os.path.isfile(config):
- return config
- return None
+ """
+ Return path of config file of the currently running kernel
+ """
+ version = utils.system_output('uname -r')
+ for config in ('/proc/config.gz', \
+ '/boot/config-%s' % version,
+ '/lib/modules/%s/build/.config' % version):
+ if os.path.isfile(config):
+ return config
+ return None
def check_for_kernel_feature(feature):
- config = running_config()
+ config = running_config()
- if not config:
- raise TypeError("Can't find kernel config file")
+ if not config:
+ raise TypeError("Can't find kernel config file")
- if config.endswith('.gz'):
- grep = 'zgrep'
- else:
- grep = 'grep'
- grep += ' ^CONFIG_%s= %s' % (feature, config)
+ if config.endswith('.gz'):
+ grep = 'zgrep'
+ else:
+ grep = 'grep'
+ grep += ' ^CONFIG_%s= %s' % (feature, config)
- if not utils.system_output(grep, ignore_status=True):
- raise ValueError("Kernel doesn't have a %s feature" % (feature))
+ if not utils.system_output(grep, ignore_status=True):
+ raise ValueError("Kernel doesn't have a %s feature" % (feature))
def cpu_online_map():
- """
- Check out the available cpu online map
- """
- cpus = []
- for line in open('/proc/cpuinfo', 'r').readlines():
- if line.startswith('processor'):
- cpus.append(line.split()[2]) # grab cpu number
- return cpus
+ """
+ Check out the available cpu online map
+ """
+ cpus = []
+ for line in open('/proc/cpuinfo', 'r').readlines():
+ if line.startswith('processor'):
+ cpus.append(line.split()[2]) # grab cpu number
+ return cpus
def check_glibc_ver(ver):
- glibc_ver = commands.getoutput('ldd --version').splitlines()[0]
- glibc_ver = re.search(r'(\d+\.\d+(\.\d+)?)', glibc_ver).group()
- if glibc_ver.split('.') < ver.split('.'):
- raise error.TestError("Glibc is too old (%s). Glibc >= %s is needed." % \
- (glibc_ver, ver))
+ glibc_ver = commands.getoutput('ldd --version').splitlines()[0]
+ glibc_ver = re.search(r'(\d+\.\d+(\.\d+)?)', glibc_ver).group()
+ if glibc_ver.split('.') < ver.split('.'):
+ raise error.TestError("Glibc is too old (%s). Glibc >= %s is needed." % \
+ (glibc_ver, ver))
def check_kernel_ver(ver):
- kernel_ver = utils.system_output('uname -r')
- kv_tmp = re.split(r'[-]', kernel_ver)[0:3]
- if kv_tmp[0].split('.') < ver.split('.'):
- raise error.TestError("Kernel is too old (%s). Kernel > %s is needed." % \
- (kernel_ver, ver))
+ kernel_ver = utils.system_output('uname -r')
+ kv_tmp = re.split(r'[-]', kernel_ver)[0:3]
+ if kv_tmp[0].split('.') < ver.split('.'):
+ raise error.TestError("Kernel is too old (%s). Kernel > %s is needed." % \
+ (kernel_ver, ver))
def human_format(number):
- # Convert number to kilo / mega / giga format.
- if number < 1024:
- return "%d" % number
- kilo = float(number) / 1024.0
- if kilo < 1024:
- return "%.2fk" % kilo
- meg = kilo / 1024.0
- if meg < 1024:
- return "%.2fM" % meg
- gig = meg / 1024.0
- return "%.2fG" % gig
+ # Convert number to kilo / mega / giga format.
+ if number < 1024:
+ return "%d" % number
+ kilo = float(number) / 1024.0
+ if kilo < 1024:
+ return "%.2fk" % kilo
+ meg = kilo / 1024.0
+ if meg < 1024:
+ return "%.2fM" % meg
+ gig = meg / 1024.0
+ return "%.2fG" % gig
def numa_nodes():
- node_paths = glob.glob('/sys/devices/system/node/node*')
- nodes = [int(re.sub(r'.*node(\d+)', r'\1', x)) for x in node_paths]
- return (sorted(nodes))
+ node_paths = glob.glob('/sys/devices/system/node/node*')
+ nodes = [int(re.sub(r'.*node(\d+)', r'\1', x)) for x in node_paths]
+ return (sorted(nodes))
def node_size():
- nodes = max(len(numa_nodes()), 1)
- return ((memtotal() * 1024) / nodes)
+ nodes = max(len(numa_nodes()), 1)
+ return ((memtotal() * 1024) / nodes)
def to_seconds(time_string):
- """Converts a string in M+:SS.SS format to S+.SS"""
- elts = time_string.split(':')
- if len(elts) == 1:
- return time_string
- return str(int(elts[0]) * 60 + float(elts[1]))
+ """Converts a string in M+:SS.SS format to S+.SS"""
+ elts = time_string.split(':')
+ if len(elts) == 1:
+ return time_string
+ return str(int(elts[0]) * 60 + float(elts[1]))
def extract_all_time_results(results_string):
- """Extract user, system, and elapsed times into a list of tuples"""
- pattern = re.compile(r"(.*?)user (.*?)system (.*?)elapsed")
- results = []
- for result in pattern.findall(results_string):
- results.append(tuple([to_seconds(elt) for elt in result]))
- return results
+ """Extract user, system, and elapsed times into a list of tuples"""
+ pattern = re.compile(r"(.*?)user (.*?)system (.*?)elapsed")
+ results = []
+ for result in pattern.findall(results_string):
+ results.append(tuple([to_seconds(elt) for elt in result]))
+ return results
def pickle_load(filename):
- return pickle.load(open(filename, 'r'))
+ return pickle.load(open(filename, 'r'))
# Return the kernel version and build timestamp.
def running_os_release():
- return os.uname()[2:4]
+ return os.uname()[2:4]
def running_os_ident():
- (version, timestamp) = running_os_release()
- return version + '::' + timestamp
+ (version, timestamp) = running_os_release()
+ return version + '::' + timestamp
# much like find . -name 'pattern'
def locate(pattern, root=os.getcwd()):
- for path, dirs, files in os.walk(root):
- for f in [os.path.abspath(os.path.join(path, f))
- for f in files if fnmatch.fnmatch(f, pattern)]:
- yield f
+ for path, dirs, files in os.walk(root):
+ for f in [os.path.abspath(os.path.join(path, f))
+ for f in files if fnmatch.fnmatch(f, pattern)]:
+ yield f
def freespace(path):
- """Return the disk free space, in bytes"""
- s = os.statvfs(path)
- return s.f_bavail * s.f_bsize
+ """Return the disk free space, in bytes"""
+ s = os.statvfs(path)
+ return s.f_bavail * s.f_bsize
def disk_block_size(path):
- """Return the disk block size, in bytes"""
- return os.statvfs(path).f_bsize
+ """Return the disk block size, in bytes"""
+ return os.statvfs(path).f_bsize
def get_cpu_family():
- procinfo = utils.system_output('cat /proc/cpuinfo')
- CPU_FAMILY_RE = re.compile(r'^cpu family\s+:\s+(\S+)', re.M)
- matches = CPU_FAMILY_RE.findall(procinfo)
- if matches:
- return int(matches[0])
- else:
- raise error.TestError('Could not get valid cpu family data')
+ procinfo = utils.system_output('cat /proc/cpuinfo')
+ CPU_FAMILY_RE = re.compile(r'^cpu family\s+:\s+(\S+)', re.M)
+ matches = CPU_FAMILY_RE.findall(procinfo)
+ if matches:
+ return int(matches[0])
+ else:
+ raise error.TestError('Could not get valid cpu family data')
def get_disks():
- df_output = utils.system_output('df')
- disk_re = re.compile(r'^(/dev/hd[a-z]+)3', re.M)
- return disk_re.findall(df_output)
+ df_output = utils.system_output('df')
+ disk_re = re.compile(r'^(/dev/hd[a-z]+)3', re.M)
+ return disk_re.findall(df_output)
def load_module(module_name):
- # Checks if a module has already been loaded
- if module_is_loaded(module_name):
- return False
+ # Checks if a module has already been loaded
+ if module_is_loaded(module_name):
+ return False
- utils.system('/sbin/modprobe ' + module_name)
- return True
+ utils.system('/sbin/modprobe ' + module_name)
+ return True
def unload_module(module_name):
- utils.system('/sbin/rmmod ' + module_name)
+ utils.system('/sbin/rmmod ' + module_name)
def module_is_loaded(module_name):
- module_name = module_name.replace('-', '_')
- modules = utils.system_output('/sbin/lsmod').splitlines()
- for module in modules:
- if module.startswith(module_name) and module[len(module_name)] == ' ':
- return True
- return False
+ module_name = module_name.replace('-', '_')
+ modules = utils.system_output('/sbin/lsmod').splitlines()
+ for module in modules:
+ if module.startswith(module_name) and module[len(module_name)] == ' ':
+ return True
+ return False
def get_loaded_modules():
- lsmod_output = utils.system_output('/sbin/lsmod').splitlines()[1:]
- return [line.split(None, 1)[0] for line in lsmod_output]
+ lsmod_output = utils.system_output('/sbin/lsmod').splitlines()[1:]
+ return [line.split(None, 1)[0] for line in lsmod_output]
def get_huge_page_size():
- output = utils.system_output('grep Hugepagesize /proc/meminfo')
- return int(output.split()[1]) # Assumes units always in kB. :(
+ output = utils.system_output('grep Hugepagesize /proc/meminfo')
+ return int(output.split()[1]) # Assumes units always in kB. :(
def get_num_huge_pages():
- raw_hugepages = utils.system_output('/sbin/sysctl vm.nr_hugepages')
- return int(raw_hugepages.split()[2])
+ raw_hugepages = utils.system_output('/sbin/sysctl vm.nr_hugepages')
+ return int(raw_hugepages.split()[2])
def set_num_huge_pages(num):
- utils.system('/sbin/sysctl vm.nr_hugepages=%d' % num)
+ utils.system('/sbin/sysctl vm.nr_hugepages=%d' % num)
def get_system_nodes():
- nodes = os.listdir('/sys/devices/system/node')
- nodes.sort()
- return nodes
+ nodes = os.listdir('/sys/devices/system/node')
+ nodes.sort()
+ return nodes
def get_cpu_vendor():
- cpuinfo = open('/proc/cpuinfo').read()
- vendors = re.findall(r'(?m)^vendor_id\s*:\s*(\S+)\s*$', cpuinfo)
- for i in xrange(1, len(vendors)):
- if vendors[i] != vendors[0]:
- raise error.TestError('multiple cpu vendors found: ' + str(vendors))
- return vendors[0]
+ cpuinfo = open('/proc/cpuinfo').read()
+ vendors = re.findall(r'(?m)^vendor_id\s*:\s*(\S+)\s*$', cpuinfo)
+ for i in xrange(1, len(vendors)):
+ if vendors[i] != vendors[0]:
+ raise error.TestError('multiple cpu vendors found: ' + str(vendors))
+ return vendors[0]
def probe_cpus():
- """
- This routine returns a list of cpu devices found under /sys/devices/system/cpu.
- """
- output = utils.system_output(
- 'find /sys/devices/system/cpu/ -maxdepth 1 -type d -name cpu*')
- return output.splitlines()
+ """
+ This routine returns a list of cpu devices found under /sys/devices/system/cpu.
+ """
+ output = utils.system_output(
+ 'find /sys/devices/system/cpu/ -maxdepth 1 -type d -name cpu*')
+ return output.splitlines()
def ping_default_gateway():
- """Ping the default gateway."""
-
- network = open('/etc/sysconfig/network')
- m = re.search('GATEWAY=(\S+)', network.read())
+ """Ping the default gateway."""
- if m:
- gw = m.group(1)
- cmd = 'ping %s -c 5 > /dev/null' % gw
- return utils.system(cmd, ignore_status=True)
-
- raise error.TestError('Unable to find default gateway')
+ network = open('/etc/sysconfig/network')
+ m = re.search('GATEWAY=(\S+)', network.read())
+
+ if m:
+ gw = m.group(1)
+ cmd = 'ping %s -c 5 > /dev/null' % gw
+ return utils.system(cmd, ignore_status=True)
+
+ raise error.TestError('Unable to find default gateway')
try:
- from site_utils import *
+ from site_utils import *
except ImportError:
- pass
+ pass
diff --git a/client/bin/boottool.py b/client/bin/boottool.py
index 43f7f39..7c10618 100644
--- a/client/bin/boottool.py
+++ b/client/bin/boottool.py
@@ -2,125 +2,124 @@
from autotest_lib.client.common_lib import utils, error
class boottool:
- def __init__(self, boottool_exec=None):
- #variable to indicate if in mode to write entries for Xen
- self.xen_mode = False
+ def __init__(self, boottool_exec=None):
+ #variable to indicate if in mode to write entries for Xen
+ self.xen_mode = False
- if boottool_exec:
- self.boottool_exec = boottool_exec
- else:
- autodir = os.environ['AUTODIR']
- self.boottool_exec = autodir + '/tools/boottool'
+ if boottool_exec:
+ self.boottool_exec = boottool_exec
+ else:
+ autodir = os.environ['AUTODIR']
+ self.boottool_exec = autodir + '/tools/boottool'
- if not self.boottool_exec:
- raise error.AutotestError('Failed to set boottool_exec')
+ if not self.boottool_exec:
+ raise error.AutotestError('Failed to set boottool_exec')
- def run_boottool(self, params):
- return utils.system_output('%s %s' % (self.boottool_exec, params))
+ def run_boottool(self, params):
+ return utils.system_output('%s %s' % (self.boottool_exec, params))
- def bootloader(self):
- return self.run_boottool('--bootloader-probe')
+ def bootloader(self):
+ return self.run_boottool('--bootloader-probe')
- def architecture(self):
- return self.run_boottool('--arch-probe')
+ def architecture(self):
+ return self.run_boottool('--arch-probe')
- def list_titles(self):
- print self.run_boottool('--info all | grep title')
+ def list_titles(self):
+ print self.run_boottool('--info all | grep title')
- def print_entry(self, index):
- print self.run_boottool('--info=%s' % index)
+ def print_entry(self, index):
+ print self.run_boottool('--info=%s' % index)
- def get_default(self):
- self.run_boottool('--default')
+ def get_default(self):
+ self.run_boottool('--default')
- def set_default(self, index):
- print self.run_boottool('--set-default=%s' % index)
+ def set_default(self, index):
+ print self.run_boottool('--set-default=%s' % index)
- def enable_xen_mode(self):
- self.xen_mode = True
+ def enable_xen_mode(self):
+ self.xen_mode = True
- def disable_xen_mode(self):
- self.xen_mode = False
+ def disable_xen_mode(self):
+ self.xen_mode = False
- def get_xen_mode(self):
- return self.xen_mode
+ def get_xen_mode(self):
+ return self.xen_mode
- # 'kernel' can be an position number or a title
- def add_args(self, kernel, args):
- parameters = '--update-kernel=%s --args="%s"' % (kernel, args)
+ # 'kernel' can be an position number or a title
+ def add_args(self, kernel, args):
+ parameters = '--update-kernel=%s --args="%s"' % (kernel, args)
- #add parameter if this is a Xen entry
- if self.xen_mode:
- parameters += ' --xen'
+ #add parameter if this is a Xen entry
+ if self.xen_mode:
+ parameters += ' --xen'
- print self.run_boottool(parameters)
+ print self.run_boottool(parameters)
- def add_xen_hypervisor_args(self, kernel, args):
- self.run_boottool('--xen --update-xenhyper=%s --xha="%s"') %(kernel, args)
-
-
- def remove_args(self, kernel, args):
- parameters = '--update-kernel=%s --remove-args=%s' % (kernel, args)
-
- #add parameter if this is a Xen entry
- if self.xen_mode:
- parameters += ' --xen'
-
- print self.run_boottool(parameters)
+ def add_xen_hypervisor_args(self, kernel, args):
+ self.run_boottool('--xen --update-xenhyper=%s --xha="%s"') %(kernel, args)
- def remove_xen_hypervisor_args(self, kernel, args):
- self.run_boottool('--xen --update-xenhyper=%s --remove-args="%s"') \
- % (kernel, args)
+ def remove_args(self, kernel, args):
+ parameters = '--update-kernel=%s --remove-args=%s' % (kernel, args)
+
+ #add parameter if this is a Xen entry
+ if self.xen_mode:
+ parameters += ' --xen'
+
+ print self.run_boottool(parameters)
- def add_kernel(self, path, title='autotest', initrd='', xen_hypervisor='', args=None, root=None, position='end'):
- parameters = '--add-kernel=%s --title=%s' % (path, title)
-
- # add an initrd now or forever hold your peace
- if initrd:
- parameters += ' --initrd=%s' % initrd
-
- # add parameter if this is a Xen entry
- if self.xen_mode:
- parameters += ' --xen'
- if xen_hypervisor:
- parameters += ' --xenhyper=%s' % xen_hypervisor
-
- if args:
- parameters += ' --args="%s"' % args
- if root:
- parameters += ' --root="%s"' % root
- if position:
- parameters += ' --position="%s"' % position
-
- print self.run_boottool(parameters)
+ def remove_xen_hypervisor_args(self, kernel, args):
+ self.run_boottool('--xen --update-xenhyper=%s --remove-args="%s"') \
+ % (kernel, args)
- def remove_kernel(self, kernel):
- print self.run_boottool('--remove-kernel=%s' % kernel)
+ def add_kernel(self, path, title='autotest', initrd='', xen_hypervisor='', args=None, root=None, position='end'):
+ parameters = '--add-kernel=%s --title=%s' % (path, title)
+
+ # add an initrd now or forever hold your peace
+ if initrd:
+ parameters += ' --initrd=%s' % initrd
+
+ # add parameter if this is a Xen entry
+ if self.xen_mode:
+ parameters += ' --xen'
+ if xen_hypervisor:
+ parameters += ' --xenhyper=%s' % xen_hypervisor
+
+ if args:
+ parameters += ' --args="%s"' % args
+ if root:
+ parameters += ' --root="%s"' % root
+ if position:
+ parameters += ' --position="%s"' % position
+
+ print self.run_boottool(parameters)
- def boot_once(self, title):
- print self.run_boottool('--boot-once --title=%s' % title)
+ def remove_kernel(self, kernel):
+ print self.run_boottool('--remove-kernel=%s' % kernel)
- def info(self, index):
- return self.run_boottool('--info=%s' % index)
+ def boot_once(self, title):
+ print self.run_boottool('--boot-once --title=%s' % title)
+
+
+ def info(self, index):
+ return self.run_boottool('--info=%s' % index)
# TODO: backup()
# TODO: set_timeout()
-
diff --git a/client/bin/common.py b/client/bin/common.py
index 74ed759..6881386 100644
--- a/client/bin/common.py
+++ b/client/bin/common.py
@@ -5,4 +5,4 @@
import setup_modules
sys.path.pop(0)
setup_modules.setup(base_path=client_dir,
- root_module_name="autotest_lib.client")
+ root_module_name="autotest_lib.client")
diff --git a/client/bin/config.py b/client/bin/config.py
index 59b8279..477bece 100644
--- a/client/bin/config.py
+++ b/client/bin/config.py
@@ -8,7 +8,7 @@
Please no StudlyCaps.
For example:
- boot.default_args
+ boot.default_args
"""
__author__ = """Copyright Andy Whitcroft 2006"""
@@ -16,33 +16,33 @@
import os
class config:
- """The BASIC job configuration
+ """The BASIC job configuration
- Properties:
- job
- The job object for this job
- config
- The job configuration dictionary
- """
+ Properties:
+ job
+ The job object for this job
+ config
+ The job configuration dictionary
+ """
- def __init__(self, job):
- """
- job
- The job object for this job
- """
- self.job = job
- self.config = {}
+ def __init__(self, job):
+ """
+ job
+ The job object for this job
+ """
+ self.job = job
+ self.config = {}
- def set(self, name, value):
- if name == "proxy":
- os.environ['http_proxy'] = value
- os.environ['ftp_proxy'] = value
+ def set(self, name, value):
+ if name == "proxy":
+ os.environ['http_proxy'] = value
+ os.environ['ftp_proxy'] = value
- self.config[name] = value
+ self.config[name] = value
- def get(self, name):
- if name in self.config:
- return self.config[name]
- else:
- return None
+ def get(self, name):
+ if name in self.config:
+ return self.config[name]
+ else:
+ return None
diff --git a/client/bin/cpuset.py b/client/bin/cpuset.py
index a870e76..fe583ac 100644
--- a/client/bin/cpuset.py
+++ b/client/bin/cpuset.py
@@ -9,265 +9,265 @@
# Convert '1-3,7,9-12' to [1,2,3,7,9,10,11,12]
def rangelist_to_list(rangelist):
- result = []
- if not rangelist:
- return result
- for x in rangelist.split(','):
- if re.match(r'^(\d+)$', x):
- result.append(int(x))
- continue
- m = re.match(r'^(\d+)-(\d+)$', x)
- if m:
- start = int(m.group(1))
- end = int(m.group(2))
- result += range(start, end+1)
- continue
- msg = 'Cannot understand data input: %s %s' % (x, rangelist)
- raise ValueError(msg)
- return result
+ result = []
+ if not rangelist:
+ return result
+ for x in rangelist.split(','):
+ if re.match(r'^(\d+)$', x):
+ result.append(int(x))
+ continue
+ m = re.match(r'^(\d+)-(\d+)$', x)
+ if m:
+ start = int(m.group(1))
+ end = int(m.group(2))
+ result += range(start, end+1)
+ continue
+ msg = 'Cannot understand data input: %s %s' % (x, rangelist)
+ raise ValueError(msg)
+ return result
def rounded_memtotal():
- # Get total of all physical mem, in Kbytes
- usable_Kbytes = autotest_utils.memtotal()
- # usable_Kbytes is system's usable DRAM in Kbytes,
- # as reported by memtotal() from device /proc/meminfo memtotal
- # after Linux deducts 1.5% to 5.1% for system table overhead
- # Undo the unknown actual deduction by rounding up
- # to next small multiple of a big power-of-two
- # eg 12GB - 5.1% gets rounded back up to 12GB
- mindeduct = 0.015 # 1.5 percent
- maxdeduct = 0.055 # 5.5 percent
- # deduction range 1.5% .. 5.5% supports physical mem sizes
- # 6GB .. 12GB in steps of .5GB
- # 12GB .. 24GB in steps of 1 GB
- # 24GB .. 48GB in steps of 2 GB ...
- # Finer granularity in physical mem sizes would require
- # tighter spread between min and max possible deductions
+ # Get total of all physical mem, in Kbytes
+ usable_Kbytes = autotest_utils.memtotal()
+ # usable_Kbytes is system's usable DRAM in Kbytes,
+ # as reported by memtotal() from device /proc/meminfo memtotal
+ # after Linux deducts 1.5% to 5.1% for system table overhead
+ # Undo the unknown actual deduction by rounding up
+ # to next small multiple of a big power-of-two
+ # eg 12GB - 5.1% gets rounded back up to 12GB
+ mindeduct = 0.015 # 1.5 percent
+ maxdeduct = 0.055 # 5.5 percent
+ # deduction range 1.5% .. 5.5% supports physical mem sizes
+ # 6GB .. 12GB in steps of .5GB
+ # 12GB .. 24GB in steps of 1 GB
+ # 24GB .. 48GB in steps of 2 GB ...
+ # Finer granularity in physical mem sizes would require
+ # tighter spread between min and max possible deductions
- # increase mem size by at least min deduction, without rounding
- min_Kbytes = int(usable_Kbytes / (1.0 - mindeduct))
- # increase mem size further by 2**n rounding, by 0..roundKb or more
- round_Kbytes = int(usable_Kbytes / (1.0 - maxdeduct)) - min_Kbytes
- # find least binary roundup 2**n that covers worst-cast roundKb
- mod2n = 1 << int(math.ceil(math.log(round_Kbytes, 2)))
- # have round_Kbytes <= mod2n < round_Kbytes*2
- # round min_Kbytes up to next multiple of mod2n
- phys_Kbytes = min_Kbytes + mod2n - 1
- phys_Kbytes = phys_Kbytes - (phys_Kbytes % mod2n) # clear low bits
- return phys_Kbytes
+ # increase mem size by at least min deduction, without rounding
+ min_Kbytes = int(usable_Kbytes / (1.0 - mindeduct))
+ # increase mem size further by 2**n rounding, by 0..roundKb or more
+ round_Kbytes = int(usable_Kbytes / (1.0 - maxdeduct)) - min_Kbytes
+ # find least binary roundup 2**n that covers worst-cast roundKb
+ mod2n = 1 << int(math.ceil(math.log(round_Kbytes, 2)))
+ # have round_Kbytes <= mod2n < round_Kbytes*2
+ # round min_Kbytes up to next multiple of mod2n
+ phys_Kbytes = min_Kbytes + mod2n - 1
+ phys_Kbytes = phys_Kbytes - (phys_Kbytes % mod2n) # clear low bits
+ return phys_Kbytes
def my_container_name():
- # Get current process's inherited or self-built container name
- # within /dev/cpuset. Is '/' for root container, '/sys', etc.
- return utils.read_one_line('/proc/%i/cpuset' % os.getpid())
+ # Get current process's inherited or self-built container name
+ # within /dev/cpuset. Is '/' for root container, '/sys', etc.
+ return utils.read_one_line('/proc/%i/cpuset' % os.getpid())
def get_mem_nodes(container_full_name):
- file_name = os.path.join(container_full_name, "mems")
- if os.path.exists(file_name):
- return rangelist_to_list(utils.read_one_line(file_name))
- else:
- return []
+ file_name = os.path.join(container_full_name, "mems")
+ if os.path.exists(file_name):
+ return rangelist_to_list(utils.read_one_line(file_name))
+ else:
+ return []
def available_exclusive_mem_nodes(parent_container):
- # Get list of numa memory nodes of parent container which could
- # be allocated exclusively to new child containers.
- # This excludes any nodes now allocated (exclusively or not)
- # to existing children.
- available = set(get_mem_nodes(parent_container))
- for child_container in glob.glob('%s/*/mems' % parent_container):
- child_container = os.path.dirname(child_container)
- busy = set(get_mem_nodes(child_container))
- available -= busy
- return list(available)
+ # Get list of numa memory nodes of parent container which could
+ # be allocated exclusively to new child containers.
+ # This excludes any nodes now allocated (exclusively or not)
+ # to existing children.
+ available = set(get_mem_nodes(parent_container))
+ for child_container in glob.glob('%s/*/mems' % parent_container):
+ child_container = os.path.dirname(child_container)
+ busy = set(get_mem_nodes(child_container))
+ available -= busy
+ return list(available)
def my_mem_nodes():
- # Get list of numa memory nodes owned by current process's container.
- return get_mem_nodes('/dev/cpuset%s' % my_container_name())
+ # Get list of numa memory nodes owned by current process's container.
+ return get_mem_nodes('/dev/cpuset%s' % my_container_name())
def my_available_exclusive_mem_nodes():
- # Get list of numa memory nodes owned by current process's
- # container, which could be allocated exclusively to new child
- # containers. This excludes any nodes now allocated
- # (exclusively or not) to existing children.
- return available_exclusive_mem_nodes('/dev/cpuset%s' % my_container_name())
+ # Get list of numa memory nodes owned by current process's
+ # container, which could be allocated exclusively to new child
+ # containers. This excludes any nodes now allocated
+ # (exclusively or not) to existing children.
+ return available_exclusive_mem_nodes('/dev/cpuset%s' % my_container_name())
def mbytes_per_mem_node():
- # Get mbyte size of each numa mem node, as float
- # Replaces autotest_utils.node_size().
- # Based on guessed total physical mem size, not on kernel's
- # lesser 'available memory' after various system tables.
- # Can be non-integer when kernel sets up 15 nodes instead of 16.
- return rounded_memtotal() / (len(autotest_utils.numa_nodes()) * 1024.0)
+ # Get mbyte size of each numa mem node, as float
+ # Replaces autotest_utils.node_size().
+ # Based on guessed total physical mem size, not on kernel's
+ # lesser 'available memory' after various system tables.
+ # Can be non-integer when kernel sets up 15 nodes instead of 16.
+ return rounded_memtotal() / (len(autotest_utils.numa_nodes()) * 1024.0)
def get_cpus(container_full_name):
- file_name = os.path.join(container_full_name, "cpus")
- if os.path.exists(file_name):
- return rangelist_to_list(utils.read_one_line(file_name))
- else:
- return []
+ file_name = os.path.join(container_full_name, "cpus")
+ if os.path.exists(file_name):
+ return rangelist_to_list(utils.read_one_line(file_name))
+ else:
+ return []
def my_cpus():
- # Get list of cpu cores owned by current process's container.
- return get_cpus('/dev/cpuset%s' % my_container_name())
+ # Get list of cpu cores owned by current process's container.
+ return get_cpus('/dev/cpuset%s' % my_container_name())
def get_tasks(setname):
- return [x.rstrip() for x in open(setname+'/tasks').readlines()]
+ return [x.rstrip() for x in open(setname+'/tasks').readlines()]
def print_one_cpuset(name):
- dir = os.path.join('/dev/cpuset', name)
- cpus = utils.read_one_line(dir + '/cpus')
- mems = utils.read_one_line(dir + '/mems')
- node_size_ = int(mbytes_per_mem_node()) << 20
- memtotal = node_size_ * len(rangelist_to_list(mems))
- tasks = ','.join(get_tasks(dir))
- print "cpuset %s: size %s; tasks %s; cpus %s; mems %s" % \
- (name, autotest_utils.human_format(memtotal), tasks, cpus, mems)
+ dir = os.path.join('/dev/cpuset', name)
+ cpus = utils.read_one_line(dir + '/cpus')
+ mems = utils.read_one_line(dir + '/mems')
+ node_size_ = int(mbytes_per_mem_node()) << 20
+ memtotal = node_size_ * len(rangelist_to_list(mems))
+ tasks = ','.join(get_tasks(dir))
+ print "cpuset %s: size %s; tasks %s; cpus %s; mems %s" % \
+ (name, autotest_utils.human_format(memtotal), tasks, cpus, mems)
def print_all_cpusets():
- for cpuset in glob.glob('/dev/cpuset/*'):
- print_one_cpuset(re.sub(r'.*/', '', cpuset))
+ for cpuset in glob.glob('/dev/cpuset/*'):
+ print_one_cpuset(re.sub(r'.*/', '', cpuset))
def release_dead_containers(parent=super_root):
- # Delete temp subcontainers nested within parent container
- # that are now dead (having no tasks and no sub-containers)
- # and recover their cpu and mem resources.
- # Must not call when a parallel task may be allocating containers!
- # Limit to test* names to preserve permanent containers.
- for child in glob.glob('%s/test*' % parent):
- print 'releasing dead container', child
- release_dead_containers(child) # bottom-up tree walk
- # rmdir has no effect when container still
- # has tasks or sub-containers
- os.rmdir(child)
+ # Delete temp subcontainers nested within parent container
+ # that are now dead (having no tasks and no sub-containers)
+ # and recover their cpu and mem resources.
+ # Must not call when a parallel task may be allocating containers!
+ # Limit to test* names to preserve permanent containers.
+ for child in glob.glob('%s/test*' % parent):
+ print 'releasing dead container', child
+ release_dead_containers(child) # bottom-up tree walk
+ # rmdir has no effect when container still
+ # has tasks or sub-containers
+ os.rmdir(child)
class cpuset:
- def display(self):
- print_one_cpuset(os.path.join(self.root, self.name))
+ def display(self):
+ print_one_cpuset(os.path.join(self.root, self.name))
- def release(self):
- print "releasing ", self.cpudir
- parent_t = os.path.join(self.root, 'tasks')
- # Transfer survivors (and self) to parent
- for task in get_tasks(self.cpudir):
- utils.write_one_line(parent_t, task)
- os.rmdir(self.cpudir)
- if os.path.exists(self.cpudir):
- raise error.AutotestError('Could not delete container '
- + self.cpudir)
+ def release(self):
+ print "releasing ", self.cpudir
+ parent_t = os.path.join(self.root, 'tasks')
+ # Transfer survivors (and self) to parent
+ for task in get_tasks(self.cpudir):
+ utils.write_one_line(parent_t, task)
+ os.rmdir(self.cpudir)
+ if os.path.exists(self.cpudir):
+ raise error.AutotestError('Could not delete container '
+ + self.cpudir)
- def __init__(self, name, job_size=None, job_pid=None, cpus=None,
- root=None):
- """\
- Create a cpuset container and move job_pid into it
- Allocate the list "cpus" of cpus to that container
+ def __init__(self, name, job_size=None, job_pid=None, cpus=None,
+ root=None):
+ """\
+ Create a cpuset container and move job_pid into it
+ Allocate the list "cpus" of cpus to that container
- name = arbitrary string tag
- job_size = reqested memory for job in megabytes
- job_pid = pid of job we're putting into the container
- cpu = list of cpu indicies to associate with the cpuset
- root = the cpuset to create this new set in
- """
- if not os.path.exists(os.path.join(super_root, "cpus")):
- raise error.AutotestError('Root container /dev/cpuset '
- 'is empty; please reboot')
+ name = arbitrary string tag
+ job_size = reqested memory for job in megabytes
+ job_pid = pid of job we're putting into the container
+ cpu = list of cpu indicies to associate with the cpuset
+ root = the cpuset to create this new set in
+ """
+ if not os.path.exists(os.path.join(super_root, "cpus")):
+ raise error.AutotestError('Root container /dev/cpuset '
+ 'is empty; please reboot')
- self.name = name
+ self.name = name
- if root == None:
- # default to nested in process's current container
- root = my_container_name()[1:]
- self.root = os.path.join(super_root, root)
- if not os.path.exists(self.root):
- raise error.AutotestError(('Parent container %s'
- ' does not exist')
- % self.root)
+ if root == None:
+ # default to nested in process's current container
+ root = my_container_name()[1:]
+ self.root = os.path.join(super_root, root)
+ if not os.path.exists(self.root):
+ raise error.AutotestError(('Parent container %s'
+ ' does not exist')
+ % self.root)
- if job_size == None:
- # default to biggest container we can make under root
- job_size = int( mbytes_per_mem_node() *
- len(available_exclusive_mem_nodes(self.root)) )
- if not job_size:
- raise error.AutotestError('Creating container '
- 'with no mem')
- self.memory = job_size
+ if job_size == None:
+ # default to biggest container we can make under root
+ job_size = int( mbytes_per_mem_node() *
+ len(available_exclusive_mem_nodes(self.root)) )
+ if not job_size:
+ raise error.AutotestError('Creating container '
+ 'with no mem')
+ self.memory = job_size
- if cpus == None:
- # default to biggest container we can make under root
- cpus = get_cpus(self.root)
- if not cpus:
- raise error.AutotestError('Creating container '
- 'with no cpus')
- self.cpus = cpus
+ if cpus == None:
+ # default to biggest container we can make under root
+ cpus = get_cpus(self.root)
+ if not cpus:
+ raise error.AutotestError('Creating container '
+ 'with no cpus')
+ self.cpus = cpus
- # default to the current pid
- if not job_pid:
- job_pid = os.getpid()
+ # default to the current pid
+ if not job_pid:
+ job_pid = os.getpid()
- print "cpuset(name=%s, root=%s, job_size=%d, pid=%d)" % \
- (name, root, job_size, job_pid)
+ print "cpuset(name=%s, root=%s, job_size=%d, pid=%d)" % \
+ (name, root, job_size, job_pid)
- self.cpudir = os.path.join(self.root, name)
- if os.path.exists(self.cpudir):
- self.release() # destructively replace old
+ self.cpudir = os.path.join(self.root, name)
+ if os.path.exists(self.cpudir):
+ self.release() # destructively replace old
- nodes_needed = int(math.ceil( float(job_size) /
- math.ceil(mbytes_per_mem_node()) ))
+ nodes_needed = int(math.ceil( float(job_size) /
+ math.ceil(mbytes_per_mem_node()) ))
- if nodes_needed > len(get_mem_nodes(self.root)):
- raise error.AutotestError("Container's memory "
- "is bigger than parent's")
+ if nodes_needed > len(get_mem_nodes(self.root)):
+ raise error.AutotestError("Container's memory "
+ "is bigger than parent's")
- while True:
- # Pick specific free mem nodes for this cpuset
- mems = available_exclusive_mem_nodes(self.root)
- if len(mems) < nodes_needed:
- raise error.AutotestError(('Existing container'
- ' hold %d mem nodes'
- ' needed by new'
- 'container')
- % (nodes_needed
- - len(mems)))
- mems = mems[-nodes_needed:]
- mems_spec = ','.join(['%d' % x for x in mems])
- os.mkdir(self.cpudir)
- utils.write_one_line(os.path.join(self.cpudir,
- 'mem_exclusive'), '1')
- utils.write_one_line(os.path.join(self.cpudir,
- 'mems'),
- mems_spec)
- # Above sends err msg to client.log.0, but no exception,
- # if mems_spec contained any now-taken nodes
- # Confirm that siblings didn't grab our chosen mems:
- nodes_gotten = len(get_mem_nodes(self.cpudir))
- if nodes_gotten >= nodes_needed:
- break # success
- print "cpuset %s lost race for nodes" % name, mems_spec
- # Return any mem we did get, and try again
- os.rmdir(self.cpudir)
+ while True:
+ # Pick specific free mem nodes for this cpuset
+ mems = available_exclusive_mem_nodes(self.root)
+ if len(mems) < nodes_needed:
+ raise error.AutotestError(('Existing container'
+ ' hold %d mem nodes'
+ ' needed by new'
+ 'container')
+ % (nodes_needed
+ - len(mems)))
+ mems = mems[-nodes_needed:]
+ mems_spec = ','.join(['%d' % x for x in mems])
+ os.mkdir(self.cpudir)
+ utils.write_one_line(os.path.join(self.cpudir,
+ 'mem_exclusive'), '1')
+ utils.write_one_line(os.path.join(self.cpudir,
+ 'mems'),
+ mems_spec)
+ # Above sends err msg to client.log.0, but no exception,
+ # if mems_spec contained any now-taken nodes
+ # Confirm that siblings didn't grab our chosen mems:
+ nodes_gotten = len(get_mem_nodes(self.cpudir))
+ if nodes_gotten >= nodes_needed:
+ break # success
+ print "cpuset %s lost race for nodes" % name, mems_spec
+ # Return any mem we did get, and try again
+ os.rmdir(self.cpudir)
- # add specified cpu cores and own task pid to container:
- cpu_spec = ','.join(['%d' % x for x in cpus])
- utils.write_one_line(os.path.join(self.cpudir,
- 'cpus'),
- cpu_spec)
- utils.write_one_line(os.path.join(self.cpudir,
- 'tasks'),
- "%d" % job_pid)
- self.display()
+ # add specified cpu cores and own task pid to container:
+ cpu_spec = ','.join(['%d' % x for x in cpus])
+ utils.write_one_line(os.path.join(self.cpudir,
+ 'cpus'),
+ cpu_spec)
+ utils.write_one_line(os.path.join(self.cpudir,
+ 'tasks'),
+ "%d" % job_pid)
+ self.display()
diff --git a/client/bin/fd_stack.py b/client/bin/fd_stack.py
index c377cf0..e0f3f4c 100755
--- a/client/bin/fd_stack.py
+++ b/client/bin/fd_stack.py
@@ -3,137 +3,137 @@
import sys, os
class fd_stack:
- """a stack of fd redirects
+ """a stack of fd redirects
- Redirects cause existing fd's to be pushed on the stack; restore()
- causes the current set of redirects to be popped, restoring the previous
- filehandle destinations.
+ Redirects cause existing fd's to be pushed on the stack; restore()
+ causes the current set of redirects to be popped, restoring the previous
+ filehandle destinations.
- Note that we need to redirect both the sys.stdout type descriptor
- (which print, etc use) and the low level OS numbered descriptor
- which os.system() etc use.
- """
+ Note that we need to redirect both the sys.stdout type descriptor
+ (which print, etc use) and the low level OS numbered descriptor
+ which os.system() etc use.
+ """
- def __init__(self, fd, filehandle):
- self.fd = fd # eg 1
- self.filehandle = filehandle # eg sys.stdout
- self.stack = [(fd, filehandle)]
+ def __init__(self, fd, filehandle):
+ self.fd = fd # eg 1
+ self.filehandle = filehandle # eg sys.stdout
+ self.stack = [(fd, filehandle)]
- def update_handle(self, new):
- if (self.filehandle == sys.stdout):
- sys.stdout = new
- if (self.filehandle == sys.stderr):
- sys.stderr = new
- self.filehandle = new
+ def update_handle(self, new):
+ if (self.filehandle == sys.stdout):
+ sys.stdout = new
+ if (self.filehandle == sys.stderr):
+ sys.stderr = new
+ self.filehandle = new
- def redirect(self, filename):
- """Redirect output to the specified file
+ def redirect(self, filename):
+ """Redirect output to the specified file
- Overwrites the previous contents, if any.
- """
- self.filehandle.flush()
- fdcopy = os.dup(self.fd)
- self.stack.append( (fdcopy, self.filehandle, 0) )
- # self.filehandle = file(filename, 'w')
- if (os.path.isfile(filename)):
- newfd = os.open(filename, os.O_WRONLY)
- else:
- newfd = os.open(filename, os.O_WRONLY | os.O_CREAT)
- os.dup2(newfd, self.fd)
- os.close(newfd)
- self.update_handle(os.fdopen(self.fd, 'w'))
+ Overwrites the previous contents, if any.
+ """
+ self.filehandle.flush()
+ fdcopy = os.dup(self.fd)
+ self.stack.append( (fdcopy, self.filehandle, 0) )
+ # self.filehandle = file(filename, 'w')
+ if (os.path.isfile(filename)):
+ newfd = os.open(filename, os.O_WRONLY)
+ else:
+ newfd = os.open(filename, os.O_WRONLY | os.O_CREAT)
+ os.dup2(newfd, self.fd)
+ os.close(newfd)
+ self.update_handle(os.fdopen(self.fd, 'w'))
- def tee_redirect(self, filename):
- """Tee output to the specified file
+ def tee_redirect(self, filename):
+ """Tee output to the specified file
- Overwrites the previous contents, if any.
- """
- self.filehandle.flush()
- #print_to_tty("tee_redirect to " + filename)
- #where_art_thy_filehandles()
- fdcopy = os.dup(self.fd)
- r, w = os.pipe()
- pid = os.fork()
- if pid: # parent
- os.close(r)
- os.dup2(w, self.fd)
- os.close(w)
- self.stack.append( (fdcopy, self.filehandle, pid) )
- self.update_handle(os.fdopen(self.fd, 'w', 0))
- #where_art_thy_filehandles()
- #print_to_tty("done tee_redirect to " + filename)
- else: # child
- os.close(w)
- os.dup2(r, 0)
- os.dup2(fdcopy, 1)
- os.close(r)
- os.close(fdcopy)
- os.execlp('tee', 'tee', '-a', filename)
+ Overwrites the previous contents, if any.
+ """
+ self.filehandle.flush()
+ #print_to_tty("tee_redirect to " + filename)
+ #where_art_thy_filehandles()
+ fdcopy = os.dup(self.fd)
+ r, w = os.pipe()
+ pid = os.fork()
+ if pid: # parent
+ os.close(r)
+ os.dup2(w, self.fd)
+ os.close(w)
+ self.stack.append( (fdcopy, self.filehandle, pid) )
+ self.update_handle(os.fdopen(self.fd, 'w', 0))
+ #where_art_thy_filehandles()
+ #print_to_tty("done tee_redirect to " + filename)
+ else: # child
+ os.close(w)
+ os.dup2(r, 0)
+ os.dup2(fdcopy, 1)
+ os.close(r)
+ os.close(fdcopy)
+ os.execlp('tee', 'tee', '-a', filename)
-
- def restore(self):
- """unredirect one level"""
- self.filehandle.flush()
- # print_to_tty("ENTERING RESTORE %d" % self.fd)
- # where_art_thy_filehandles()
- (old_fd, old_filehandle, pid) = self.stack.pop()
- # print_to_tty("old_fd %d" % old_fd)
- # print_to_tty("self.fd %d" % self.fd)
- self.filehandle.close() # seems to close old_fd as well.
- if pid:
- os.waitpid(pid, 0)
- # where_art_thy_filehandles()
- os.dup2(old_fd, self.fd)
- # print_to_tty("CLOSING FD %d" % old_fd)
- os.close(old_fd)
- # where_art_thy_filehandles()
- self.update_handle(old_filehandle)
- # where_art_thy_filehandles()
- # print_to_tty("EXIT RESTORE %d" % self.fd)
+
+ def restore(self):
+ """unredirect one level"""
+ self.filehandle.flush()
+ # print_to_tty("ENTERING RESTORE %d" % self.fd)
+ # where_art_thy_filehandles()
+ (old_fd, old_filehandle, pid) = self.stack.pop()
+ # print_to_tty("old_fd %d" % old_fd)
+ # print_to_tty("self.fd %d" % self.fd)
+ self.filehandle.close() # seems to close old_fd as well.
+ if pid:
+ os.waitpid(pid, 0)
+ # where_art_thy_filehandles()
+ os.dup2(old_fd, self.fd)
+ # print_to_tty("CLOSING FD %d" % old_fd)
+ os.close(old_fd)
+ # where_art_thy_filehandles()
+ self.update_handle(old_filehandle)
+ # where_art_thy_filehandles()
+ # print_to_tty("EXIT RESTORE %d" % self.fd)
def tee_output_logdir(fn):
- """\
- Method decorator for a class to tee the output to the objects log_dir.
- """
- def tee_logdir_wrapper(self, *args, **dargs):
- self.job.stdout.tee_redirect(os.path.join(self.log_dir, 'stdout'))
- self.job.stderr.tee_redirect(os.path.join(self.log_dir, 'stderr'))
- try:
- result = fn(self, *args, **dargs)
- finally:
- self.job.stderr.restore()
- self.job.stdout.restore()
- return result
- return tee_logdir_wrapper
+ """\
+ Method decorator for a class to tee the output to the objects log_dir.
+ """
+ def tee_logdir_wrapper(self, *args, **dargs):
+ self.job.stdout.tee_redirect(os.path.join(self.log_dir, 'stdout'))
+ self.job.stderr.tee_redirect(os.path.join(self.log_dir, 'stderr'))
+ try:
+ result = fn(self, *args, **dargs)
+ finally:
+ self.job.stderr.restore()
+ self.job.stdout.restore()
+ return result
+ return tee_logdir_wrapper
def __mark(filename, msg):
- file = open(filename, 'a')
- file.write(msg)
- file.close()
+ file = open(filename, 'a')
+ file.write(msg)
+ file.close()
def tee_output_logdir_mark(fn):
- def tee_logdir_mark_wrapper(self, *args, **dargs):
- mark = self.__class__.__name__ + "." + fn.__name__
- outfile = os.path.join(self.log_dir, 'stdout')
- errfile = os.path.join(self.log_dir, 'stderr')
- __mark(outfile, "--- START " + mark + " ---\n")
- __mark(errfile, "--- START " + mark + " ---\n")
- self.job.stdout.tee_redirect(outfile)
- self.job.stderr.tee_redirect(errfile)
- try:
- result = fn(self, *args, **dargs)
- finally:
- self.job.stderr.restore()
- self.job.stdout.restore()
- __mark(outfile, "--- END " + mark + " ---\n")
- __mark(errfile, "--- END " + mark + " ---\n")
+ def tee_logdir_mark_wrapper(self, *args, **dargs):
+ mark = self.__class__.__name__ + "." + fn.__name__
+ outfile = os.path.join(self.log_dir, 'stdout')
+ errfile = os.path.join(self.log_dir, 'stderr')
+ __mark(outfile, "--- START " + mark + " ---\n")
+ __mark(errfile, "--- START " + mark + " ---\n")
+ self.job.stdout.tee_redirect(outfile)
+ self.job.stderr.tee_redirect(errfile)
+ try:
+ result = fn(self, *args, **dargs)
+ finally:
+ self.job.stderr.restore()
+ self.job.stdout.restore()
+ __mark(outfile, "--- END " + mark + " ---\n")
+ __mark(errfile, "--- END " + mark + " ---\n")
- return result
+ return result
- tee_logdir_mark_wrapper.__name__ = fn.__name__
- return tee_logdir_mark_wrapper
+ tee_logdir_mark_wrapper.__name__ = fn.__name__
+ return tee_logdir_mark_wrapper
diff --git a/client/bin/filesystem.py b/client/bin/filesystem.py
index 2c24141..47c8c44 100755
--- a/client/bin/filesystem.py
+++ b/client/bin/filesystem.py
@@ -5,170 +5,170 @@
from autotest_lib.client.common_lib import error, utils
def list_mount_devices():
- devices = []
- # list mounted filesystems
- for line in utils.system_output('mount').splitlines():
- devices.append(line.split()[0])
- # list mounted swap devices
- for line in utils.system_output('swapon -s').splitlines():
- if line.startswith('/'): # skip header line
- devices.append(line.split()[0])
- return devices
+ devices = []
+ # list mounted filesystems
+ for line in utils.system_output('mount').splitlines():
+ devices.append(line.split()[0])
+ # list mounted swap devices
+ for line in utils.system_output('swapon -s').splitlines():
+ if line.startswith('/'): # skip header line
+ devices.append(line.split()[0])
+ return devices
def list_mount_points():
- mountpoints = []
- for line in utils.system_output('mount').splitlines():
- mountpoints.append(line.split()[2])
- return mountpoints
+ mountpoints = []
+ for line in utils.system_output('mount').splitlines():
+ mountpoints.append(line.split()[2])
+ return mountpoints
class filesystem:
- """
- Class for handling filesystems
- """
+ """
+ Class for handling filesystems
+ """
- def __init__(self, job, device, mountpoint, loop_size = 0):
- """
- device should be able to be a file as well
- which we mount as loopback
+ def __init__(self, job, device, mountpoint, loop_size = 0):
+ """
+ device should be able to be a file as well
+ which we mount as loopback
- device
- The device in question (eg "/dev/hda2")
- mountpoint
- Default mountpoint for the device.
- loop_size
- size of loopback device (in MB)
- """
+ device
+ The device in question (eg "/dev/hda2")
+ mountpoint
+ Default mountpoint for the device.
+ loop_size
+ size of loopback device (in MB)
+ """
- part = re.compile(r'^part(\d+)$')
- m = part.match(device)
- if m:
- number = int(m.groups()[0])
- partitions = job.config_get('filesystem.partitions')
- try:
- device = partitions[number]
- except:
- raise NameError("Partition '" + device + "' not available")
+ part = re.compile(r'^part(\d+)$')
+ m = part.match(device)
+ if m:
+ number = int(m.groups()[0])
+ partitions = job.config_get('filesystem.partitions')
+ try:
+ device = partitions[number]
+ except:
+ raise NameError("Partition '" + device + "' not available")
- self.device = device
- self.mountpoint = mountpoint
- self.job = job
- self.fstype = None
- self.loop = loop_size
- if self.loop:
- utils.system('dd if=/dev/zero of=%s bs=1M count=%d' % \
- (device, loop_size))
+ self.device = device
+ self.mountpoint = mountpoint
+ self.job = job
+ self.fstype = None
+ self.loop = loop_size
+ if self.loop:
+ utils.system('dd if=/dev/zero of=%s bs=1M count=%d' % \
+ (device, loop_size))
- def mkfs(self, fstype = 'ext2', args = ''):
- """
- Format a partition to fstype
- """
- if list_mount_devices().count(self.device):
- raise NameError('Attempted to format mounted device')
- if fstype == 'xfs':
- args += ' -f'
- if self.loop:
- # BAH. Inconsistent mkfs syntax SUCKS.
- if fstype == 'ext2' or fstype == 'ext3':
- args += ' -F'
- if fstype == 'reiserfs':
- args += ' -f'
- args = args.lstrip()
- mkfs_cmd = "mkfs -t %s %s %s" % (fstype, args, self.device)
- print mkfs_cmd
- sys.stdout.flush()
- try:
- utils.system("yes | " + mkfs_cmd)
- except:
- self.job.record('FAIL', None, mkfs_cmd, error.format_error())
- raise
- else:
- self.job.record('GOOD', None, mkfs_cmd)
- self.fstype = fstype
+ def mkfs(self, fstype = 'ext2', args = ''):
+ """
+ Format a partition to fstype
+ """
+ if list_mount_devices().count(self.device):
+ raise NameError('Attempted to format mounted device')
+ if fstype == 'xfs':
+ args += ' -f'
+ if self.loop:
+ # BAH. Inconsistent mkfs syntax SUCKS.
+ if fstype == 'ext2' or fstype == 'ext3':
+ args += ' -F'
+ if fstype == 'reiserfs':
+ args += ' -f'
+ args = args.lstrip()
+ mkfs_cmd = "mkfs -t %s %s %s" % (fstype, args, self.device)
+ print mkfs_cmd
+ sys.stdout.flush()
+ try:
+ utils.system("yes | " + mkfs_cmd)
+ except:
+ self.job.record('FAIL', None, mkfs_cmd, error.format_error())
+ raise
+ else:
+ self.job.record('GOOD', None, mkfs_cmd)
+ self.fstype = fstype
- def fsck(self, args = '-n'):
- # I hate reiserfstools.
- # Requires an explit Yes for some inane reason
- fsck_cmd = 'fsck %s %s' % (self.device, args)
- if self.fstype == 'reiserfs':
- fsck_cmd = 'yes "Yes" | ' + fsck_cmd
- print fsck_cmd
- sys.stdout.flush()
- try:
- utils.system("yes | " + fsck_cmd)
- except:
- self.job.record('FAIL', None, fsck_cmd, error.format_error())
- raise
- else:
- self.job.record('GOOD', None, fsck_cmd)
-
-
- def mount(self, mountpoint = None, args = ''):
- if self.fstype:
- args += ' -t ' + self.fstype
- if self.loop:
- args += ' -o loop'
- args = args.lstrip()
-
- if not mountpoint:
- mountpoint = self.mountpoint
- mount_cmd = "mount %s %s %s" % (args, self.device, mountpoint)
-
- if list_mount_devices().count(self.device):
- err = 'Attempted to mount mounted device'
- self.job.record('FAIL', None, mount_cmd, err)
- raise NameError(err)
- if list_mount_points().count(mountpoint):
- err = 'Attempted to mount busy mountpoint'
- self.job.record('FAIL', None, mount_cmd, err)
- raise NameError(err)
-
- print mount_cmd
- sys.stdout.flush()
- try:
- utils.system(mount_cmd)
- except:
- self.job.record('FAIL', None, mount_cmd, error.format_error())
- raise
- else:
- self.job.record('GOOD', None, mount_cmd)
+ def fsck(self, args = '-n'):
+ # I hate reiserfstools.
+ # Requires an explit Yes for some inane reason
+ fsck_cmd = 'fsck %s %s' % (self.device, args)
+ if self.fstype == 'reiserfs':
+ fsck_cmd = 'yes "Yes" | ' + fsck_cmd
+ print fsck_cmd
+ sys.stdout.flush()
+ try:
+ utils.system("yes | " + fsck_cmd)
+ except:
+ self.job.record('FAIL', None, fsck_cmd, error.format_error())
+ raise
+ else:
+ self.job.record('GOOD', None, fsck_cmd)
- def unmount(self, handle=None):
- if not handle:
- handle = self.device
- umount_cmd = "umount " + handle
- print umount_cmd
- sys.stdout.flush()
- try:
- utils.system(umount_cmd)
- except:
- self.job.record('FAIL', None, umount_cmd, error.format_error())
- raise
- else:
- self.job.record('GOOD', None, umount_cmd)
+ def mount(self, mountpoint = None, args = ''):
+ if self.fstype:
+ args += ' -t ' + self.fstype
+ if self.loop:
+ args += ' -o loop'
+ args = args.lstrip()
+
+ if not mountpoint:
+ mountpoint = self.mountpoint
+ mount_cmd = "mount %s %s %s" % (args, self.device, mountpoint)
+
+ if list_mount_devices().count(self.device):
+ err = 'Attempted to mount mounted device'
+ self.job.record('FAIL', None, mount_cmd, err)
+ raise NameError(err)
+ if list_mount_points().count(mountpoint):
+ err = 'Attempted to mount busy mountpoint'
+ self.job.record('FAIL', None, mount_cmd, err)
+ raise NameError(err)
+
+ print mount_cmd
+ sys.stdout.flush()
+ try:
+ utils.system(mount_cmd)
+ except:
+ self.job.record('FAIL', None, mount_cmd, error.format_error())
+ raise
+ else:
+ self.job.record('GOOD', None, mount_cmd)
- def get_io_scheduler_list(self, device_name):
- names = open(self.__sched_path(device_name)).read()
- return names.translate(string.maketrans('[]', ' ')).split()
+ def unmount(self, handle=None):
+ if not handle:
+ handle = self.device
+ umount_cmd = "umount " + handle
+ print umount_cmd
+ sys.stdout.flush()
+ try:
+ utils.system(umount_cmd)
+ except:
+ self.job.record('FAIL', None, umount_cmd, error.format_error())
+ raise
+ else:
+ self.job.record('GOOD', None, umount_cmd)
- def get_io_scheduler(self, device_name):
- return re.split('[\[\]]',
- open(self.__sched_path(device_name)).read())[1]
+ def get_io_scheduler_list(self, device_name):
+ names = open(self.__sched_path(device_name)).read()
+ return names.translate(string.maketrans('[]', ' ')).split()
- def set_io_scheduler(self, device_name, name):
- if name not in self.get_io_scheduler_list(device_name):
- raise NameError('No such IO scheduler: %s' % name)
- f = open(self.__sched_path(device_name), 'w')
- print >> f, name
- f.close()
+ def get_io_scheduler(self, device_name):
+ return re.split('[\[\]]',
+ open(self.__sched_path(device_name)).read())[1]
- def __sched_path(self, device_name):
- return '/sys/block/%s/queue/scheduler' % device_name
+ def set_io_scheduler(self, device_name, name):
+ if name not in self.get_io_scheduler_list(device_name):
+ raise NameError('No such IO scheduler: %s' % name)
+ f = open(self.__sched_path(device_name), 'w')
+ print >> f, name
+ f.close()
+
+
+ def __sched_path(self, device_name):
+ return '/sys/block/%s/queue/scheduler' % device_name
diff --git a/client/bin/grub.py b/client/bin/grub.py
index 68b9bc5..81ea3de 100755
--- a/client/bin/grub.py
+++ b/client/bin/grub.py
@@ -4,139 +4,139 @@
import shutil
import re
-import os
+import os
import os.path
import string
class grub:
- config_locations = ['/boot/grub/grub.conf', '/boot/grub/menu.lst',
- '/etc/grub.conf']
+ config_locations = ['/boot/grub/grub.conf', '/boot/grub/menu.lst',
+ '/etc/grub.conf']
- def __init__(self, config_file=None):
- if config_file:
- self.config = config_file
- else:
- self.config = self.detect()
- self.read()
+ def __init__(self, config_file=None):
+ if config_file:
+ self.config = config_file
+ else:
+ self.config = self.detect()
+ self.read()
- def read(self):
- conf_file = file(self.config, 'r')
- self.lines = conf_file.readlines()
- conf_file.close()
+ def read(self):
+ conf_file = file(self.config, 'r')
+ self.lines = conf_file.readlines()
+ conf_file.close()
- self.entries = [] # list of stanzas
- self.titles = {} # dictionary of titles
- entry = grub_entry(-1)
- count = 0
- for line in self.lines:
- if re.match(r'\s*title', line):
- self.entries.append(entry)
- entry = grub_entry(count)
- count = count + 1
- title = line.replace('title ', '')
- title = title.rstrip('\n')
- entry.set('title', title)
- self.titles[title] = entry
- # if line.startswith('initrd'):
- if re.match(r'\s*initrd', line):
- entry.set('initrd',
- re.sub(r'\s*initrd\s+', '', line))
- if re.match(r'\s*kernel', line):
- entry.set('kernel',
- re.sub(r'\s*kernel\s+', '', line))
- entry.lines.append(line)
- self.entries.append(entry)
- self.preamble = self.entries.pop(0) # separate preamble
+ self.entries = [] # list of stanzas
+ self.titles = {} # dictionary of titles
+ entry = grub_entry(-1)
+ count = 0
+ for line in self.lines:
+ if re.match(r'\s*title', line):
+ self.entries.append(entry)
+ entry = grub_entry(count)
+ count = count + 1
+ title = line.replace('title ', '')
+ title = title.rstrip('\n')
+ entry.set('title', title)
+ self.titles[title] = entry
+ # if line.startswith('initrd'):
+ if re.match(r'\s*initrd', line):
+ entry.set('initrd',
+ re.sub(r'\s*initrd\s+', '', line))
+ if re.match(r'\s*kernel', line):
+ entry.set('kernel',
+ re.sub(r'\s*kernel\s+', '', line))
+ entry.lines.append(line)
+ self.entries.append(entry)
+ self.preamble = self.entries.pop(0) # separate preamble
- def write(self):
- conf_file = file(self.config, 'w')
- conf_file.write(self.preamble)
- for entry in self.entries:
- conf_file.write(entry.lines)
- conf_file.close()
+ def write(self):
+ conf_file = file(self.config, 'w')
+ conf_file.write(self.preamble)
+ for entry in self.entries:
+ conf_file.write(entry.lines)
+ conf_file.close()
- def dump(self):
- for line in self.preamble.lines:
- print line,
- for entry in self.entries:
- for line in entry.lines:
- print line,
+ def dump(self):
+ for line in self.preamble.lines:
+ print line,
+ for entry in self.entries:
+ for line in entry.lines:
+ print line,
- def backup(self):
- shutil.copyfile(self.config, self.config+'.bak')
- restore = file(autodir + '/var/autotest.boot.restore', 'w')
- restore.write('cp ' + self.config+'.bak ' + self.config + '\n')
- restore.close()
+ def backup(self):
+ shutil.copyfile(self.config, self.config+'.bak')
+ restore = file(autodir + '/var/autotest.boot.restore', 'w')
+ restore.write('cp ' + self.config+'.bak ' + self.config + '\n')
+ restore.close()
- def bootloader(self):
- return 'grub'
+ def bootloader(self):
+ return 'grub'
- def detect(self):
- for config in grub.config_locations:
- if os.path.isfile(config) and not os.path.islink(config):
- return config
+ def detect(self):
+ for config in grub.config_locations:
+ if os.path.isfile(config) and not os.path.islink(config):
+ return config
- def list_titles(self):
- list = []
- for entry in self.entries:
- list.append(entry.get('title'))
- return list
+ def list_titles(self):
+ list = []
+ for entry in self.entries:
+ list.append(entry.get('title'))
+ return list
- def print_entry(self, index):
- entry = self.entries[index]
- entry.print_entry()
+ def print_entry(self, index):
+ entry = self.entries[index]
+ entry.print_entry()
- def renamed_entry(self, index, newname, args=False):
- "print a specified entry, renaming it as specified"
- entry = self.entries[index]
- entry.set('title', newname)
- if args:
- entry.set_autotest_kernel()
- entry.print_entry()
+ def renamed_entry(self, index, newname, args=False):
+ "print a specified entry, renaming it as specified"
+ entry = self.entries[index]
+ entry.set('title', newname)
+ if args:
+ entry.set_autotest_kernel()
+ entry.print_entry()
- def omit_markers(self, marker):
- # print, ommitting entries between specified markers
- print_state = True
- for line in lines:
- if line.count(marker):
- print_state = not print_state
- else:
- if print_state:
- print line
+ def omit_markers(self, marker):
+ # print, ommitting entries between specified markers
+ print_state = True
+ for line in lines:
+ if line.count(marker):
+ print_state = not print_state
+ else:
+ if print_state:
+ print line
- def select(self, title, boot_options=None):
- entry = self.titles[title]
- print "grub: will boot entry %d (0-based)" % entry.index
- self.set_default(entry.index)
- self.set_timeout()
+ def select(self, title, boot_options=None):
+ entry = self.titles[title]
+ print "grub: will boot entry %d (0-based)" % entry.index
+ self.set_default(entry.index)
+ self.set_timeout()
- def set_default(self, index):
- lines = (self.preamble).lines
- for i in range(len(lines)):
- default = 'default %d' % index
- lines[i] = re.sub(r'^\s*default.*',
- default, lines[i])
+ def set_default(self, index):
+ lines = (self.preamble).lines
+ for i in range(len(lines)):
+ default = 'default %d' % index
+ lines[i] = re.sub(r'^\s*default.*',
+ default, lines[i])
- def set_timeout(self):
- lines = (self.preamble).lines
- for i in range(len(lines)):
- lines[i] = re.sub(r'^timeout.*/',
- 'timeout 60', lines[i])
- lines[i] = re.sub(r'^(\s*terminal .*--timeout)=\d+',
- r'\1=30', lines[i])
-
+ def set_timeout(self):
+ lines = (self.preamble).lines
+ for i in range(len(lines)):
+ lines[i] = re.sub(r'^timeout.*/',
+ 'timeout 60', lines[i])
+ lines[i] = re.sub(r'^(\s*terminal .*--timeout)=\d+',
+ r'\1=30', lines[i])
+
# ----------------------------------------------------------------------
@@ -145,49 +145,49 @@
# and bits we don't understand.
class grub_entry:
- def __init__(self, count):
- self.lines = []
- self.fields = {} # title, initrd, kernel, etc
- self.index = count
+ def __init__(self, count):
+ self.lines = []
+ self.fields = {} # title, initrd, kernel, etc
+ self.index = count
- def set(self, field, value):
- print "setting '%s' to '%s'" % (field, value)
- self.fields[field] = value
- for i in range(len(self.lines)):
- m = re.match(r'\s*' + field + r'\s+', self.lines[i])
- if m:
- self.lines[i] = m.group() + value + '\n'
+ def set(self, field, value):
+ print "setting '%s' to '%s'" % (field, value)
+ self.fields[field] = value
+ for i in range(len(self.lines)):
+ m = re.match(r'\s*' + field + r'\s+', self.lines[i])
+ if m:
+ self.lines[i] = m.group() + value + '\n'
- def get(self, field):
- return self.fields[field]
+ def get(self, field):
+ return self.fields[field]
- def print_entry(self):
- print self.lines
+ def print_entry(self):
+ print self.lines
- def set_kernel_options(self, options):
- kernel = self.get('kernel')
- re.sub(r'(autotest_args:).*', r'\1'+options, kernel)
- self.set('kernel', kernel)
+ def set_kernel_options(self, options):
+ kernel = self.get('kernel')
+ re.sub(r'(autotest_args:).*', r'\1'+options, kernel)
+ self.set('kernel', kernel)
- def set_autotest_kernel(self):
- kernel_words = []
- found_path = False
- # Want to copy most of the entry, replacing the 'path'
- # part of the entry with vmlinux-autotest in the same
- # dir, and make sure autotest_args: is (uniquely) added
- for word in (self.get('kernel')).split():
- if word.startswith('--'):
- kernel_words.append(word)
- continue
- if not found_path:
- word = os.path.dirname(word)+'vmlinuz-autotest'
- found_path = True
- if re.match(r'auto(bench|test)_args:', word):
- break
- kernel_words.append(word)
- kernel_words.append('autotest_args: ')
- self.set('kernel', string.join(kernel_words))
+ def set_autotest_kernel(self):
+ kernel_words = []
+ found_path = False
+ # Want to copy most of the entry, replacing the 'path'
+ # part of the entry with vmlinux-autotest in the same
+ # dir, and make sure autotest_args: is (uniquely) added
+ for word in (self.get('kernel')).split():
+ if word.startswith('--'):
+ kernel_words.append(word)
+ continue
+ if not found_path:
+ word = os.path.dirname(word)+'vmlinuz-autotest'
+ found_path = True
+ if re.match(r'auto(bench|test)_args:', word):
+ break
+ kernel_words.append(word)
+ kernel_words.append('autotest_args: ')
+ self.set('kernel', string.join(kernel_words))
diff --git a/client/bin/harness.py b/client/bin/harness.py
index 2a6378f..86441b8 100755
--- a/client/bin/harness.py
+++ b/client/bin/harness.py
@@ -8,77 +8,77 @@
import os, sys
class harness:
- """The NULL server harness
+ """The NULL server harness
- Properties:
- job
- The job object for this job
- """
+ Properties:
+ job
+ The job object for this job
+ """
- def __init__(self, job):
- """
- job
- The job object for this job
- """
- self.setup(job)
+ def __init__(self, job):
+ """
+ job
+ The job object for this job
+ """
+ self.setup(job)
- def setup(self, job):
- """
- job
- The job object for this job
- """
- self.job = job
+ def setup(self, job):
+ """
+ job
+ The job object for this job
+ """
+ self.job = job
- configd = os.path.join(os.environ['AUTODIR'], 'configs')
- if os.path.isdir(configd):
- (name, dirs, files) = os.walk(configd).next()
- job.config_set('kernel.default_config_set',
- [ configd + '/' ] + files)
+ configd = os.path.join(os.environ['AUTODIR'], 'configs')
+ if os.path.isdir(configd):
+ (name, dirs, files) = os.walk(configd).next()
+ job.config_set('kernel.default_config_set',
+ [ configd + '/' ] + files)
- def run_start(self):
- """A run within this job is starting"""
- pass
+ def run_start(self):
+ """A run within this job is starting"""
+ pass
- def run_pause(self):
- """A run within this job is completing (expect continue)"""
- pass
+ def run_pause(self):
+ """A run within this job is completing (expect continue)"""
+ pass
- def run_reboot(self):
- """A run within this job is performing a reboot
- (expect continue following reboot)
- """
- pass
+ def run_reboot(self):
+ """A run within this job is performing a reboot
+ (expect continue following reboot)
+ """
+ pass
- def run_abort(self):
- """A run within this job is aborting. It all went wrong"""
- pass
+ def run_abort(self):
+ """A run within this job is aborting. It all went wrong"""
+ pass
- def run_complete(self):
- """A run within this job is completing (all done)"""
- pass
+ def run_complete(self):
+ """A run within this job is completing (all done)"""
+ pass
- def test_status(self, status, tag):
- """A test within this job is completing"""
- pass
+ def test_status(self, status, tag):
+ """A test within this job is completing"""
+ pass
- def test_status_detail(self, code, subdir, operation, status, tag):
- """A test within this job is completing (detail)"""
- pass
+ def test_status_detail(self, code, subdir, operation, status, tag):
+ """A test within this job is completing (detail)"""
+ pass
def select(which, job):
- if not which:
- which = 'standalone'
-
- exec "import harness_%s" % (which)
- exec "myharness = harness_%s.harness_%s(job)" % (which, which)
+ if not which:
+ which = 'standalone'
- return myharness
+ exec "import harness_%s" % (which)
+ exec "myharness = harness_%s.harness_%s(job)" % (which, which)
+
+ return myharness
diff --git a/client/bin/harness_ABAT.py b/client/bin/harness_ABAT.py
index e5f2f63..8fadb2a 100755
--- a/client/bin/harness_ABAT.py
+++ b/client/bin/harness_ABAT.py
@@ -10,145 +10,145 @@
import os, harness, time, re
def autobench_load(fn):
- disks = re.compile(r'^\s*DATS_FREE_DISKS\s*=(.*\S)\s*$')
- parts = re.compile(r'^\s*DATS_FREE_PARTITIONS\s*=(.*\S)\s*$')
- modules = re.compile(r'^\s*INITRD_MODULES\s*=(.*\S)\s*$')
+ disks = re.compile(r'^\s*DATS_FREE_DISKS\s*=(.*\S)\s*$')
+ parts = re.compile(r'^\s*DATS_FREE_PARTITIONS\s*=(.*\S)\s*$')
+ modules = re.compile(r'^\s*INITRD_MODULES\s*=(.*\S)\s*$')
- conf = {}
+ conf = {}
- try:
- fd = file(fn, "r")
- except:
- return conf
- for ln in fd.readlines():
- m = disks.match(ln)
- if m:
- val = m.groups()[0]
- conf['disks'] = val.strip('"').split()
- m = parts.match(ln)
- if m:
- val = m.groups()[0]
- conf['partitions'] = val.strip('"').split()
- m = modules.match(ln)
- if m:
- val = m.groups()[0]
- conf['modules'] = val.strip('"').split()
- fd.close()
+ try:
+ fd = file(fn, "r")
+ except:
+ return conf
+ for ln in fd.readlines():
+ m = disks.match(ln)
+ if m:
+ val = m.groups()[0]
+ conf['disks'] = val.strip('"').split()
+ m = parts.match(ln)
+ if m:
+ val = m.groups()[0]
+ conf['partitions'] = val.strip('"').split()
+ m = modules.match(ln)
+ if m:
+ val = m.groups()[0]
+ conf['modules'] = val.strip('"').split()
+ fd.close()
- return conf
+ return conf
class harness_ABAT(harness.harness):
- """The ABAT server harness
+ """The ABAT server harness
- Properties:
- job
- The job object for this job
- """
+ Properties:
+ job
+ The job object for this job
+ """
- def __init__(self, job):
- """
- job
- The job object for this job
- """
- self.setup(job)
+ def __init__(self, job):
+ """
+ job
+ The job object for this job
+ """
+ self.setup(job)
- if 'ABAT_STATUS' in os.environ:
- self.status = file(os.environ['ABAT_STATUS'], "w")
- else:
- self.status = None
+ if 'ABAT_STATUS' in os.environ:
+ self.status = file(os.environ['ABAT_STATUS'], "w")
+ else:
+ self.status = None
- def __send(self, msg):
- if self.status:
- msg = msg.rstrip()
- self.status.write(msg + "\n")
- self.status.flush()
+ def __send(self, msg):
+ if self.status:
+ msg = msg.rstrip()
+ self.status.write(msg + "\n")
+ self.status.flush()
- def __send_status(self, code, subdir, operation, msg):
- self.__send("STATUS %s %s %s %s" % \
- (code, subdir, operation, msg))
+ def __send_status(self, code, subdir, operation, msg):
+ self.__send("STATUS %s %s %s %s" % \
+ (code, subdir, operation, msg))
- def __root_device(self):
- device = None
- root = re.compile(r'^\S*(/dev/\S+).*\s/\s*$')
-
- df = utils.system_output('df -lP')
- for line in df.split("\n"):
- m = root.match(line)
- if m:
- device = m.groups()[0]
+ def __root_device(self):
+ device = None
+ root = re.compile(r'^\S*(/dev/\S+).*\s/\s*$')
- return device
+ df = utils.system_output('df -lP')
+ for line in df.split("\n"):
+ m = root.match(line)
+ if m:
+ device = m.groups()[0]
+
+ return device
- def run_start(self):
- """A run within this job is starting"""
- self.__send_status('GOOD', '----', '----', 'run starting')
+ def run_start(self):
+ """A run within this job is starting"""
+ self.__send_status('GOOD', '----', '----', 'run starting')
- # Load up the autobench.conf if it exists.
- conf = autobench_load("/etc/autobench.conf")
- if 'partitions' in conf:
- self.job.config_set('filesystem.partitions',
- conf['partitions'])
+ # Load up the autobench.conf if it exists.
+ conf = autobench_load("/etc/autobench.conf")
+ if 'partitions' in conf:
+ self.job.config_set('filesystem.partitions',
+ conf['partitions'])
- # Search the boot loader configuration for the autobench entry,
- # and extract its args.
- entry_args = None
- args = None
- for line in self.job.bootloader.info('all').split('\n'):
- if line.startswith('args'):
- entry_args = line.split(None, 2)[2]
- if line.startswith('title'):
- title = line.split()[2]
- if title == 'autobench':
- args = entry_args
+ # Search the boot loader configuration for the autobench entry,
+ # and extract its args.
+ entry_args = None
+ args = None
+ for line in self.job.bootloader.info('all').split('\n'):
+ if line.startswith('args'):
+ entry_args = line.split(None, 2)[2]
+ if line.startswith('title'):
+ title = line.split()[2]
+ if title == 'autobench':
+ args = entry_args
- if args:
- args = re.sub(r'autobench_args:.*', '', args)
- args = re.sub(r'root=\S*', '', args)
- args += " root=" + self.__root_device()
+ if args:
+ args = re.sub(r'autobench_args:.*', '', args)
+ args = re.sub(r'root=\S*', '', args)
+ args += " root=" + self.__root_device()
- self.job.config_set('boot.default_args', args)
+ self.job.config_set('boot.default_args', args)
- # Turn off boot_once semantics.
- self.job.config_set('boot.set_default', True)
+ # Turn off boot_once semantics.
+ self.job.config_set('boot.set_default', True)
- # For RedHat installs we do not load up the module.conf
- # as they cannot be builtin. Pass them as arguments.
- vendor = autotest_utils.get_os_vendor()
- if vendor in ['Red Hat', 'Fedora Core'] and 'modules' in conf:
- args = '--allow-missing'
- for mod in conf['modules']:
- args += " --with " + mod
- self.job.config_set('kernel.mkinitrd_extra_args', args)
+ # For RedHat installs we do not load up the module.conf
+ # as they cannot be builtin. Pass them as arguments.
+ vendor = autotest_utils.get_os_vendor()
+ if vendor in ['Red Hat', 'Fedora Core'] and 'modules' in conf:
+ args = '--allow-missing'
+ for mod in conf['modules']:
+ args += " --with " + mod
+ self.job.config_set('kernel.mkinitrd_extra_args', args)
- def run_reboot(self):
- """A run within this job is performing a reboot
- (expect continue following reboot)
- """
- self.__send("REBOOT")
+ def run_reboot(self):
+ """A run within this job is performing a reboot
+ (expect continue following reboot)
+ """
+ self.__send("REBOOT")
- def run_complete(self):
- """A run within this job is completing (all done)"""
- self.__send("DONE")
+ def run_complete(self):
+ """A run within this job is completing (all done)"""
+ self.__send("DONE")
- def test_status_detail(self, code, subdir, operation, msg, tag):
- """A test within this job is completing (detail)"""
+ def test_status_detail(self, code, subdir, operation, msg, tag):
+ """A test within this job is completing (detail)"""
- # Send the first line with the status code as a STATUS message.
- lines = msg.split("\n")
- self.__send_status(code, subdir, operation, lines[0])
+ # Send the first line with the status code as a STATUS message.
+ lines = msg.split("\n")
+ self.__send_status(code, subdir, operation, lines[0])
- def test_status(self, msg, tag):
- lines = msg.split("\n")
+ def test_status(self, msg, tag):
+ lines = msg.split("\n")
- # Send each line as a SUMMARY message.
- for line in lines:
- self.__send("SUMMARY :" + line)
+ # Send each line as a SUMMARY message.
+ for line in lines:
+ self.__send("SUMMARY :" + line)
diff --git a/client/bin/harness_simple.py b/client/bin/harness_simple.py
index 7f104da..5ff90d4 100755
--- a/client/bin/harness_simple.py
+++ b/client/bin/harness_simple.py
@@ -7,31 +7,31 @@
import os, harness, time
class harness_simple(harness.harness):
- """
- The simple server harness
+ """
+ The simple server harness
- Properties:
- job
- The job object for this job
- """
+ Properties:
+ job
+ The job object for this job
+ """
- def __init__(self, job):
- """
- job
- The job object for this job
- """
- self.setup(job)
+ def __init__(self, job):
+ """
+ job
+ The job object for this job
+ """
+ self.setup(job)
- self.status = os.fdopen(3, 'w')
+ self.status = os.fdopen(3, 'w')
- def test_status(self, status, tag):
- """A test within this job is completing"""
- if self.status:
- for line in status.split('\n'):
- # prepend status messages with
- # AUTOTEST_STATUS:tag: so that we can tell
- # which lines were sent by the autotest client
- pre = 'AUTOTEST_STATUS:%s:' % (tag,)
- self.status.write(pre + line + '\n')
- self.status.flush()
+ def test_status(self, status, tag):
+ """A test within this job is completing"""
+ if self.status:
+ for line in status.split('\n'):
+ # prepend status messages with
+ # AUTOTEST_STATUS:tag: so that we can tell
+ # which lines were sent by the autotest client
+ pre = 'AUTOTEST_STATUS:%s:' % (tag,)
+ self.status.write(pre + line + '\n')
+ self.status.flush()
diff --git a/client/bin/harness_standalone.py b/client/bin/harness_standalone.py
index dccdcab..d6a4687 100644
--- a/client/bin/harness_standalone.py
+++ b/client/bin/harness_standalone.py
@@ -9,40 +9,40 @@
import os, harness, shutil
class harness_standalone(harness.harness):
- """The standalone server harness
+ """The standalone server harness
- Properties:
- job
- The job object for this job
- """
+ Properties:
+ job
+ The job object for this job
+ """
- def __init__(self, job):
- """
- job
- The job object for this job
- """
- self.autodir = os.path.abspath(os.environ['AUTODIR'])
- self.setup(job)
+ def __init__(self, job):
+ """
+ job
+ The job object for this job
+ """
+ self.autodir = os.path.abspath(os.environ['AUTODIR'])
+ self.setup(job)
- src = job.control_get()
- dest = os.path.join(self.autodir, 'control')
- if os.path.abspath(src) != os.path.abspath(dest):
- shutil.copyfile(src, dest)
- job.control_set(dest)
+ src = job.control_get()
+ dest = os.path.join(self.autodir, 'control')
+ if os.path.abspath(src) != os.path.abspath(dest):
+ shutil.copyfile(src, dest)
+ job.control_set(dest)
- print 'Symlinking init scripts'
- rc = os.path.join(self.autodir, 'tools/autotest')
- # see if system supports event.d versus inittab
- if os.path.exists('/etc/event.d'):
- # NB: assuming current runlevel is default
- initdefault = utils.system_output('runlevel').split()[1]
- else:
- initdefault = utils.system_output('grep :initdefault: /etc/inittab')
- initdefault = initdefault.split(':')[1]
+ print 'Symlinking init scripts'
+ rc = os.path.join(self.autodir, 'tools/autotest')
+ # see if system supports event.d versus inittab
+ if os.path.exists('/etc/event.d'):
+ # NB: assuming current runlevel is default
+ initdefault = utils.system_output('runlevel').split()[1]
+ else:
+ initdefault = utils.system_output('grep :initdefault: /etc/inittab')
+ initdefault = initdefault.split(':')[1]
- try:
- utils.system('ln -sf %s /etc/init.d/autotest' % rc)
- utils.system('ln -sf %s /etc/rc%s.d/S99autotest' % \
- (rc, initdefault))
- except:
- print "WARNING: linking init scripts failed"
+ try:
+ utils.system('ln -sf %s /etc/init.d/autotest' % rc)
+ utils.system('ln -sf %s /etc/rc%s.d/S99autotest' % \
+ (rc, initdefault))
+ except:
+ print "WARNING: linking init scripts failed"
diff --git a/client/bin/job.py b/client/bin/job.py
index ac2745b..870b63e 100755
--- a/client/bin/job.py
+++ b/client/bin/job.py
@@ -20,995 +20,995 @@
"""
class StepError(error.AutotestError):
- pass
+ pass
class base_job:
- """The actual job against which we do everything.
+ """The actual job against which we do everything.
- Properties:
- autodir
- The top level autotest directory (/usr/local/autotest).
- Comes from os.environ['AUTODIR'].
- bindir
- <autodir>/bin/
- libdir
- <autodir>/lib/
- testdir
- <autodir>/tests/
- site_testdir
- <autodir>/site_tests/
- profdir
- <autodir>/profilers/
- tmpdir
- <autodir>/tmp/
- resultdir
- <autodir>/results/<jobtag>
- stdout
- fd_stack object for stdout
- stderr
- fd_stack object for stderr
- profilers
- the profilers object for this job
- harness
- the server harness object for this job
- config
- the job configuration for this job
- """
+ Properties:
+ autodir
+ The top level autotest directory (/usr/local/autotest).
+ Comes from os.environ['AUTODIR'].
+ bindir
+ <autodir>/bin/
+ libdir
+ <autodir>/lib/
+ testdir
+ <autodir>/tests/
+ site_testdir
+ <autodir>/site_tests/
+ profdir
+ <autodir>/profilers/
+ tmpdir
+ <autodir>/tmp/
+ resultdir
+ <autodir>/results/<jobtag>
+ stdout
+ fd_stack object for stdout
+ stderr
+ fd_stack object for stderr
+ profilers
+ the profilers object for this job
+ harness
+ the server harness object for this job
+ config
+ the job configuration for this job
+ """
- DEFAULT_LOG_FILENAME = "status"
+ DEFAULT_LOG_FILENAME = "status"
- def __init__(self, control, jobtag, cont, harness_type=None,
- use_external_logging = False):
- """
- control
- The control file (pathname of)
- jobtag
- The job tag string (eg "default")
- cont
- If this is the continuation of this job
- harness_type
- An alternative server harness
- """
- self.autodir = os.environ['AUTODIR']
- self.bindir = os.path.join(self.autodir, 'bin')
- self.libdir = os.path.join(self.autodir, 'lib')
- self.testdir = os.path.join(self.autodir, 'tests')
- self.site_testdir = os.path.join(self.autodir, 'site_tests')
- self.profdir = os.path.join(self.autodir, 'profilers')
- self.tmpdir = os.path.join(self.autodir, 'tmp')
- self.resultdir = os.path.join(self.autodir, 'results', jobtag)
- self.sysinfodir = os.path.join(self.resultdir, 'sysinfo')
- self.control = os.path.abspath(control)
- self.state_file = self.control + '.state'
- self.current_step_ancestry = []
- self.next_step_index = 0
- self.__load_state()
+ def __init__(self, control, jobtag, cont, harness_type=None,
+ use_external_logging = False):
+ """
+ control
+ The control file (pathname of)
+ jobtag
+ The job tag string (eg "default")
+ cont
+ If this is the continuation of this job
+ harness_type
+ An alternative server harness
+ """
+ self.autodir = os.environ['AUTODIR']
+ self.bindir = os.path.join(self.autodir, 'bin')
+ self.libdir = os.path.join(self.autodir, 'lib')
+ self.testdir = os.path.join(self.autodir, 'tests')
+ self.site_testdir = os.path.join(self.autodir, 'site_tests')
+ self.profdir = os.path.join(self.autodir, 'profilers')
+ self.tmpdir = os.path.join(self.autodir, 'tmp')
+ self.resultdir = os.path.join(self.autodir, 'results', jobtag)
+ self.sysinfodir = os.path.join(self.resultdir, 'sysinfo')
+ self.control = os.path.abspath(control)
+ self.state_file = self.control + '.state'
+ self.current_step_ancestry = []
+ self.next_step_index = 0
+ self.__load_state()
- if not cont:
- """
- Don't cleanup the tmp dir (which contains the lockfile)
- in the constructor, this would be a problem for multiple
- jobs starting at the same time on the same client. Instead
- do the delete at the server side. We simply create the tmp
- directory here if it does not already exist.
- """
- if not os.path.exists(self.tmpdir):
- os.mkdir(self.tmpdir)
+ if not cont:
+ """
+ Don't cleanup the tmp dir (which contains the lockfile)
+ in the constructor, this would be a problem for multiple
+ jobs starting at the same time on the same client. Instead
+ do the delete at the server side. We simply create the tmp
+ directory here if it does not already exist.
+ """
+ if not os.path.exists(self.tmpdir):
+ os.mkdir(self.tmpdir)
- results = os.path.join(self.autodir, 'results')
- if not os.path.exists(results):
- os.mkdir(results)
-
- download = os.path.join(self.testdir, 'download')
- if not os.path.exists(download):
- os.mkdir(download)
+ results = os.path.join(self.autodir, 'results')
+ if not os.path.exists(results):
+ os.mkdir(results)
- if os.path.exists(self.resultdir):
- utils.system('rm -rf '
- + self.resultdir)
- os.mkdir(self.resultdir)
- os.mkdir(self.sysinfodir)
+ download = os.path.join(self.testdir, 'download')
+ if not os.path.exists(download):
+ os.mkdir(download)
- os.mkdir(os.path.join(self.resultdir, 'debug'))
- os.mkdir(os.path.join(self.resultdir, 'analysis'))
+ if os.path.exists(self.resultdir):
+ utils.system('rm -rf '
+ + self.resultdir)
+ os.mkdir(self.resultdir)
+ os.mkdir(self.sysinfodir)
- shutil.copyfile(self.control,
- os.path.join(self.resultdir, 'control'))
+ os.mkdir(os.path.join(self.resultdir, 'debug'))
+ os.mkdir(os.path.join(self.resultdir, 'analysis'))
+ shutil.copyfile(self.control,
+ os.path.join(self.resultdir, 'control'))
- self.control = control
- self.jobtag = jobtag
- self.log_filename = self.DEFAULT_LOG_FILENAME
- self.container = None
- self.stdout = fd_stack.fd_stack(1, sys.stdout)
- self.stderr = fd_stack.fd_stack(2, sys.stderr)
+ self.control = control
+ self.jobtag = jobtag
+ self.log_filename = self.DEFAULT_LOG_FILENAME
+ self.container = None
- self._init_group_level()
+ self.stdout = fd_stack.fd_stack(1, sys.stdout)
+ self.stderr = fd_stack.fd_stack(2, sys.stderr)
- self.config = config.config(self)
+ self._init_group_level()
- self.harness = harness.select(harness_type, self)
+ self.config = config.config(self)
- self.profilers = profilers.profilers(self)
+ self.harness = harness.select(harness_type, self)
- try:
- tool = self.config_get('boottool.executable')
- self.bootloader = boottool.boottool(tool)
- except:
- pass
+ self.profilers = profilers.profilers(self)
- sysinfo.log_per_reboot_data(self.sysinfodir)
+ try:
+ tool = self.config_get('boottool.executable')
+ self.bootloader = boottool.boottool(tool)
+ except:
+ pass
- if not cont:
- self.record('START', None, None)
- self._increment_group_level()
+ sysinfo.log_per_reboot_data(self.sysinfodir)
- self.harness.run_start()
-
- if use_external_logging:
- self.enable_external_logging()
+ if not cont:
+ self.record('START', None, None)
+ self._increment_group_level()
- # load the max disk usage rate - default to no monitoring
- self.max_disk_usage_rate = self.get_state('__monitor_disk',
- default=0.0)
+ self.harness.run_start()
+ if use_external_logging:
+ self.enable_external_logging()
- def monitor_disk_usage(self, max_rate):
- """\
- Signal that the job should monitor disk space usage on /
- and generate a warning if a test uses up disk space at a
- rate exceeding 'max_rate'.
+ # load the max disk usage rate - default to no monitoring
+ self.max_disk_usage_rate = self.get_state('__monitor_disk',
+ default=0.0)
- Parameters:
- max_rate - the maximium allowed rate of disk consumption
- during a test, in MB/hour, or 0 to indicate
- no limit.
- """
- self.set_state('__monitor_disk', max_rate)
- self.max_disk_usage_rate = max_rate
+ def monitor_disk_usage(self, max_rate):
+ """\
+ Signal that the job should monitor disk space usage on /
+ and generate a warning if a test uses up disk space at a
+ rate exceeding 'max_rate'.
- def relative_path(self, path):
- """\
- Return a patch relative to the job results directory
- """
- head = len(self.resultdir) + 1 # remove the / inbetween
- return path[head:]
+ Parameters:
+ max_rate - the maximium allowed rate of disk consumption
+ during a test, in MB/hour, or 0 to indicate
+ no limit.
+ """
+ self.set_state('__monitor_disk', max_rate)
+ self.max_disk_usage_rate = max_rate
- def control_get(self):
- return self.control
+ def relative_path(self, path):
+ """\
+ Return a patch relative to the job results directory
+ """
+ head = len(self.resultdir) + 1 # remove the / inbetween
+ return path[head:]
- def control_set(self, control):
- self.control = os.path.abspath(control)
+ def control_get(self):
+ return self.control
- def harness_select(self, which):
- self.harness = harness.select(which, self)
+ def control_set(self, control):
+ self.control = os.path.abspath(control)
- def config_set(self, name, value):
- self.config.set(name, value)
+ def harness_select(self, which):
+ self.harness = harness.select(which, self)
- def config_get(self, name):
- return self.config.get(name)
+ def config_set(self, name, value):
+ self.config.set(name, value)
- def setup_dirs(self, results_dir, tmp_dir):
- if not tmp_dir:
- tmp_dir = os.path.join(self.tmpdir, 'build')
- if not os.path.exists(tmp_dir):
- os.mkdir(tmp_dir)
- if not os.path.isdir(tmp_dir):
- e_msg = "Temp dir (%s) is not a dir - args backwards?" % self.tmpdir
- raise ValueError(e_msg)
- # We label the first build "build" and then subsequent ones
- # as "build.2", "build.3", etc. Whilst this is a little bit
- # inconsistent, 99.9% of jobs will only have one build
- # (that's not done as kernbench, sparse, or buildtest),
- # so it works out much cleaner. One of life's comprimises.
- if not results_dir:
- results_dir = os.path.join(self.resultdir, 'build')
- i = 2
- while os.path.exists(results_dir):
- results_dir = os.path.join(self.resultdir, 'build.%d' % i)
- i += 1
- if not os.path.exists(results_dir):
- os.mkdir(results_dir)
+ def config_get(self, name):
+ return self.config.get(name)
- return (results_dir, tmp_dir)
+ def setup_dirs(self, results_dir, tmp_dir):
+ if not tmp_dir:
+ tmp_dir = os.path.join(self.tmpdir, 'build')
+ if not os.path.exists(tmp_dir):
+ os.mkdir(tmp_dir)
+ if not os.path.isdir(tmp_dir):
+ e_msg = "Temp dir (%s) is not a dir - args backwards?" % self.tmpdir
+ raise ValueError(e_msg)
+ # We label the first build "build" and then subsequent ones
+ # as "build.2", "build.3", etc. Whilst this is a little bit
+ # inconsistent, 99.9% of jobs will only have one build
+ # (that's not done as kernbench, sparse, or buildtest),
+ # so it works out much cleaner. One of life's comprimises.
+ if not results_dir:
+ results_dir = os.path.join(self.resultdir, 'build')
+ i = 2
+ while os.path.exists(results_dir):
+ results_dir = os.path.join(self.resultdir, 'build.%d' % i)
+ i += 1
+ if not os.path.exists(results_dir):
+ os.mkdir(results_dir)
- def xen(self, base_tree, results_dir = '', tmp_dir = '', leave = False, \
- kjob = None ):
- """Summon a xen object"""
- (results_dir, tmp_dir) = self.setup_dirs(results_dir, tmp_dir)
- build_dir = 'xen'
- return xen.xen(self, base_tree, results_dir, tmp_dir, build_dir, leave, kjob)
+ return (results_dir, tmp_dir)
- def kernel(self, base_tree, results_dir = '', tmp_dir = '', leave = False):
- """Summon a kernel object"""
- (results_dir, tmp_dir) = self.setup_dirs(results_dir, tmp_dir)
- build_dir = 'linux'
- return kernel.auto_kernel(self, base_tree, results_dir,
- tmp_dir, build_dir, leave)
+ def xen(self, base_tree, results_dir = '', tmp_dir = '', leave = False, \
+ kjob = None ):
+ """Summon a xen object"""
+ (results_dir, tmp_dir) = self.setup_dirs(results_dir, tmp_dir)
+ build_dir = 'xen'
+ return xen.xen(self, base_tree, results_dir, tmp_dir, build_dir, leave, kjob)
- def barrier(self, *args, **kwds):
- """Create a barrier object"""
- return barrier.barrier(*args, **kwds)
+ def kernel(self, base_tree, results_dir = '', tmp_dir = '', leave = False):
+ """Summon a kernel object"""
+ (results_dir, tmp_dir) = self.setup_dirs(results_dir, tmp_dir)
+ build_dir = 'linux'
+ return kernel.auto_kernel(self, base_tree, results_dir,
+ tmp_dir, build_dir, leave)
- def setup_dep(self, deps):
- """Set up the dependencies for this test.
-
- deps is a list of libraries required for this test.
- """
- for dep in deps:
- try:
- os.chdir(os.path.join(self.autodir, 'deps', dep))
- utils.system('./' + dep + '.py')
- except:
- err = "setting up dependency " + dep + "\n"
- raise error.UnhandledError(err)
+ def barrier(self, *args, **kwds):
+ """Create a barrier object"""
+ return barrier.barrier(*args, **kwds)
- def __runtest(self, url, tag, args, dargs):
- try:
- l = lambda : test.runtest(self, url, tag, args, dargs)
- pid = parallel.fork_start(self.resultdir, l)
- parallel.fork_waitfor(self.resultdir, pid)
- except error.AutotestError:
- raise
- except Exception, e:
- msg = "Unhandled %s error occured during test\n"
- msg %= str(e.__class__.__name__)
- raise error.UnhandledError(msg)
+ def setup_dep(self, deps):
+ """Set up the dependencies for this test.
+ deps is a list of libraries required for this test.
+ """
+ for dep in deps:
+ try:
+ os.chdir(os.path.join(self.autodir, 'deps', dep))
+ utils.system('./' + dep + '.py')
+ except:
+ err = "setting up dependency " + dep + "\n"
+ raise error.UnhandledError(err)
- def run_test(self, url, *args, **dargs):
- """Summon a test object and run it.
-
- tag
- tag to add to testname
- url
- url of the test to run
- """
- if not url:
- raise TypeError("Test name is invalid. "
- "Switched arguments?")
- (group, testname) = test.testname(url)
- namelen = len(testname)
- dargs = dargs.copy()
- tntag = dargs.pop('tag', None)
- if tntag: # testname tag is included in reported test name
- testname += '.' + tntag
- subdir = testname
- sdtag = dargs.pop('subdir_tag', None)
- if sdtag: # subdir-only tag is not included in reports
- subdir = subdir + '.' + sdtag
- tag = subdir[namelen+1:] # '' if none
+ def __runtest(self, url, tag, args, dargs):
+ try:
+ l = lambda : test.runtest(self, url, tag, args, dargs)
+ pid = parallel.fork_start(self.resultdir, l)
+ parallel.fork_waitfor(self.resultdir, pid)
+ except error.AutotestError:
+ raise
+ except Exception, e:
+ msg = "Unhandled %s error occured during test\n"
+ msg %= str(e.__class__.__name__)
+ raise error.UnhandledError(msg)
- outputdir = os.path.join(self.resultdir, subdir)
- if os.path.exists(outputdir):
- msg = ("%s already exists, test <%s> may have"
- " already run with tag <%s>"
- % (outputdir, testname, tag) )
- raise error.TestError(msg)
- os.mkdir(outputdir)
-
- container = dargs.pop('container', None)
- if container:
- cname = container.get('name', None)
- if not cname: # get old name
- cname = container.get('container_name', None)
- mbytes = container.get('mbytes', None)
- if not mbytes: # get old name
- mbytes = container.get('mem', None)
- cpus = container.get('cpus', None)
- if not cpus: # get old name
- cpus = container.get('cpu', None)
- root = container.get('root', None)
- self.new_container(mbytes=mbytes, cpus=cpus,
- root=root, name=cname)
- # We are running in a container now...
- def log_warning(reason):
- self.record("WARN", subdir, testname, reason)
- @disk_usage_monitor.watch(log_warning, "/",
- self.max_disk_usage_rate)
- def group_func():
- try:
- self.__runtest(url, tag, args, dargs)
- except error.TestNAError, detail:
- self.record('TEST_NA', subdir, testname,
- str(detail))
- raise
- except Exception, detail:
- self.record('FAIL', subdir, testname,
- str(detail))
- raise
- else:
- self.record('GOOD', subdir, testname,
- 'completed successfully')
+ def run_test(self, url, *args, **dargs):
+ """Summon a test object and run it.
- result, exc_info = self.__rungroup(subdir, testname, group_func)
- if container:
- self.release_container()
- if exc_info and isinstance(exc_info[1], error.TestError):
- return False
- elif exc_info:
- raise exc_info[0], exc_info[1], exc_info[2]
- else:
- return True
+ tag
+ tag to add to testname
+ url
+ url of the test to run
+ """
+ if not url:
+ raise TypeError("Test name is invalid. "
+ "Switched arguments?")
+ (group, testname) = test.testname(url)
+ namelen = len(testname)
+ dargs = dargs.copy()
+ tntag = dargs.pop('tag', None)
+ if tntag: # testname tag is included in reported test name
+ testname += '.' + tntag
+ subdir = testname
+ sdtag = dargs.pop('subdir_tag', None)
+ if sdtag: # subdir-only tag is not included in reports
+ subdir = subdir + '.' + sdtag
+ tag = subdir[namelen+1:] # '' if none
- def __rungroup(self, subdir, testname, function, *args, **dargs):
- """\
- subdir:
- name of the group
- testname:
- name of the test to run, or support step
- function:
- subroutine to run
- *args:
- arguments for the function
+ outputdir = os.path.join(self.resultdir, subdir)
+ if os.path.exists(outputdir):
+ msg = ("%s already exists, test <%s> may have"
+ " already run with tag <%s>"
+ % (outputdir, testname, tag) )
+ raise error.TestError(msg)
+ os.mkdir(outputdir)
- Returns a 2-tuple (result, exc_info) where result
- is the return value of function, and exc_info is
- the sys.exc_info() of the exception thrown by the
- function (which may be None).
- """
+ container = dargs.pop('container', None)
+ if container:
+ cname = container.get('name', None)
+ if not cname: # get old name
+ cname = container.get('container_name', None)
+ mbytes = container.get('mbytes', None)
+ if not mbytes: # get old name
+ mbytes = container.get('mem', None)
+ cpus = container.get('cpus', None)
+ if not cpus: # get old name
+ cpus = container.get('cpu', None)
+ root = container.get('root', None)
+ self.new_container(mbytes=mbytes, cpus=cpus,
+ root=root, name=cname)
+ # We are running in a container now...
- result, exc_info = None, None
- try:
- self.record('START', subdir, testname)
- self._increment_group_level()
- result = function(*args, **dargs)
- self._decrement_group_level()
- self.record('END GOOD', subdir, testname)
- except error.TestNAError, e:
- self._decrement_group_level()
- self.record('END TEST_NA', subdir, testname, str(e))
- except Exception, e:
- exc_info = sys.exc_info()
- self._decrement_group_level()
- err_msg = str(e) + '\n' + traceback.format_exc()
- self.record('END FAIL', subdir, testname, err_msg)
+ def log_warning(reason):
+ self.record("WARN", subdir, testname, reason)
+ @disk_usage_monitor.watch(log_warning, "/",
+ self.max_disk_usage_rate)
+ def group_func():
+ try:
+ self.__runtest(url, tag, args, dargs)
+ except error.TestNAError, detail:
+ self.record('TEST_NA', subdir, testname,
+ str(detail))
+ raise
+ except Exception, detail:
+ self.record('FAIL', subdir, testname,
+ str(detail))
+ raise
+ else:
+ self.record('GOOD', subdir, testname,
+ 'completed successfully')
- return result, exc_info
+ result, exc_info = self.__rungroup(subdir, testname, group_func)
+ if container:
+ self.release_container()
+ if exc_info and isinstance(exc_info[1], error.TestError):
+ return False
+ elif exc_info:
+ raise exc_info[0], exc_info[1], exc_info[2]
+ else:
+ return True
- def run_group(self, function, *args, **dargs):
- """\
- function:
- subroutine to run
- *args:
- arguments for the function
- """
+ def __rungroup(self, subdir, testname, function, *args, **dargs):
+ """\
+ subdir:
+ name of the group
+ testname:
+ name of the test to run, or support step
+ function:
+ subroutine to run
+ *args:
+ arguments for the function
- # Allow the tag for the group to be specified
- name = function.__name__
- tag = dargs.pop('tag', None)
- if tag:
- name = tag
+ Returns a 2-tuple (result, exc_info) where result
+ is the return value of function, and exc_info is
+ the sys.exc_info() of the exception thrown by the
+ function (which may be None).
+ """
- outputdir = os.path.join(self.resultdir, name)
- if os.path.exists(outputdir):
- msg = ("%s already exists, test <%s> may have"
- " already run with tag <%s>"
- % (outputdir, name, name) )
- raise error.TestError(msg)
- os.mkdir(outputdir)
+ result, exc_info = None, None
+ try:
+ self.record('START', subdir, testname)
+ self._increment_group_level()
+ result = function(*args, **dargs)
+ self._decrement_group_level()
+ self.record('END GOOD', subdir, testname)
+ except error.TestNAError, e:
+ self._decrement_group_level()
+ self.record('END TEST_NA', subdir, testname, str(e))
+ except Exception, e:
+ exc_info = sys.exc_info()
+ self._decrement_group_level()
+ err_msg = str(e) + '\n' + traceback.format_exc()
+ self.record('END FAIL', subdir, testname, err_msg)
- result, exc_info = self.__rungroup(name, name, function,
- *args, **dargs)
+ return result, exc_info
- # if there was a non-TestError exception, raise it
- if exc_info and not isinstance(exc_info[1], error.TestError):
- err = ''.join(traceback.format_exception(*exc_info))
- raise error.TestError(name + ' failed\n' + err)
- # pass back the actual return value from the function
- return result
+ def run_group(self, function, *args, **dargs):
+ """\
+ function:
+ subroutine to run
+ *args:
+ arguments for the function
+ """
+ # Allow the tag for the group to be specified
+ name = function.__name__
+ tag = dargs.pop('tag', None)
+ if tag:
+ name = tag
- def new_container(self, mbytes=None, cpus=None, root=None, name=None):
- if not autotest_utils.grep('cpuset', '/proc/filesystems'):
- print "Containers not enabled by latest reboot"
- return # containers weren't enabled in this kernel boot
- pid = os.getpid()
- if not name:
- name = 'test%d' % pid # make arbitrary unique name
- self.container = cpuset.cpuset(name, job_size=mbytes,
- job_pid=pid, cpus=cpus, root=root)
- # This job's python shell is now running in the new container
- # and all forked test processes will inherit that container
+ outputdir = os.path.join(self.resultdir, name)
+ if os.path.exists(outputdir):
+ msg = ("%s already exists, test <%s> may have"
+ " already run with tag <%s>"
+ % (outputdir, name, name) )
+ raise error.TestError(msg)
+ os.mkdir(outputdir)
+
+ result, exc_info = self.__rungroup(name, name, function,
+ *args, **dargs)
+
+ # if there was a non-TestError exception, raise it
+ if exc_info and not isinstance(exc_info[1], error.TestError):
+ err = ''.join(traceback.format_exception(*exc_info))
+ raise error.TestError(name + ' failed\n' + err)
+
+ # pass back the actual return value from the function
+ return result
+
+
+ def new_container(self, mbytes=None, cpus=None, root=None, name=None):
+ if not autotest_utils.grep('cpuset', '/proc/filesystems'):
+ print "Containers not enabled by latest reboot"
+ return # containers weren't enabled in this kernel boot
+ pid = os.getpid()
+ if not name:
+ name = 'test%d' % pid # make arbitrary unique name
+ self.container = cpuset.cpuset(name, job_size=mbytes,
+ job_pid=pid, cpus=cpus, root=root)
+ # This job's python shell is now running in the new container
+ # and all forked test processes will inherit that container
+
+
+ def release_container(self):
+ if self.container:
+ self.container.release()
+ self.container = None
+
+ def cpu_count(self):
+ if self.container:
+ return len(self.container.cpus)
+ return autotest_utils.count_cpus() # use total system count
- def release_container(self):
- if self.container:
- self.container.release()
- self.container = None
+ # Check the passed kernel identifier against the command line
+ # and the running kernel, abort the job on missmatch.
+ def kernel_check_ident(self, expected_when, expected_id, subdir,
+ type = 'src', patches=[]):
+ print (("POST BOOT: checking booted kernel " +
+ "mark=%d identity='%s' type='%s'") %
+ (expected_when, expected_id, type))
- def cpu_count(self):
- if self.container:
- return len(self.container.cpus)
- return autotest_utils.count_cpus() # use total system count
+ running_id = autotest_utils.running_os_ident()
+ cmdline = utils.read_one_line("/proc/cmdline")
- # Check the passed kernel identifier against the command line
- # and the running kernel, abort the job on missmatch.
- def kernel_check_ident(self, expected_when, expected_id, subdir,
- type = 'src', patches=[]):
- print (("POST BOOT: checking booted kernel " +
- "mark=%d identity='%s' type='%s'") %
- (expected_when, expected_id, type))
+ find_sum = re.compile(r'.*IDENT=(\d+)')
+ m = find_sum.match(cmdline)
+ cmdline_when = -1
+ if m:
+ cmdline_when = int(m.groups()[0])
- running_id = autotest_utils.running_os_ident()
+ # We have all the facts, see if they indicate we
+ # booted the requested kernel or not.
+ bad = False
+ if (type == 'src' and expected_id != running_id or
+ type == 'rpm' and
+ not running_id.startswith(expected_id + '::')):
+ print "check_kernel_ident: kernel identifier mismatch"
+ bad = True
+ if expected_when != cmdline_when:
+ print "check_kernel_ident: kernel command line mismatch"
+ bad = True
- cmdline = utils.read_one_line("/proc/cmdline")
+ if bad:
+ print " Expected Ident: " + expected_id
+ print " Running Ident: " + running_id
+ print " Expected Mark: %d" % (expected_when)
+ print "Command Line Mark: %d" % (cmdline_when)
+ print " Command Line: " + cmdline
- find_sum = re.compile(r'.*IDENT=(\d+)')
- m = find_sum.match(cmdline)
- cmdline_when = -1
- if m:
- cmdline_when = int(m.groups()[0])
+ raise error.JobError("boot failure", "reboot.verify")
- # We have all the facts, see if they indicate we
- # booted the requested kernel or not.
- bad = False
- if (type == 'src' and expected_id != running_id or
- type == 'rpm' and
- not running_id.startswith(expected_id + '::')):
- print "check_kernel_ident: kernel identifier mismatch"
- bad = True
- if expected_when != cmdline_when:
- print "check_kernel_ident: kernel command line mismatch"
- bad = True
+ kernel_info = {'kernel': expected_id}
+ for i, patch in enumerate(patches):
+ kernel_info["patch%d" % i] = patch
+ self.record('GOOD', subdir, 'reboot.verify', expected_id)
+ self._decrement_group_level()
+ self.record('END GOOD', subdir, 'reboot',
+ optional_fields=kernel_info)
- if bad:
- print " Expected Ident: " + expected_id
- print " Running Ident: " + running_id
- print " Expected Mark: %d" % (expected_when)
- print "Command Line Mark: %d" % (cmdline_when)
- print " Command Line: " + cmdline
- raise error.JobError("boot failure", "reboot.verify")
+ def filesystem(self, device, mountpoint = None, loop_size = 0):
+ if not mountpoint:
+ mountpoint = self.tmpdir
+ return filesystem.filesystem(self, device, mountpoint,loop_size)
- kernel_info = {'kernel': expected_id}
- for i, patch in enumerate(patches):
- kernel_info["patch%d" % i] = patch
- self.record('GOOD', subdir, 'reboot.verify', expected_id)
- self._decrement_group_level()
- self.record('END GOOD', subdir, 'reboot',
- optional_fields=kernel_info)
+ def enable_external_logging(self):
+ pass
- def filesystem(self, device, mountpoint = None, loop_size = 0):
- if not mountpoint:
- mountpoint = self.tmpdir
- return filesystem.filesystem(self, device, mountpoint,loop_size)
-
- def enable_external_logging(self):
- pass
+ def disable_external_logging(self):
+ pass
- def disable_external_logging(self):
- pass
-
+ def reboot_setup(self):
+ pass
- def reboot_setup(self):
- pass
+ def reboot(self, tag='autotest'):
+ self.reboot_setup()
+ self.record('START', None, 'reboot')
+ self._increment_group_level()
+ self.record('GOOD', None, 'reboot.start')
+ self.harness.run_reboot()
+ default = self.config_get('boot.set_default')
+ if default:
+ self.bootloader.set_default(tag)
+ else:
+ self.bootloader.boot_once(tag)
+ cmd = "(sleep 5; reboot) </dev/null >/dev/null 2>&1 &"
+ utils.system(cmd)
+ self.quit()
- def reboot(self, tag='autotest'):
- self.reboot_setup()
- self.record('START', None, 'reboot')
- self._increment_group_level()
- self.record('GOOD', None, 'reboot.start')
- self.harness.run_reboot()
- default = self.config_get('boot.set_default')
- if default:
- self.bootloader.set_default(tag)
- else:
- self.bootloader.boot_once(tag)
- cmd = "(sleep 5; reboot) </dev/null >/dev/null 2>&1 &"
- utils.system(cmd)
- self.quit()
+ def noop(self, text):
+ print "job: noop: " + text
- def noop(self, text):
- print "job: noop: " + text
+ def parallel(self, *tasklist):
+ """Run tasks in parallel"""
- def parallel(self, *tasklist):
- """Run tasks in parallel"""
+ pids = []
+ old_log_filename = self.log_filename
+ for i, task in enumerate(tasklist):
+ self.log_filename = old_log_filename + (".%d" % i)
+ task_func = lambda: task[0](*task[1:])
+ pids.append(parallel.fork_start(self.resultdir,
+ task_func))
- pids = []
- old_log_filename = self.log_filename
- for i, task in enumerate(tasklist):
- self.log_filename = old_log_filename + (".%d" % i)
- task_func = lambda: task[0](*task[1:])
- pids.append(parallel.fork_start(self.resultdir,
- task_func))
+ old_log_path = os.path.join(self.resultdir, old_log_filename)
+ old_log = open(old_log_path, "a")
+ exceptions = []
+ for i, pid in enumerate(pids):
+ # wait for the task to finish
+ try:
+ parallel.fork_waitfor(self.resultdir, pid)
+ except Exception, e:
+ exceptions.append(e)
+ # copy the logs from the subtask into the main log
+ new_log_path = old_log_path + (".%d" % i)
+ if os.path.exists(new_log_path):
+ new_log = open(new_log_path)
+ old_log.write(new_log.read())
+ new_log.close()
+ old_log.flush()
+ os.remove(new_log_path)
+ old_log.close()
- old_log_path = os.path.join(self.resultdir, old_log_filename)
- old_log = open(old_log_path, "a")
- exceptions = []
- for i, pid in enumerate(pids):
- # wait for the task to finish
- try:
- parallel.fork_waitfor(self.resultdir, pid)
- except Exception, e:
- exceptions.append(e)
- # copy the logs from the subtask into the main log
- new_log_path = old_log_path + (".%d" % i)
- if os.path.exists(new_log_path):
- new_log = open(new_log_path)
- old_log.write(new_log.read())
- new_log.close()
- old_log.flush()
- os.remove(new_log_path)
- old_log.close()
+ self.log_filename = old_log_filename
- self.log_filename = old_log_filename
+ # handle any exceptions raised by the parallel tasks
+ if exceptions:
+ msg = "%d task(s) failed" % len(exceptions)
+ raise error.JobError(msg, str(exceptions), exceptions)
- # handle any exceptions raised by the parallel tasks
- if exceptions:
- msg = "%d task(s) failed" % len(exceptions)
- raise error.JobError(msg, str(exceptions), exceptions)
+ def quit(self):
+ # XXX: should have a better name.
+ self.harness.run_pause()
+ raise error.JobContinue("more to come")
- def quit(self):
- # XXX: should have a better name.
- self.harness.run_pause()
- raise error.JobContinue("more to come")
+ def complete(self, status):
+ """Clean up and exit"""
+ # We are about to exit 'complete' so clean up the control file.
+ try:
+ os.unlink(self.state_file)
+ except:
+ pass
- def complete(self, status):
- """Clean up and exit"""
- # We are about to exit 'complete' so clean up the control file.
- try:
- os.unlink(self.state_file)
- except:
- pass
+ self.harness.run_complete()
+ self.disable_external_logging()
+ sys.exit(status)
- self.harness.run_complete()
- self.disable_external_logging()
- sys.exit(status)
+ def set_state(self, var, val):
+ # Deep copies make sure that the state can't be altered
+ # without it being re-written. Perf wise, deep copies
+ # are overshadowed by pickling/loading.
+ self.state[var] = copy.deepcopy(val)
+ pickle.dump(self.state, open(self.state_file, 'w'))
- def set_state(self, var, val):
- # Deep copies make sure that the state can't be altered
- # without it being re-written. Perf wise, deep copies
- # are overshadowed by pickling/loading.
- self.state[var] = copy.deepcopy(val)
- pickle.dump(self.state, open(self.state_file, 'w'))
+ def __load_state(self):
+ assert not hasattr(self, "state")
+ try:
+ self.state = pickle.load(open(self.state_file, 'r'))
+ self.state_existed = True
+ except Exception:
+ print "Initializing the state engine."
+ self.state = {}
+ self.set_state('__steps', []) # writes pickle file
+ self.state_existed = False
- def __load_state(self):
- assert not hasattr(self, "state")
- try:
- self.state = pickle.load(open(self.state_file, 'r'))
- self.state_existed = True
- except Exception:
- print "Initializing the state engine."
- self.state = {}
- self.set_state('__steps', []) # writes pickle file
- self.state_existed = False
+ def get_state(self, var, default=None):
+ if var in self.state or default == None:
+ val = self.state[var]
+ else:
+ val = default
+ return copy.deepcopy(val)
- def get_state(self, var, default=None):
- if var in self.state or default == None:
- val = self.state[var]
- else:
- val = default
- return copy.deepcopy(val)
+ def __create_step_tuple(self, fn, args, dargs):
+ # Legacy code passes in an array where the first arg is
+ # the function or its name.
+ if isinstance(fn, list):
+ assert(len(args) == 0)
+ assert(len(dargs) == 0)
+ args = fn[1:]
+ fn = fn[0]
+ # Pickling actual functions is harry, thus we have to call
+ # them by name. Unfortunately, this means only functions
+ # defined globally can be used as a next step.
+ if callable(fn):
+ fn = fn.__name__
+ if not isinstance(fn, types.StringTypes):
+ raise StepError("Next steps must be functions or "
+ "strings containing the function name")
+ ancestry = copy.copy(self.current_step_ancestry)
+ return (ancestry, fn, args, dargs)
- def __create_step_tuple(self, fn, args, dargs):
- # Legacy code passes in an array where the first arg is
- # the function or its name.
- if isinstance(fn, list):
- assert(len(args) == 0)
- assert(len(dargs) == 0)
- args = fn[1:]
- fn = fn[0]
- # Pickling actual functions is harry, thus we have to call
- # them by name. Unfortunately, this means only functions
- # defined globally can be used as a next step.
- if callable(fn):
- fn = fn.__name__
- if not isinstance(fn, types.StringTypes):
- raise StepError("Next steps must be functions or "
- "strings containing the function name")
- ancestry = copy.copy(self.current_step_ancestry)
- return (ancestry, fn, args, dargs)
+ def next_step_append(self, fn, *args, **dargs):
+ """Define the next step and place it at the end"""
+ steps = self.get_state('__steps')
+ steps.append(self.__create_step_tuple(fn, args, dargs))
+ self.set_state('__steps', steps)
- def next_step_append(self, fn, *args, **dargs):
- """Define the next step and place it at the end"""
- steps = self.get_state('__steps')
- steps.append(self.__create_step_tuple(fn, args, dargs))
- self.set_state('__steps', steps)
+ def next_step(self, fn, *args, **dargs):
+ """Create a new step and place it after any steps added
+ while running the current step but before any steps added in
+ previous steps"""
+ steps = self.get_state('__steps')
+ steps.insert(self.next_step_index,
+ self.__create_step_tuple(fn, args, dargs))
+ self.next_step_index += 1
+ self.set_state('__steps', steps)
- def next_step(self, fn, *args, **dargs):
- """Create a new step and place it after any steps added
- while running the current step but before any steps added in
- previous steps"""
- steps = self.get_state('__steps')
- steps.insert(self.next_step_index,
- self.__create_step_tuple(fn, args, dargs))
- self.next_step_index += 1
- self.set_state('__steps', steps)
+ def next_step_prepend(self, fn, *args, **dargs):
+ """Insert a new step, executing first"""
+ steps = self.get_state('__steps')
+ steps.insert(0, self.__create_step_tuple(fn, args, dargs))
+ self.next_step_index += 1
+ self.set_state('__steps', steps)
- def next_step_prepend(self, fn, *args, **dargs):
- """Insert a new step, executing first"""
- steps = self.get_state('__steps')
- steps.insert(0, self.__create_step_tuple(fn, args, dargs))
- self.next_step_index += 1
- self.set_state('__steps', steps)
+ def _run_step_fn(self, local_vars, fn, args, dargs):
+ """Run a (step) function within the given context"""
- def _run_step_fn(self, local_vars, fn, args, dargs):
- """Run a (step) function within the given context"""
+ local_vars['__args'] = args
+ local_vars['__dargs'] = dargs
+ exec('__ret = %s(*__args, **__dargs)' % fn,
+ local_vars, local_vars)
+ return local_vars['__ret']
- local_vars['__args'] = args
- local_vars['__dargs'] = dargs
- exec('__ret = %s(*__args, **__dargs)' % fn,
- local_vars, local_vars)
- return local_vars['__ret']
+ def _create_frame(self, global_vars, ancestry, fn_name):
+ """Set up the environment like it would have been when this
+ function was first defined.
- def _create_frame(self, global_vars, ancestry, fn_name):
- """Set up the environment like it would have been when this
- function was first defined.
+ Child step engine 'implementations' must have 'return locals()'
+ at end end of their steps. Because of this, we can call the
+ parent function and get back all child functions (i.e. those
+ defined within it).
- Child step engine 'implementations' must have 'return locals()'
- at end end of their steps. Because of this, we can call the
- parent function and get back all child functions (i.e. those
- defined within it).
+ Unfortunately, the call stack of the function calling
+ job.next_step might have been deeper than the function it
+ added. In order to make sure that the environment is what it
+ should be, we need to then pop off the frames we built until
+ we find the frame where the function was first defined."""
- Unfortunately, the call stack of the function calling
- job.next_step might have been deeper than the function it
- added. In order to make sure that the environment is what it
- should be, we need to then pop off the frames we built until
- we find the frame where the function was first defined."""
+ # The copies ensure that the parent frames are not modified
+ # while building child frames. This matters if we then
+ # pop some frames in the next part of this function.
+ current_frame = copy.copy(global_vars)
+ frames = [current_frame]
+ for steps_fn_name in ancestry:
+ ret = self._run_step_fn(current_frame,
+ steps_fn_name, [], {})
+ current_frame = copy.copy(ret)
+ frames.append(current_frame)
- # The copies ensure that the parent frames are not modified
- # while building child frames. This matters if we then
- # pop some frames in the next part of this function.
- current_frame = copy.copy(global_vars)
- frames = [current_frame]
- for steps_fn_name in ancestry:
- ret = self._run_step_fn(current_frame,
- steps_fn_name, [], {})
- current_frame = copy.copy(ret)
- frames.append(current_frame)
+ while len(frames) > 2:
+ if fn_name not in frames[-2]:
+ break
+ if frames[-2][fn_name] != frames[-1][fn_name]:
+ break
+ frames.pop()
+ ancestry.pop()
- while len(frames) > 2:
- if fn_name not in frames[-2]:
- break
- if frames[-2][fn_name] != frames[-1][fn_name]:
- break
- frames.pop()
- ancestry.pop()
+ return (frames[-1], ancestry)
- return (frames[-1], ancestry)
+ def _add_step_init(self, local_vars, current_function):
+ """If the function returned a dictionary that includes a
+ function named 'step_init', prepend it to our list of steps.
+ This will only get run the first time a function with a nested
+ use of the step engine is run."""
- def _add_step_init(self, local_vars, current_function):
- """If the function returned a dictionary that includes a
- function named 'step_init', prepend it to our list of steps.
- This will only get run the first time a function with a nested
- use of the step engine is run."""
+ if (isinstance(local_vars, dict) and
+ 'step_init' in local_vars and
+ callable(local_vars['step_init'])):
+ # The init step is a child of the function
+ # we were just running.
+ self.current_step_ancestry.append(current_function)
+ self.next_step_prepend('step_init')
- if (isinstance(local_vars, dict) and
- 'step_init' in local_vars and
- callable(local_vars['step_init'])):
- # The init step is a child of the function
- # we were just running.
- self.current_step_ancestry.append(current_function)
- self.next_step_prepend('step_init')
+ def step_engine(self):
+ """the stepping engine -- if the control file defines
+ step_init we will be using this engine to drive multiple runs.
+ """
+ """Do the next step"""
- def step_engine(self):
- """the stepping engine -- if the control file defines
- step_init we will be using this engine to drive multiple runs.
- """
- """Do the next step"""
+ # Set up the environment and then interpret the control file.
+ # Some control files will have code outside of functions,
+ # which means we need to have our state engine initialized
+ # before reading in the file.
+ global_control_vars = {'job': self}
+ exec(JOB_PREAMBLE, global_control_vars, global_control_vars)
+ execfile(self.control, global_control_vars, global_control_vars)
- # Set up the environment and then interpret the control file.
- # Some control files will have code outside of functions,
- # which means we need to have our state engine initialized
- # before reading in the file.
- global_control_vars = {'job': self}
- exec(JOB_PREAMBLE, global_control_vars, global_control_vars)
- execfile(self.control, global_control_vars, global_control_vars)
+ # If we loaded in a mid-job state file, then we presumably
+ # know what steps we have yet to run.
+ if not self.state_existed:
+ if global_control_vars.has_key('step_init'):
+ self.next_step(global_control_vars['step_init'])
- # If we loaded in a mid-job state file, then we presumably
- # know what steps we have yet to run.
- if not self.state_existed:
- if global_control_vars.has_key('step_init'):
- self.next_step(global_control_vars['step_init'])
+ # Iterate through the steps. If we reboot, we'll simply
+ # continue iterating on the next step.
+ while len(self.get_state('__steps')) > 0:
+ steps = self.get_state('__steps')
+ (ancestry, fn_name, args, dargs) = steps.pop(0)
+ self.set_state('__steps', steps)
- # Iterate through the steps. If we reboot, we'll simply
- # continue iterating on the next step.
- while len(self.get_state('__steps')) > 0:
- steps = self.get_state('__steps')
- (ancestry, fn_name, args, dargs) = steps.pop(0)
- self.set_state('__steps', steps)
+ self.next_step_index = 0
+ ret = self._create_frame(global_control_vars, ancestry,
+ fn_name)
+ local_vars, self.current_step_ancestry = ret
+ local_vars = self._run_step_fn(local_vars, fn_name,
+ args, dargs)
+ self._add_step_init(local_vars, fn_name)
- self.next_step_index = 0
- ret = self._create_frame(global_control_vars, ancestry,
- fn_name)
- local_vars, self.current_step_ancestry = ret
- local_vars = self._run_step_fn(local_vars, fn_name,
- args, dargs)
- self._add_step_init(local_vars, fn_name)
+ def _init_group_level(self):
+ self.group_level = self.get_state("__group_level", default=0)
- def _init_group_level(self):
- self.group_level = self.get_state("__group_level", default=0)
+ def _increment_group_level(self):
+ self.group_level += 1
+ self.set_state("__group_level", self.group_level)
- def _increment_group_level(self):
- self.group_level += 1
- self.set_state("__group_level", self.group_level)
+ def _decrement_group_level(self):
+ self.group_level -= 1
+ self.set_state("__group_level", self.group_level)
- def _decrement_group_level(self):
- self.group_level -= 1
- self.set_state("__group_level", self.group_level)
+ def record(self, status_code, subdir, operation, status = '',
+ optional_fields=None):
+ """
+ Record job-level status
- def record(self, status_code, subdir, operation, status = '',
- optional_fields=None):
- """
- Record job-level status
+ The intent is to make this file both machine parseable and
+ human readable. That involves a little more complexity, but
+ really isn't all that bad ;-)
- The intent is to make this file both machine parseable and
- human readable. That involves a little more complexity, but
- really isn't all that bad ;-)
+ Format is <status code>\t<subdir>\t<operation>\t<status>
- Format is <status code>\t<subdir>\t<operation>\t<status>
+ status code: (GOOD|WARN|FAIL|ABORT)
+ or START
+ or END (GOOD|WARN|FAIL|ABORT)
- status code: (GOOD|WARN|FAIL|ABORT)
- or START
- or END (GOOD|WARN|FAIL|ABORT)
+ subdir: MUST be a relevant subdirectory in the results,
+ or None, which will be represented as '----'
- subdir: MUST be a relevant subdirectory in the results,
- or None, which will be represented as '----'
+ operation: description of what you ran (e.g. "dbench", or
+ "mkfs -t foobar /dev/sda9")
- operation: description of what you ran (e.g. "dbench", or
- "mkfs -t foobar /dev/sda9")
+ status: error message or "completed sucessfully"
- status: error message or "completed sucessfully"
+ ------------------------------------------------------------
- ------------------------------------------------------------
+ Initial tabs indicate indent levels for grouping, and is
+ governed by self.group_level
- Initial tabs indicate indent levels for grouping, and is
- governed by self.group_level
+ multiline messages have secondary lines prefaced by a double
+ space (' ')
+ """
- multiline messages have secondary lines prefaced by a double
- space (' ')
- """
+ if subdir:
+ if re.match(r'[\n\t]', subdir):
+ raise ValueError("Invalid character in "
+ "subdir string")
+ substr = subdir
+ else:
+ substr = '----'
- if subdir:
- if re.match(r'[\n\t]', subdir):
- raise ValueError("Invalid character in "
- "subdir string")
- substr = subdir
- else:
- substr = '----'
-
- if not logging.is_valid_status(status_code):
- raise ValueError("Invalid status code supplied: %s" %
- status_code)
- if not operation:
- operation = '----'
+ if not logging.is_valid_status(status_code):
+ raise ValueError("Invalid status code supplied: %s" %
+ status_code)
+ if not operation:
+ operation = '----'
- if re.match(r'[\n\t]', operation):
- raise ValueError("Invalid character in "
- "operation string")
- operation = operation.rstrip()
+ if re.match(r'[\n\t]', operation):
+ raise ValueError("Invalid character in "
+ "operation string")
+ operation = operation.rstrip()
- if not optional_fields:
- optional_fields = {}
+ if not optional_fields:
+ optional_fields = {}
- status = status.rstrip()
- status = re.sub(r"\t", " ", status)
- # Ensure any continuation lines are marked so we can
- # detect them in the status file to ensure it is parsable.
- status = re.sub(r"\n", "\n" + "\t" * self.group_level + " ",
- status)
+ status = status.rstrip()
+ status = re.sub(r"\t", " ", status)
+ # Ensure any continuation lines are marked so we can
+ # detect them in the status file to ensure it is parsable.
+ status = re.sub(r"\n", "\n" + "\t" * self.group_level + " ",
+ status)
- # Generate timestamps for inclusion in the logs
- epoch_time = int(time.time()) # seconds since epoch, in UTC
- local_time = time.localtime(epoch_time)
- optional_fields["timestamp"] = str(epoch_time)
- optional_fields["localtime"] = time.strftime("%b %d %H:%M:%S",
- local_time)
+ # Generate timestamps for inclusion in the logs
+ epoch_time = int(time.time()) # seconds since epoch, in UTC
+ local_time = time.localtime(epoch_time)
+ optional_fields["timestamp"] = str(epoch_time)
+ optional_fields["localtime"] = time.strftime("%b %d %H:%M:%S",
+ local_time)
- fields = [status_code, substr, operation]
- fields += ["%s=%s" % x for x in optional_fields.iteritems()]
- fields.append(status)
+ fields = [status_code, substr, operation]
+ fields += ["%s=%s" % x for x in optional_fields.iteritems()]
+ fields.append(status)
- msg = '\t'.join(str(x) for x in fields)
- msg = '\t' * self.group_level + msg
+ msg = '\t'.join(str(x) for x in fields)
+ msg = '\t' * self.group_level + msg
- msg_tag = ""
- if "." in self.log_filename:
- msg_tag = self.log_filename.split(".", 1)[1]
+ msg_tag = ""
+ if "." in self.log_filename:
+ msg_tag = self.log_filename.split(".", 1)[1]
- self.harness.test_status_detail(status_code, substr,
- operation, status, msg_tag)
- self.harness.test_status(msg, msg_tag)
+ self.harness.test_status_detail(status_code, substr,
+ operation, status, msg_tag)
+ self.harness.test_status(msg, msg_tag)
- # log to stdout (if enabled)
- #if self.log_filename == self.DEFAULT_LOG_FILENAME:
- print msg
+ # log to stdout (if enabled)
+ #if self.log_filename == self.DEFAULT_LOG_FILENAME:
+ print msg
- # log to the "root" status log
- status_file = os.path.join(self.resultdir, self.log_filename)
- open(status_file, "a").write(msg + "\n")
+ # log to the "root" status log
+ status_file = os.path.join(self.resultdir, self.log_filename)
+ open(status_file, "a").write(msg + "\n")
- # log to the subdir status log (if subdir is set)
- if subdir:
- dir = os.path.join(self.resultdir, subdir)
- status_file = os.path.join(dir,
- self.DEFAULT_LOG_FILENAME)
- open(status_file, "a").write(msg + "\n")
+ # log to the subdir status log (if subdir is set)
+ if subdir:
+ dir = os.path.join(self.resultdir, subdir)
+ status_file = os.path.join(dir,
+ self.DEFAULT_LOG_FILENAME)
+ open(status_file, "a").write(msg + "\n")
class disk_usage_monitor:
- def __init__(self, logging_func, device, max_mb_per_hour):
- self.func = logging_func
- self.device = device
- self.max_mb_per_hour = max_mb_per_hour
+ def __init__(self, logging_func, device, max_mb_per_hour):
+ self.func = logging_func
+ self.device = device
+ self.max_mb_per_hour = max_mb_per_hour
- def start(self):
- self.initial_space = autotest_utils.freespace(self.device)
- self.start_time = time.time()
+ def start(self):
+ self.initial_space = autotest_utils.freespace(self.device)
+ self.start_time = time.time()
- def stop(self):
- # if no maximum usage rate was set, we don't need to
- # generate any warnings
- if not self.max_mb_per_hour:
- return
+ def stop(self):
+ # if no maximum usage rate was set, we don't need to
+ # generate any warnings
+ if not self.max_mb_per_hour:
+ return
- final_space = autotest_utils.freespace(self.device)
- used_space = self.initial_space - final_space
- stop_time = time.time()
- total_time = stop_time - self.start_time
- # round up the time to one minute, to keep extremely short
- # tests from generating false positives due to short, badly
- # timed bursts of activity
- total_time = max(total_time, 60.0)
+ final_space = autotest_utils.freespace(self.device)
+ used_space = self.initial_space - final_space
+ stop_time = time.time()
+ total_time = stop_time - self.start_time
+ # round up the time to one minute, to keep extremely short
+ # tests from generating false positives due to short, badly
+ # timed bursts of activity
+ total_time = max(total_time, 60.0)
- # determine the usage rate
- bytes_per_sec = used_space / total_time
- mb_per_sec = bytes_per_sec / 1024**2
- mb_per_hour = mb_per_sec * 60 * 60
+ # determine the usage rate
+ bytes_per_sec = used_space / total_time
+ mb_per_sec = bytes_per_sec / 1024**2
+ mb_per_hour = mb_per_sec * 60 * 60
- if mb_per_hour > self.max_mb_per_hour:
- msg = ("disk space on %s was consumed at a rate of "
- "%.2f MB/hour")
- msg %= (self.device, mb_per_hour)
- self.func(msg)
+ if mb_per_hour > self.max_mb_per_hour:
+ msg = ("disk space on %s was consumed at a rate of "
+ "%.2f MB/hour")
+ msg %= (self.device, mb_per_hour)
+ self.func(msg)
- @classmethod
- def watch(cls, *monitor_args, **monitor_dargs):
- """ Generic decorator to wrap a function call with the
- standard create-monitor -> start -> call -> stop idiom."""
- def decorator(func):
- def watched_func(*args, **dargs):
- monitor = cls(*monitor_args, **monitor_dargs)
- monitor.start()
- try:
- func(*args, **dargs)
- finally:
- monitor.stop()
- return watched_func
- return decorator
+ @classmethod
+ def watch(cls, *monitor_args, **monitor_dargs):
+ """ Generic decorator to wrap a function call with the
+ standard create-monitor -> start -> call -> stop idiom."""
+ def decorator(func):
+ def watched_func(*args, **dargs):
+ monitor = cls(*monitor_args, **monitor_dargs)
+ monitor.start()
+ try:
+ func(*args, **dargs)
+ finally:
+ monitor.stop()
+ return watched_func
+ return decorator
def runjob(control, cont = False, tag = "default", harness_type = '',
- use_external_logging = False):
- """The main interface to this module
+ use_external_logging = False):
+ """The main interface to this module
- control
- The control file to use for this job.
- cont
- Whether this is the continuation of a previously started job
- """
- control = os.path.abspath(control)
- state = control + '.state'
+ control
+ The control file to use for this job.
+ cont
+ Whether this is the continuation of a previously started job
+ """
+ control = os.path.abspath(control)
+ state = control + '.state'
- # instantiate the job object ready for the control file.
- myjob = None
- try:
- # Check that the control file is valid
- if not os.path.exists(control):
- raise error.JobError(control +
- ": control file not found")
+ # instantiate the job object ready for the control file.
+ myjob = None
+ try:
+ # Check that the control file is valid
+ if not os.path.exists(control):
+ raise error.JobError(control +
+ ": control file not found")
- # When continuing, the job is complete when there is no
- # state file, ensure we don't try and continue.
- if cont and not os.path.exists(state):
- raise error.JobComplete("all done")
- if cont == False and os.path.exists(state):
- os.unlink(state)
+ # When continuing, the job is complete when there is no
+ # state file, ensure we don't try and continue.
+ if cont and not os.path.exists(state):
+ raise error.JobComplete("all done")
+ if cont == False and os.path.exists(state):
+ os.unlink(state)
- myjob = job(control, tag, cont, harness_type,
- use_external_logging)
+ myjob = job(control, tag, cont, harness_type,
+ use_external_logging)
- # Load in the users control file, may do any one of:
- # 1) execute in toto
- # 2) define steps, and select the first via next_step()
- myjob.step_engine()
+ # Load in the users control file, may do any one of:
+ # 1) execute in toto
+ # 2) define steps, and select the first via next_step()
+ myjob.step_engine()
- except error.JobContinue:
- sys.exit(5)
+ except error.JobContinue:
+ sys.exit(5)
- except error.JobComplete:
- sys.exit(1)
+ except error.JobComplete:
+ sys.exit(1)
- except error.JobError, instance:
- print "JOB ERROR: " + instance.args[0]
- if myjob:
- command = None
- if len(instance.args) > 1:
- command = instance.args[1]
- myjob.record('ABORT', None, command, instance.args[0])
- myjob._decrement_group_level()
- myjob.record('END ABORT', None, None)
- assert(myjob.group_level == 0)
- myjob.complete(1)
- else:
- sys.exit(1)
+ except error.JobError, instance:
+ print "JOB ERROR: " + instance.args[0]
+ if myjob:
+ command = None
+ if len(instance.args) > 1:
+ command = instance.args[1]
+ myjob.record('ABORT', None, command, instance.args[0])
+ myjob._decrement_group_level()
+ myjob.record('END ABORT', None, None)
+ assert(myjob.group_level == 0)
+ myjob.complete(1)
+ else:
+ sys.exit(1)
- except Exception, e:
- msg = str(e) + '\n' + traceback.format_exc()
- print "JOB ERROR: " + msg
- if myjob:
- myjob.record('ABORT', None, None, msg)
- myjob._decrement_group_level()
- myjob.record('END ABORT', None, None)
- assert(myjob.group_level == 0)
- myjob.complete(1)
- else:
- sys.exit(1)
+ except Exception, e:
+ msg = str(e) + '\n' + traceback.format_exc()
+ print "JOB ERROR: " + msg
+ if myjob:
+ myjob.record('ABORT', None, None, msg)
+ myjob._decrement_group_level()
+ myjob.record('END ABORT', None, None)
+ assert(myjob.group_level == 0)
+ myjob.complete(1)
+ else:
+ sys.exit(1)
- # If we get here, then we assume the job is complete and good.
- myjob._decrement_group_level()
- myjob.record('END GOOD', None, None)
- assert(myjob.group_level == 0)
+ # If we get here, then we assume the job is complete and good.
+ myjob._decrement_group_level()
+ myjob.record('END GOOD', None, None)
+ assert(myjob.group_level == 0)
- myjob.complete(0)
+ myjob.complete(0)
# site_job.py may be non-existant or empty, make sure that an appropriate
# site_job class is created nevertheless
try:
- from site_job import site_job
+ from site_job import site_job
except ImportError:
- class site_job(base_job):
- pass
+ class site_job(base_job):
+ pass
class job(site_job):
- pass
+ pass
diff --git a/client/bin/kernel.py b/client/bin/kernel.py
index fca8f7e..ad65ae9 100755
--- a/client/bin/kernel.py
+++ b/client/bin/kernel.py
@@ -8,727 +8,727 @@
class kernel:
- """ Class for compiling kernels.
+ """ Class for compiling kernels.
- Data for the object includes the src files
- used to create the kernel, patches applied, config (base + changes),
- the build directory itself, and logged output
+ Data for the object includes the src files
+ used to create the kernel, patches applied, config (base + changes),
+ the build directory itself, and logged output
- Properties:
- job
- Backpointer to the job object we're part of
- autodir
- Path to the top level autotest dir (/usr/local/autotest)
- src_dir
- <tmp_dir>/src/
- build_dir
- <tmp_dir>/linux/
- config_dir
- <results_dir>/config/
- log_dir
- <results_dir>/debug/
- results_dir
- <results_dir>/results/
- """
+ Properties:
+ job
+ Backpointer to the job object we're part of
+ autodir
+ Path to the top level autotest dir (/usr/local/autotest)
+ src_dir
+ <tmp_dir>/src/
+ build_dir
+ <tmp_dir>/linux/
+ config_dir
+ <results_dir>/config/
+ log_dir
+ <results_dir>/debug/
+ results_dir
+ <results_dir>/results/
+ """
- autodir = ''
+ autodir = ''
- def __init__(self, job, base_tree, subdir, tmp_dir, build_dir, leave = False):
- """Initialize the kernel build environment
+ def __init__(self, job, base_tree, subdir, tmp_dir, build_dir, leave = False):
+ """Initialize the kernel build environment
- job
- which job this build is part of
- base_tree
- base kernel tree. Can be one of the following:
- 1. A local tarball
- 2. A URL to a tarball
- 3. A local directory (will symlink it)
- 4. A shorthand expandable (eg '2.6.11-git3')
- subdir
- subdir in the results directory (eg "build")
- (holds config/, debug/, results/)
- tmp_dir
+ job
+ which job this build is part of
+ base_tree
+ base kernel tree. Can be one of the following:
+ 1. A local tarball
+ 2. A URL to a tarball
+ 3. A local directory (will symlink it)
+ 4. A shorthand expandable (eg '2.6.11-git3')
+ subdir
+ subdir in the results directory (eg "build")
+ (holds config/, debug/, results/)
+ tmp_dir
- leave
- Boolean, whether to leave existing tmpdir or not
- """
- self.job = job
- self.autodir = job.autodir
+ leave
+ Boolean, whether to leave existing tmpdir or not
+ """
+ self.job = job
+ self.autodir = job.autodir
- self.src_dir = os.path.join(tmp_dir, 'src')
- self.build_dir = os.path.join(tmp_dir, build_dir)
- # created by get_kernel_tree
- self.config_dir = os.path.join(subdir, 'config')
- self.log_dir = os.path.join(subdir, 'debug')
- self.results_dir = os.path.join(subdir, 'results')
- self.subdir = os.path.basename(subdir)
+ self.src_dir = os.path.join(tmp_dir, 'src')
+ self.build_dir = os.path.join(tmp_dir, build_dir)
+ # created by get_kernel_tree
+ self.config_dir = os.path.join(subdir, 'config')
+ self.log_dir = os.path.join(subdir, 'debug')
+ self.results_dir = os.path.join(subdir, 'results')
+ self.subdir = os.path.basename(subdir)
- self.installed_as = None
+ self.installed_as = None
- if not leave:
- if os.path.isdir(self.src_dir):
- utils.system('rm -rf ' + self.src_dir)
- if os.path.isdir(self.build_dir):
- utils.system('rm -rf ' + self.build_dir)
+ if not leave:
+ if os.path.isdir(self.src_dir):
+ utils.system('rm -rf ' + self.src_dir)
+ if os.path.isdir(self.build_dir):
+ utils.system('rm -rf ' + self.build_dir)
- if not os.path.exists(self.src_dir):
- os.mkdir(self.src_dir)
- for path in [self.config_dir, self.log_dir, self.results_dir]:
- if os.path.exists(path):
- utils.system('rm -rf ' + path)
- os.mkdir(path)
+ if not os.path.exists(self.src_dir):
+ os.mkdir(self.src_dir)
+ for path in [self.config_dir, self.log_dir, self.results_dir]:
+ if os.path.exists(path):
+ utils.system('rm -rf ' + path)
+ os.mkdir(path)
- logpath = os.path.join(self.log_dir, 'build_log')
- self.logfile = open(logpath, 'w+')
- self.applied_patches = []
+ logpath = os.path.join(self.log_dir, 'build_log')
+ self.logfile = open(logpath, 'w+')
+ self.applied_patches = []
- self.target_arch = None
- self.build_target = 'bzImage'
- self.build_image = None
+ self.target_arch = None
+ self.build_target = 'bzImage'
+ self.build_image = None
- arch = autotest_utils.get_current_kernel_arch()
- if arch == 's390' or arch == 's390x':
- self.build_target = 'image'
- elif arch == 'ia64':
- self.build_target = 'all'
- self.build_image = 'vmlinux.gz'
+ arch = autotest_utils.get_current_kernel_arch()
+ if arch == 's390' or arch == 's390x':
+ self.build_target = 'image'
+ elif arch == 'ia64':
+ self.build_target = 'all'
+ self.build_image = 'vmlinux.gz'
- if leave:
- return
+ if leave:
+ return
- self.logfile.write('BASE: %s\n' % base_tree)
+ self.logfile.write('BASE: %s\n' % base_tree)
- # Where we have direct version hint record that
- # for later configuration selection.
- shorthand = re.compile(r'^\d+\.\d+\.\d+')
- if shorthand.match(base_tree):
- self.base_tree_version = base_tree
- else:
- self.base_tree_version = None
-
- # Actually extract the tree. Make sure we know it occured
- self.extract(base_tree)
+ # Where we have direct version hint record that
+ # for later configuration selection.
+ shorthand = re.compile(r'^\d+\.\d+\.\d+')
+ if shorthand.match(base_tree):
+ self.base_tree_version = base_tree
+ else:
+ self.base_tree_version = None
+ # Actually extract the tree. Make sure we know it occured
+ self.extract(base_tree)
- def kernelexpand(self, kernel):
- # If we have something like a path, just use it as it is
- if '/' in kernel:
- return [kernel]
- # Find the configured mirror list.
- mirrors = self.job.config_get('mirror.mirrors')
- if not mirrors:
- # LEGACY: convert the kernel.org mirror
- mirror = self.job.config_get('mirror.ftp_kernel_org')
- if mirror:
- korg = 'http://www.kernel.org/pub/linux/kernel'
- mirrors = [
- [ korg + '/v2.6', mirror + '/v2.6' ],
- [ korg + '/people/akpm/patches/2.6',
- mirror + '/akpm' ],
- [ korg + '/people/mbligh',
- mirror + '/mbligh' ],
- ]
+ def kernelexpand(self, kernel):
+ # If we have something like a path, just use it as it is
+ if '/' in kernel:
+ return [kernel]
- patches = kernelexpand.expand_classic(kernel, mirrors)
- print patches
+ # Find the configured mirror list.
+ mirrors = self.job.config_get('mirror.mirrors')
+ if not mirrors:
+ # LEGACY: convert the kernel.org mirror
+ mirror = self.job.config_get('mirror.ftp_kernel_org')
+ if mirror:
+ korg = 'http://www.kernel.org/pub/linux/kernel'
+ mirrors = [
+ [ korg + '/v2.6', mirror + '/v2.6' ],
+ [ korg + '/people/akpm/patches/2.6',
+ mirror + '/akpm' ],
+ [ korg + '/people/mbligh',
+ mirror + '/mbligh' ],
+ ]
- return patches
+ patches = kernelexpand.expand_classic(kernel, mirrors)
+ print patches
+ return patches
- @logging.record
- @tee_output_logdir_mark
- def extract(self, base_tree):
- if os.path.exists(base_tree):
- self.get_kernel_tree(base_tree)
- else:
- base_components = self.kernelexpand(base_tree)
- print 'kernelexpand: '
- print base_components
- self.get_kernel_tree(base_components.pop(0))
- if base_components: # apply remaining patches
- self.patch(*base_components)
+ @logging.record
+ @tee_output_logdir_mark
+ def extract(self, base_tree):
+ if os.path.exists(base_tree):
+ self.get_kernel_tree(base_tree)
+ else:
+ base_components = self.kernelexpand(base_tree)
+ print 'kernelexpand: '
+ print base_components
+ self.get_kernel_tree(base_components.pop(0))
+ if base_components: # apply remaining patches
+ self.patch(*base_components)
- @logging.record
- @tee_output_logdir_mark
- def patch(self, *patches):
- """Apply a list of patches (in order)"""
- if not patches:
- return
- print 'Applying patches: ', patches
- self.apply_patches(self.get_patches(patches))
+ @logging.record
+ @tee_output_logdir_mark
+ def patch(self, *patches):
+ """Apply a list of patches (in order)"""
+ if not patches:
+ return
+ print 'Applying patches: ', patches
+ self.apply_patches(self.get_patches(patches))
- @logging.record
- @tee_output_logdir_mark
- def config(self, config_file = '', config_list = None, defconfig = False):
- self.set_cross_cc()
- config = kernel_config.kernel_config(self.job, self.build_dir,
- self.config_dir, config_file, config_list,
- defconfig, self.base_tree_version)
+ @logging.record
+ @tee_output_logdir_mark
+ def config(self, config_file = '', config_list = None, defconfig = False):
+ self.set_cross_cc()
+ config = kernel_config.kernel_config(self.job, self.build_dir,
+ self.config_dir, config_file, config_list,
+ defconfig, self.base_tree_version)
- def get_patches(self, patches):
- """fetch the patches to the local src_dir"""
- local_patches = []
- for patch in patches:
- dest = os.path.join(self.src_dir, basename(patch))
- # FIXME: this isn't unique. Append something to it
- # like wget does if it's not there?
- print "get_file %s %s %s %s" % (patch, dest, self.src_dir, basename(patch))
- utils.get_file(patch, dest)
- # probably safer to use the command, not python library
- md5sum = utils.system_output('md5sum ' + dest).split()[0]
- local_patches.append((patch, dest, md5sum))
- return local_patches
+ def get_patches(self, patches):
+ """fetch the patches to the local src_dir"""
+ local_patches = []
+ for patch in patches:
+ dest = os.path.join(self.src_dir, basename(patch))
+ # FIXME: this isn't unique. Append something to it
+ # like wget does if it's not there?
+ print "get_file %s %s %s %s" % (patch, dest, self.src_dir, basename(patch))
+ utils.get_file(patch, dest)
+ # probably safer to use the command, not python library
+ md5sum = utils.system_output('md5sum ' + dest).split()[0]
+ local_patches.append((patch, dest, md5sum))
+ return local_patches
- def apply_patches(self, local_patches):
- """apply the list of patches, in order"""
- builddir = self.build_dir
- os.chdir(builddir)
- if not local_patches:
- return None
- for (spec, local, md5sum) in local_patches:
- if local.endswith('.bz2') or local.endswith('.gz'):
- ref = spec
- else:
- ref = force_copy(local, self.results_dir)
- ref = self.job.relative_path(ref)
- patch_id = "%s %s %s" % (spec, ref, md5sum)
- log = "PATCH: " + patch_id + "\n"
- print log
- cat_file_to_cmd(local, 'patch -p1 > /dev/null')
- self.logfile.write(log)
- self.applied_patches.append(patch_id)
+ def apply_patches(self, local_patches):
+ """apply the list of patches, in order"""
+ builddir = self.build_dir
+ os.chdir(builddir)
+ if not local_patches:
+ return None
+ for (spec, local, md5sum) in local_patches:
+ if local.endswith('.bz2') or local.endswith('.gz'):
+ ref = spec
+ else:
+ ref = force_copy(local, self.results_dir)
+ ref = self.job.relative_path(ref)
+ patch_id = "%s %s %s" % (spec, ref, md5sum)
+ log = "PATCH: " + patch_id + "\n"
+ print log
+ cat_file_to_cmd(local, 'patch -p1 > /dev/null')
+ self.logfile.write(log)
+ self.applied_patches.append(patch_id)
- def get_kernel_tree(self, base_tree):
- """Extract/link base_tree to self.build_dir"""
-
- # if base_tree is a dir, assume uncompressed kernel
- if os.path.isdir(base_tree):
- print 'Symlinking existing kernel source'
- os.symlink(base_tree, self.build_dir)
- # otherwise, extract tarball
- else:
- os.chdir(os.path.dirname(self.src_dir))
- # Figure out local destination for tarball
- tarball = os.path.join(self.src_dir, os.path.basename(base_tree))
- utils.get_file(base_tree, tarball)
- print 'Extracting kernel tarball:', tarball, '...'
- autotest_utils.extract_tarball_to_dir(tarball,
- self.build_dir)
+ def get_kernel_tree(self, base_tree):
+ """Extract/link base_tree to self.build_dir"""
+ # if base_tree is a dir, assume uncompressed kernel
+ if os.path.isdir(base_tree):
+ print 'Symlinking existing kernel source'
+ os.symlink(base_tree, self.build_dir)
- def extraversion(self, tag, append=1):
- os.chdir(self.build_dir)
- extraversion_sub = r's/^EXTRAVERSION =\s*\(.*\)/EXTRAVERSION = '
- if append:
- p = extraversion_sub + '\\1-%s/' % tag
- else:
- p = extraversion_sub + '-%s/' % tag
- utils.system('mv Makefile Makefile.old')
- utils.system('sed "%s" < Makefile.old > Makefile' % p)
+ # otherwise, extract tarball
+ else:
+ os.chdir(os.path.dirname(self.src_dir))
+ # Figure out local destination for tarball
+ tarball = os.path.join(self.src_dir, os.path.basename(base_tree))
+ utils.get_file(base_tree, tarball)
+ print 'Extracting kernel tarball:', tarball, '...'
+ autotest_utils.extract_tarball_to_dir(tarball,
+ self.build_dir)
- @logging.record
- @tee_output_logdir_mark
- def build(self, make_opts = '', logfile = '', extraversion='autotest'):
- """build the kernel
+ def extraversion(self, tag, append=1):
+ os.chdir(self.build_dir)
+ extraversion_sub = r's/^EXTRAVERSION =\s*\(.*\)/EXTRAVERSION = '
+ if append:
+ p = extraversion_sub + '\\1-%s/' % tag
+ else:
+ p = extraversion_sub + '-%s/' % tag
+ utils.system('mv Makefile Makefile.old')
+ utils.system('sed "%s" < Makefile.old > Makefile' % p)
- make_opts
- additional options to make, if any
- """
- os_dep.commands('gcc', 'make')
- if logfile == '':
- logfile = os.path.join(self.log_dir, 'kernel_build')
- os.chdir(self.build_dir)
- if extraversion:
- self.extraversion(extraversion)
- self.set_cross_cc()
- # setup_config_file(config_file, config_overrides)
- # Not needed on 2.6, but hard to tell -- handle failure
- utils.system('make dep', ignore_status=True)
- threads = 2 * autotest_utils.count_cpus()
- build_string = 'make -j %d %s %s' % (threads, make_opts,
- self.build_target)
- # eg make bzImage, or make zImage
- print build_string
- system(build_string)
- if kernel_config.modules_needed('.config'):
- utils.system('make -j %d modules' % (threads))
+ @logging.record
+ @tee_output_logdir_mark
+ def build(self, make_opts = '', logfile = '', extraversion='autotest'):
+ """build the kernel
- kernel_version = self.get_kernel_build_ver()
- kernel_version = re.sub('-autotest', '', kernel_version)
- self.logfile.write('BUILD VERSION: %s\n' % kernel_version)
+ make_opts
+ additional options to make, if any
+ """
+ os_dep.commands('gcc', 'make')
+ if logfile == '':
+ logfile = os.path.join(self.log_dir, 'kernel_build')
+ os.chdir(self.build_dir)
+ if extraversion:
+ self.extraversion(extraversion)
+ self.set_cross_cc()
+ # setup_config_file(config_file, config_overrides)
- force_copy(self.build_dir+'/System.map', self.results_dir)
+ # Not needed on 2.6, but hard to tell -- handle failure
+ utils.system('make dep', ignore_status=True)
+ threads = 2 * autotest_utils.count_cpus()
+ build_string = 'make -j %d %s %s' % (threads, make_opts,
+ self.build_target)
+ # eg make bzImage, or make zImage
+ print build_string
+ system(build_string)
+ if kernel_config.modules_needed('.config'):
+ utils.system('make -j %d modules' % (threads))
+ kernel_version = self.get_kernel_build_ver()
+ kernel_version = re.sub('-autotest', '', kernel_version)
+ self.logfile.write('BUILD VERSION: %s\n' % kernel_version)
- def build_timed(self, threads, timefile = '/dev/null', make_opts = '',
- output = '/dev/null'):
- """time the bulding of the kernel"""
- os.chdir(self.build_dir)
- self.set_cross_cc()
+ force_copy(self.build_dir+'/System.map', self.results_dir)
- self.clean(logged=False)
- build_string = "/usr/bin/time -o %s make %s -j %s vmlinux" \
- % (timefile, make_opts, threads)
- build_string += ' > %s 2>&1' % output
- print build_string
- utils.system(build_string)
- if (not os.path.isfile('vmlinux')):
- errmsg = "no vmlinux found, kernel build failed"
- raise error.TestError(errmsg)
+ def build_timed(self, threads, timefile = '/dev/null', make_opts = '',
+ output = '/dev/null'):
+ """time the bulding of the kernel"""
+ os.chdir(self.build_dir)
+ self.set_cross_cc()
+
+ self.clean(logged=False)
+ build_string = "/usr/bin/time -o %s make %s -j %s vmlinux" \
+ % (timefile, make_opts, threads)
+ build_string += ' > %s 2>&1' % output
+ print build_string
+ utils.system(build_string)
+
+ if (not os.path.isfile('vmlinux')):
+ errmsg = "no vmlinux found, kernel build failed"
+ raise error.TestError(errmsg)
+
+
+ @logging.record
+ @tee_output_logdir_mark
+ def clean(self):
+ """make clean in the kernel tree"""
+ os.chdir(self.build_dir)
+ print "make clean"
+ utils.system('make clean > /dev/null 2> /dev/null')
+
+
+ @logging.record
+ @tee_output_logdir_mark
+ def mkinitrd(self, version, image, system_map, initrd):
+ """Build kernel initrd image.
+ Try to use distro specific way to build initrd image.
+ Parameters:
+ version
+ new kernel version
+ image
+ new kernel image file
+ system_map
+ System.map file
+ initrd
+ initrd image file to build
+ """
+ vendor = autotest_utils.get_os_vendor()
+ if os.path.isfile(initrd):
+ print "Existing %s file, will remove it." % initrd
+ os.remove(initrd)
- @logging.record
- @tee_output_logdir_mark
- def clean(self):
- """make clean in the kernel tree"""
- os.chdir(self.build_dir)
- print "make clean"
- utils.system('make clean > /dev/null 2> /dev/null')
+ args = self.job.config_get('kernel.mkinitrd_extra_args')
+ # don't leak 'None' into mkinitrd command
+ if not args:
+ args = ''
- @logging.record
- @tee_output_logdir_mark
- def mkinitrd(self, version, image, system_map, initrd):
- """Build kernel initrd image.
- Try to use distro specific way to build initrd image.
- Parameters:
- version
- new kernel version
- image
- new kernel image file
- system_map
- System.map file
- initrd
- initrd image file to build
- """
- vendor = autotest_utils.get_os_vendor()
-
- if os.path.isfile(initrd):
- print "Existing %s file, will remove it." % initrd
- os.remove(initrd)
+ if vendor in ['Red Hat', 'Fedora Core']:
+ utils.system('mkinitrd %s %s %s' % (args, initrd, version))
+ elif vendor in ['SUSE']:
+ utils.system('mkinitrd %s -k %s -i %s -M %s' % (args, image, initrd, system_map))
+ elif vendor in ['Debian', 'Ubuntu']:
+ if os.path.isfile('/usr/sbin/mkinitrd'):
+ cmd = '/usr/sbin/mkinitrd'
+ elif os.path.isfile('/usr/sbin/mkinitramfs'):
+ cmd = '/usr/sbin/mkinitramfs'
+ else:
+ raise error.TestError('No Debian initrd builder')
+ utils.system('%s %s -o %s %s' % (cmd, args, initrd, version))
+ else:
+ raise error.TestError('Unsupported vendor %s' % vendor)
- args = self.job.config_get('kernel.mkinitrd_extra_args')
- # don't leak 'None' into mkinitrd command
- if not args:
- args = ''
+ def set_build_image(self, image):
+ self.build_image = image
- if vendor in ['Red Hat', 'Fedora Core']:
- utils.system('mkinitrd %s %s %s' % (args, initrd, version))
- elif vendor in ['SUSE']:
- utils.system('mkinitrd %s -k %s -i %s -M %s' % (args, image, initrd, system_map))
- elif vendor in ['Debian', 'Ubuntu']:
- if os.path.isfile('/usr/sbin/mkinitrd'):
- cmd = '/usr/sbin/mkinitrd'
- elif os.path.isfile('/usr/sbin/mkinitramfs'):
- cmd = '/usr/sbin/mkinitramfs'
- else:
- raise error.TestError('No Debian initrd builder')
- utils.system('%s %s -o %s %s' % (cmd, args, initrd, version))
- else:
- raise error.TestError('Unsupported vendor %s' % vendor)
+ @logging.record
+ @tee_output_logdir_mark
+ def install(self, tag='autotest', prefix = '/'):
+ """make install in the kernel tree"""
- def set_build_image(self, image):
- self.build_image = image
+ # Record that we have installed the kernel, and
+ # the tag under which we installed it.
+ self.installed_as = tag
+ os.chdir(self.build_dir)
- @logging.record
- @tee_output_logdir_mark
- def install(self, tag='autotest', prefix = '/'):
- """make install in the kernel tree"""
+ if not os.path.isdir(prefix):
+ os.mkdir(prefix)
+ self.boot_dir = os.path.join(prefix, 'boot')
+ if not os.path.isdir(self.boot_dir):
+ os.mkdir(self.boot_dir)
- # Record that we have installed the kernel, and
- # the tag under which we installed it.
- self.installed_as = tag
+ if not self.build_image:
+ images = glob.glob('arch/*/boot/' + self.build_target)
+ if len(images):
+ self.build_image = images[0]
+ else:
+ self.build_image = self.build_target
- os.chdir(self.build_dir)
+ # remember installed files
+ self.vmlinux = self.boot_dir + '/vmlinux-' + tag
+ if (self.build_image != 'vmlinux'):
+ self.image = self.boot_dir + '/vmlinuz-' + tag
+ else:
+ self.image = self.vmlinux
+ self.system_map = self.boot_dir + '/System.map-' + tag
+ self.config = self.boot_dir + '/config-' + tag
+ self.initrd = ''
- if not os.path.isdir(prefix):
- os.mkdir(prefix)
- self.boot_dir = os.path.join(prefix, 'boot')
- if not os.path.isdir(self.boot_dir):
- os.mkdir(self.boot_dir)
+ # copy to boot dir
+ autotest_utils.force_copy('vmlinux', self.vmlinux)
+ if (self.build_image != 'vmlinux'):
+ force_copy(self.build_image, self.image)
+ autotest_utils.force_copy('System.map', self.system_map)
+ autotest_utils.force_copy('.config', self.config)
- if not self.build_image:
- images = glob.glob('arch/*/boot/' + self.build_target)
- if len(images):
- self.build_image = images[0]
- else:
- self.build_image = self.build_target
+ if not kernel_config.modules_needed('.config'):
+ return
- # remember installed files
- self.vmlinux = self.boot_dir + '/vmlinux-' + tag
- if (self.build_image != 'vmlinux'):
- self.image = self.boot_dir + '/vmlinuz-' + tag
- else:
- self.image = self.vmlinux
- self.system_map = self.boot_dir + '/System.map-' + tag
- self.config = self.boot_dir + '/config-' + tag
- self.initrd = ''
+ utils.system('make modules_install INSTALL_MOD_PATH=%s' % prefix)
+ if prefix == '/':
+ self.initrd = self.boot_dir + '/initrd-' + tag
+ self.mkinitrd(self.get_kernel_build_ver(), self.image,
+ self.system_map, self.initrd)
- # copy to boot dir
- autotest_utils.force_copy('vmlinux', self.vmlinux)
- if (self.build_image != 'vmlinux'):
- force_copy(self.build_image, self.image)
- autotest_utils.force_copy('System.map', self.system_map)
- autotest_utils.force_copy('.config', self.config)
- if not kernel_config.modules_needed('.config'):
- return
+ def add_to_bootloader(self, tag='autotest', args=''):
+ """ add this kernel to bootloader, taking an
+ optional parameter of space separated parameters
+ e.g.: kernel.add_to_bootloader('mykernel', 'ro acpi=off')
+ """
- utils.system('make modules_install INSTALL_MOD_PATH=%s' % prefix)
- if prefix == '/':
- self.initrd = self.boot_dir + '/initrd-' + tag
- self.mkinitrd(self.get_kernel_build_ver(), self.image,
- self.system_map, self.initrd)
+ # remove existing entry if present
+ self.job.bootloader.remove_kernel(tag)
+ # pull the base argument set from the job config,
+ baseargs = self.job.config_get('boot.default_args')
+ if baseargs:
+ args = baseargs + " " + args
- def add_to_bootloader(self, tag='autotest', args=''):
- """ add this kernel to bootloader, taking an
- optional parameter of space separated parameters
- e.g.: kernel.add_to_bootloader('mykernel', 'ro acpi=off')
- """
+ # otherwise populate from /proc/cmdline
+ # if not baseargs:
+ # baseargs = open('/proc/cmdline', 'r').readline().strip()
+ # NOTE: This is unnecessary, because boottool does it.
- # remove existing entry if present
- self.job.bootloader.remove_kernel(tag)
+ root = None
+ roots = [x for x in args.split() if x.startswith('root=')]
+ if roots:
+ root = re.sub('^root=', '', roots[0])
+ arglist = [x for x in args.split() if not x.startswith('root=')]
+ args = ' '.join(arglist)
- # pull the base argument set from the job config,
- baseargs = self.job.config_get('boot.default_args')
- if baseargs:
- args = baseargs + " " + args
-
- # otherwise populate from /proc/cmdline
- # if not baseargs:
- # baseargs = open('/proc/cmdline', 'r').readline().strip()
- # NOTE: This is unnecessary, because boottool does it.
+ # add the kernel entry
+ # add_kernel(image, title='autotest', initrd='')
+ self.job.bootloader.add_kernel(self.image, tag, self.initrd, \
+ args = args, root = root)
- root = None
- roots = [x for x in args.split() if x.startswith('root=')]
- if roots:
- root = re.sub('^root=', '', roots[0])
- arglist = [x for x in args.split() if not x.startswith('root=')]
- args = ' '.join(arglist)
- # add the kernel entry
- # add_kernel(image, title='autotest', initrd='')
- self.job.bootloader.add_kernel(self.image, tag, self.initrd, \
- args = args, root = root)
+ def get_kernel_build_arch(self, arch=None):
+ """
+ Work out the current kernel architecture (as a kernel arch)
+ """
+ if not arch:
+ arch = autotest_utils.get_current_kernel_arch()
+ if re.match('i.86', arch):
+ return 'i386'
+ elif re.match('sun4u', arch):
+ return 'sparc64'
+ elif re.match('arm.*', arch):
+ return 'arm'
+ elif re.match('sa110', arch):
+ return 'arm'
+ elif re.match('s390x', arch):
+ return 's390'
+ elif re.match('parisc64', arch):
+ return 'parisc'
+ elif re.match('ppc.*', arch):
+ return 'powerpc'
+ elif re.match('mips.*', arch):
+ return 'mips'
+ else:
+ return arch
- def get_kernel_build_arch(self, arch=None):
- """
- Work out the current kernel architecture (as a kernel arch)
- """
- if not arch:
- arch = autotest_utils.get_current_kernel_arch()
- if re.match('i.86', arch):
- return 'i386'
- elif re.match('sun4u', arch):
- return 'sparc64'
- elif re.match('arm.*', arch):
- return 'arm'
- elif re.match('sa110', arch):
- return 'arm'
- elif re.match('s390x', arch):
- return 's390'
- elif re.match('parisc64', arch):
- return 'parisc'
- elif re.match('ppc.*', arch):
- return 'powerpc'
- elif re.match('mips.*', arch):
- return 'mips'
- else:
- return arch
+ def get_kernel_build_release(self):
+ releasem = re.compile(r'.*UTS_RELEASE\s+"([^"]+)".*');
+ versionm = re.compile(r'.*UTS_VERSION\s+"([^"]+)".*');
+ release = None
+ version = None
- def get_kernel_build_release(self):
- releasem = re.compile(r'.*UTS_RELEASE\s+"([^"]+)".*');
- versionm = re.compile(r'.*UTS_VERSION\s+"([^"]+)".*');
+ for file in [ self.build_dir + "/include/linux/version.h",
+ self.build_dir + "/include/linux/utsrelease.h",
+ self.build_dir + "/include/linux/compile.h" ]:
+ if os.path.exists(file):
+ fd = open(file, 'r')
+ for line in fd.readlines():
+ m = releasem.match(line)
+ if m:
+ release = m.groups()[0]
+ m = versionm.match(line)
+ if m:
+ version = m.groups()[0]
+ fd.close()
- release = None
- version = None
+ return (release, version)
- for file in [ self.build_dir + "/include/linux/version.h",
- self.build_dir + "/include/linux/utsrelease.h",
- self.build_dir + "/include/linux/compile.h" ]:
- if os.path.exists(file):
- fd = open(file, 'r')
- for line in fd.readlines():
- m = releasem.match(line)
- if m:
- release = m.groups()[0]
- m = versionm.match(line)
- if m:
- version = m.groups()[0]
- fd.close()
- return (release, version)
+ def get_kernel_build_ident(self):
+ (release, version) = self.get_kernel_build_release()
-
- def get_kernel_build_ident(self):
- (release, version) = self.get_kernel_build_release()
+ if not release or not version:
+ raise error.JobError('kernel has no identity')
- if not release or not version:
- raise error.JobError('kernel has no identity')
+ return release + '::' + version
- return release + '::' + version
+ def boot(self, args='', ident=1):
+ """ install and boot this kernel, do not care how
+ just make it happen.
+ """
- def boot(self, args='', ident=1):
- """ install and boot this kernel, do not care how
- just make it happen.
- """
+ # If we can check the kernel identity do so.
+ if ident:
+ when = int(time.time())
+ ident = self.get_kernel_build_ident()
+ args += " IDENT=%d" % (when)
- # If we can check the kernel identity do so.
- if ident:
- when = int(time.time())
- ident = self.get_kernel_build_ident()
- args += " IDENT=%d" % (when)
+ self.job.next_step_prepend(["job.kernel_check_ident",
+ when, ident, self.subdir,
+ self.applied_patches])
- self.job.next_step_prepend(["job.kernel_check_ident",
- when, ident, self.subdir,
- self.applied_patches])
+ # Check if the kernel has been installed, if not install
+ # as the default tag and boot that.
+ if not self.installed_as:
+ self.install()
- # Check if the kernel has been installed, if not install
- # as the default tag and boot that.
- if not self.installed_as:
- self.install()
+ # Boot the selected tag.
+ self.add_to_bootloader(args=args, tag=self.installed_as)
- # Boot the selected tag.
- self.add_to_bootloader(args=args, tag=self.installed_as)
+ # Boot it.
+ self.job.reboot(tag=self.installed_as)
- # Boot it.
- self.job.reboot(tag=self.installed_as)
+ def get_kernel_build_ver(self):
+ """Check Makefile and .config to return kernel version"""
+ version = patchlevel = sublevel = extraversion = localversion = ''
- def get_kernel_build_ver(self):
- """Check Makefile and .config to return kernel version"""
- version = patchlevel = sublevel = extraversion = localversion = ''
+ for line in open(self.build_dir + '/Makefile', 'r').readlines():
+ if line.startswith('VERSION'):
+ version = line[line.index('=') + 1:].strip()
+ if line.startswith('PATCHLEVEL'):
+ patchlevel = line[line.index('=') + 1:].strip()
+ if line.startswith('SUBLEVEL'):
+ sublevel = line[line.index('=') + 1:].strip()
+ if line.startswith('EXTRAVERSION'):
+ extraversion = line[line.index('=') + 1:].strip()
- for line in open(self.build_dir + '/Makefile', 'r').readlines():
- if line.startswith('VERSION'):
- version = line[line.index('=') + 1:].strip()
- if line.startswith('PATCHLEVEL'):
- patchlevel = line[line.index('=') + 1:].strip()
- if line.startswith('SUBLEVEL'):
- sublevel = line[line.index('=') + 1:].strip()
- if line.startswith('EXTRAVERSION'):
- extraversion = line[line.index('=') + 1:].strip()
+ for line in open(self.build_dir + '/.config', 'r').readlines():
+ if line.startswith('CONFIG_LOCALVERSION='):
+ localversion = line.rstrip().split('"')[1]
- for line in open(self.build_dir + '/.config', 'r').readlines():
- if line.startswith('CONFIG_LOCALVERSION='):
- localversion = line.rstrip().split('"')[1]
+ return "%s.%s.%s%s%s" %(version, patchlevel, sublevel, extraversion, localversion)
- return "%s.%s.%s%s%s" %(version, patchlevel, sublevel, extraversion, localversion)
+ def set_build_target(self, build_target):
+ if build_target:
+ self.build_target = build_target
+ print 'BUILD TARGET: %s' % self.build_target
- def set_build_target(self, build_target):
- if build_target:
- self.build_target = build_target
- print 'BUILD TARGET: %s' % self.build_target
+ def set_cross_cc(self, target_arch=None, cross_compile=None,
+ build_target='bzImage'):
+ """Set up to cross-compile.
+ This is broken. We need to work out what the default
+ compile produces, and if not, THEN set the cross
+ compiler.
+ """
- def set_cross_cc(self, target_arch=None, cross_compile=None,
- build_target='bzImage'):
- """Set up to cross-compile.
- This is broken. We need to work out what the default
- compile produces, and if not, THEN set the cross
- compiler.
- """
+ if self.target_arch:
+ return
- if self.target_arch:
- return
+ # if someone has set build_target, don't clobber in set_cross_cc
+ # run set_build_target before calling set_cross_cc
+ if not self.build_target:
+ self.set_build_target(build_target)
- # if someone has set build_target, don't clobber in set_cross_cc
- # run set_build_target before calling set_cross_cc
- if not self.build_target:
- self.set_build_target(build_target)
+ # If no 'target_arch' given assume native compilation
+ if target_arch == None:
+ target_arch = autotest_utils.get_current_kernel_arch()
+ if target_arch == 'ppc64':
+ if self.build_target == 'bzImage':
+ self.build_target = 'vmlinux'
- # If no 'target_arch' given assume native compilation
- if target_arch == None:
- target_arch = autotest_utils.get_current_kernel_arch()
- if target_arch == 'ppc64':
- if self.build_target == 'bzImage':
- self.build_target = 'vmlinux'
+ if not cross_compile:
+ cross_compile = self.job.config_get('kernel.cross_cc')
- if not cross_compile:
- cross_compile = self.job.config_get('kernel.cross_cc')
+ if cross_compile:
+ os.environ['CROSS_COMPILE'] = cross_compile
+ else:
+ if os.environ.has_key('CROSS_COMPILE'):
+ del os.environ['CROSS_COMPILE']
- if cross_compile:
- os.environ['CROSS_COMPILE'] = cross_compile
- else:
- if os.environ.has_key('CROSS_COMPILE'):
- del os.environ['CROSS_COMPILE']
-
- return # HACK. Crap out for now.
+ return # HACK. Crap out for now.
- # At this point I know what arch I *want* to build for
- # but have no way of working out what arch the default
- # compiler DOES build for.
+ # At this point I know what arch I *want* to build for
+ # but have no way of working out what arch the default
+ # compiler DOES build for.
- # Oh, and BTW, install_package() doesn't exist yet.
+ # Oh, and BTW, install_package() doesn't exist yet.
- if target_arch == 'ppc64':
- install_package('ppc64-cross')
- cross_compile = os.path.join(self.autodir, 'sources/ppc64-cross/bin')
+ if target_arch == 'ppc64':
+ install_package('ppc64-cross')
+ cross_compile = os.path.join(self.autodir, 'sources/ppc64-cross/bin')
- elif target_arch == 'x86_64':
- install_package('x86_64-cross')
- cross_compile = os.path.join(self.autodir, 'sources/x86_64-cross/bin')
+ elif target_arch == 'x86_64':
+ install_package('x86_64-cross')
+ cross_compile = os.path.join(self.autodir, 'sources/x86_64-cross/bin')
- os.environ['ARCH'] = self.target_arch = target_arch
+ os.environ['ARCH'] = self.target_arch = target_arch
- self.cross_compile = cross_compile
- if self.cross_compile:
- os.environ['CROSS_COMPILE'] = self.cross_compile
+ self.cross_compile = cross_compile
+ if self.cross_compile:
+ os.environ['CROSS_COMPILE'] = self.cross_compile
- def pickle_dump(self, filename):
- """dump a pickle of ourself out to the specified filename
+ def pickle_dump(self, filename):
+ """dump a pickle of ourself out to the specified filename
- we can't pickle the backreference to job (it contains fd's),
- nor would we want to. Same for logfile (fd's).
- """
- temp = copy.copy(self)
- temp.job = None
- temp.logfile = None
- pickle.dump(temp, open(filename, 'w'))
+ we can't pickle the backreference to job (it contains fd's),
+ nor would we want to. Same for logfile (fd's).
+ """
+ temp = copy.copy(self)
+ temp.job = None
+ temp.logfile = None
+ pickle.dump(temp, open(filename, 'w'))
class rpm_kernel:
- """ Class for installing rpm kernel package
- """
+ """ Class for installing rpm kernel package
+ """
- def __init__(self, job, rpm_package, subdir):
- self.job = job
- self.rpm_package = rpm_package
- self.log_dir = os.path.join(subdir, 'debug')
- self.subdir = os.path.basename(subdir)
- if os.path.exists(self.log_dir):
- utils.system('rm -rf ' + self.log_dir)
- os.mkdir(self.log_dir)
- self.installed_as = None
+ def __init__(self, job, rpm_package, subdir):
+ self.job = job
+ self.rpm_package = rpm_package
+ self.log_dir = os.path.join(subdir, 'debug')
+ self.subdir = os.path.basename(subdir)
+ if os.path.exists(self.log_dir):
+ utils.system('rm -rf ' + self.log_dir)
+ os.mkdir(self.log_dir)
+ self.installed_as = None
- @logging.record
- @tee_output_logdir_mark
- def install(self, tag='autotest'):
- self.installed_as = tag
+ @logging.record
+ @tee_output_logdir_mark
+ def install(self, tag='autotest'):
+ self.installed_as = tag
- self.rpm_name = utils.system_output('rpm -qp ' + self.rpm_package)
+ self.rpm_name = utils.system_output('rpm -qp ' + self.rpm_package)
- # install
- utils.system('rpm -i --force ' + self.rpm_package)
+ # install
+ utils.system('rpm -i --force ' + self.rpm_package)
- # get file list
- files = utils.system_output('rpm -ql ' + self.rpm_name).splitlines()
+ # get file list
+ files = utils.system_output('rpm -ql ' + self.rpm_name).splitlines()
- # search for vmlinuz
- for file in files:
- if file.startswith('/boot/vmlinuz'):
- self.image = file
- break
- else:
- errmsg = "%s doesn't contain /boot/vmlinuz"
- errmsg %= self.rpm_package
- raise error.TestError(errmsg)
+ # search for vmlinuz
+ for file in files:
+ if file.startswith('/boot/vmlinuz'):
+ self.image = file
+ break
+ else:
+ errmsg = "%s doesn't contain /boot/vmlinuz"
+ errmsg %= self.rpm_package
+ raise error.TestError(errmsg)
- # search for initrd
- self.initrd = ''
- for file in files:
- if file.startswith('/boot/initrd'):
- self.initrd = file
- break
+ # search for initrd
+ self.initrd = ''
+ for file in files:
+ if file.startswith('/boot/initrd'):
+ self.initrd = file
+ break
- # get version and release number
- self.version, self.release = utils.system_output(
- 'rpm --queryformat="%{VERSION}\\n%{RELEASE}\\n" -q ' + self.rpm_name).splitlines()[0:2]
+ # get version and release number
+ self.version, self.release = utils.system_output(
+ 'rpm --queryformat="%{VERSION}\\n%{RELEASE}\\n" -q ' + self.rpm_name).splitlines()[0:2]
- def add_to_bootloader(self, tag='autotest', args=''):
- """ Add this kernel to bootloader
- """
+ def add_to_bootloader(self, tag='autotest', args=''):
+ """ Add this kernel to bootloader
+ """
- # remove existing entry if present
- self.job.bootloader.remove_kernel(tag)
+ # remove existing entry if present
+ self.job.bootloader.remove_kernel(tag)
- # pull the base argument set from the job config
- baseargs = self.job.config_get('boot.default_args')
- if baseargs:
- args = baseargs + ' ' + args
+ # pull the base argument set from the job config
+ baseargs = self.job.config_get('boot.default_args')
+ if baseargs:
+ args = baseargs + ' ' + args
- # otherwise populate from /proc/cmdline
- # if not baseargs:
- # baseargs = open('/proc/cmdline', 'r').readline().strip()
- # NOTE: This is unnecessary, because boottool does it.
+ # otherwise populate from /proc/cmdline
+ # if not baseargs:
+ # baseargs = open('/proc/cmdline', 'r').readline().strip()
+ # NOTE: This is unnecessary, because boottool does it.
- root = None
- roots = [x for x in args.split() if x.startswith('root=')]
- if roots:
- root = re.sub('^root=', '', roots[0])
- arglist = [x for x in args.split() if not x.startswith('root=')]
- args = ' '.join(arglist)
+ root = None
+ roots = [x for x in args.split() if x.startswith('root=')]
+ if roots:
+ root = re.sub('^root=', '', roots[0])
+ arglist = [x for x in args.split() if not x.startswith('root=')]
+ args = ' '.join(arglist)
- # add the kernel entry
- self.job.bootloader.add_kernel(self.image, tag, self.initrd, args = args, root = root)
+ # add the kernel entry
+ self.job.bootloader.add_kernel(self.image, tag, self.initrd, args = args, root = root)
- def boot(self, args='', ident=1):
- """ install and boot this kernel
- """
+ def boot(self, args='', ident=1):
+ """ install and boot this kernel
+ """
- # Check if the kernel has been installed, if not install
- # as the default tag and boot that.
- if not self.installed_as:
- self.install()
+ # Check if the kernel has been installed, if not install
+ # as the default tag and boot that.
+ if not self.installed_as:
+ self.install()
- # If we can check the kernel identity do so.
- if ident:
- when = int(time.time())
- ident = '-'.join([self.version,
- self.rpm_name.split('-')[1],
- self.release])
- args += " IDENT=%d" % (when)
+ # If we can check the kernel identity do so.
+ if ident:
+ when = int(time.time())
+ ident = '-'.join([self.version,
+ self.rpm_name.split('-')[1],
+ self.release])
+ args += " IDENT=%d" % (when)
- self.job.next_step_prepend(["job.kernel_check_ident",
- when, ident, self.subdir, 'rpm'])
+ self.job.next_step_prepend(["job.kernel_check_ident",
+ when, ident, self.subdir, 'rpm'])
- # Boot the selected tag.
- self.add_to_bootloader(args=args, tag=self.installed_as)
+ # Boot the selected tag.
+ self.add_to_bootloader(args=args, tag=self.installed_as)
- # Boot it.
- self.job.reboot(tag=self.installed_as)
+ # Boot it.
+ self.job.reboot(tag=self.installed_as)
# pull in some optional site-specific path pre-processing
try:
- import site_kernel
- preprocess_path = site_kernel.preprocess_path
- del site_kernel
+ import site_kernel
+ preprocess_path = site_kernel.preprocess_path
+ del site_kernel
except ImportError:
- # just make the preprocessor a nop
- def preprocess_path(path):
- return path
+ # just make the preprocessor a nop
+ def preprocess_path(path):
+ return path
def auto_kernel(job, path, subdir, tmp_dir, build_dir, leave=False):
- """\
- Create a kernel object, dynamically selecting the appropriate class to use
- based on the path provided.
- """
- path = preprocess_path(path)
- if path.endswith('.rpm'):
- return rpm_kernel(job, path, subdir)
- else:
- return kernel(job, path, subdir, tmp_dir, build_dir, leave)
+ """\
+ Create a kernel object, dynamically selecting the appropriate class to use
+ based on the path provided.
+ """
+ path = preprocess_path(path)
+ if path.endswith('.rpm'):
+ return rpm_kernel(job, path, subdir)
+ else:
+ return kernel(job, path, subdir, tmp_dir, build_dir, leave)
diff --git a/client/bin/kernel_config.py b/client/bin/kernel_config.py
index ed6321a..9b471b5 100755
--- a/client/bin/kernel_config.py
+++ b/client/bin/kernel_config.py
@@ -5,122 +5,122 @@
from autotest_lib.client.common_lib import error, utils
def apply_overrides(orig_file, changes_file, output_file):
- override = dict()
+ override = dict()
- # First suck all the changes into a dictionary.
- input = file(changes_file, 'r')
- for line in input.readlines():
- if line.startswith('CONFIG_'):
- key = line.split('=')[0]
- override[key] = line;
- elif line.startswith('# CONFIG_'):
- key = line.split(' ')[1]
- override[key] = line;
- input.close()
+ # First suck all the changes into a dictionary.
+ input = file(changes_file, 'r')
+ for line in input.readlines():
+ if line.startswith('CONFIG_'):
+ key = line.split('=')[0]
+ override[key] = line;
+ elif line.startswith('# CONFIG_'):
+ key = line.split(' ')[1]
+ override[key] = line;
+ input.close()
- # Now go through the input file, overriding lines where need be
- input = file(orig_file, 'r')
- output = file(output_file, 'w')
- for line in input.readlines():
- if line.startswith('CONFIG_'):
- key = line.split('=')[0]
- elif line.startswith('# CONFIG_'):
- key = line.split(' ')[1]
- else:
- key = None
- if key and key in override:
- output.write(override[key])
- else:
- output.write(line)
- input.close()
- output.close()
+ # Now go through the input file, overriding lines where need be
+ input = file(orig_file, 'r')
+ output = file(output_file, 'w')
+ for line in input.readlines():
+ if line.startswith('CONFIG_'):
+ key = line.split('=')[0]
+ elif line.startswith('# CONFIG_'):
+ key = line.split(' ')[1]
+ else:
+ key = None
+ if key and key in override:
+ output.write(override[key])
+ else:
+ output.write(line)
+ input.close()
+ output.close()
def diff_configs(old, new):
- utils.system('diff -u %s %s > %s' % (old, new, new + '.diff'),
- ignore_status=True)
+ utils.system('diff -u %s %s > %s' % (old, new, new + '.diff'),
+ ignore_status=True)
def modules_needed(config):
- return (autotest_utils.grep('CONFIG_MODULES=y', config)
- and autotest_utils.grep('=m', config))
+ return (autotest_utils.grep('CONFIG_MODULES=y', config)
+ and autotest_utils.grep('=m', config))
def config_by_name(name, set):
- version = kernel_versions.version_choose_config(name, set[1:])
- if version:
- return set[0] + version
- return None
+ version = kernel_versions.version_choose_config(name, set[1:])
+ if version:
+ return set[0] + version
+ return None
class kernel_config:
- # Build directory must be ready before init'ing config.
- #
- # Stages:
- # 1. Get original config file
- # 2. Apply overrides
- # 3. Do 'make oldconfig' to update it to current source code
- # (gets done implicitly during the process)
- #
- # You may specifiy the a defconfig within the tree to build,
- # or a custom config file you want, or None, to get machine's
- # default config file from the repo.
+ # Build directory must be ready before init'ing config.
+ #
+ # Stages:
+ # 1. Get original config file
+ # 2. Apply overrides
+ # 3. Do 'make oldconfig' to update it to current source code
+ # (gets done implicitly during the process)
+ #
+ # You may specifiy the a defconfig within the tree to build,
+ # or a custom config file you want, or None, to get machine's
+ # default config file from the repo.
- build_dir = '' # the directory we're building in
- config_dir = '' # local repository for config_file data
+ build_dir = '' # the directory we're building in
+ config_dir = '' # local repository for config_file data
- build_config = '' # the config file in the build directory
- orig_config = '' # the original config file
- over_config = '' # config file + overrides
+ build_config = '' # the config file in the build directory
+ orig_config = '' # the original config file
+ over_config = '' # config file + overrides
- def __init__(self, job, build_dir, config_dir, orig_file,
- overrides, defconfig = False, name = None):
- self.build_dir = build_dir
- self.config_dir = config_dir
+ def __init__(self, job, build_dir, config_dir, orig_file,
+ overrides, defconfig = False, name = None):
+ self.build_dir = build_dir
+ self.config_dir = config_dir
- # 1. Get original config file
- self.build_config = build_dir + '/.config'
- if (orig_file == '' and not defconfig): # use user default
- set = job.config_get("kernel.default_config_set")
- defconf = None
- if set and name:
- defconf = config_by_name(name, set)
- if not defconf:
- defconf = job.config_get("kernel.default_config")
- if defconf:
- orig_file = defconf
- if (orig_file == '' or defconfig): # use defconfig
- print "kernel_config: using defconfig to configure kernel"
- os.chdir(build_dir)
- utils.system('make defconfig')
- else:
- print "kernel_config: using " + orig_file + \
- " to configure kernel"
- self.orig_config = config_dir + '/config.orig'
- utils.get_file(orig_file, self.orig_config)
- self.update_config(self.orig_config, self.orig_config+'.new')
- diff_configs(self.orig_config, self.orig_config+'.new')
+ # 1. Get original config file
+ self.build_config = build_dir + '/.config'
+ if (orig_file == '' and not defconfig): # use user default
+ set = job.config_get("kernel.default_config_set")
+ defconf = None
+ if set and name:
+ defconf = config_by_name(name, set)
+ if not defconf:
+ defconf = job.config_get("kernel.default_config")
+ if defconf:
+ orig_file = defconf
+ if (orig_file == '' or defconfig): # use defconfig
+ print "kernel_config: using defconfig to configure kernel"
+ os.chdir(build_dir)
+ utils.system('make defconfig')
+ else:
+ print "kernel_config: using " + orig_file + \
+ " to configure kernel"
+ self.orig_config = config_dir + '/config.orig'
+ utils.get_file(orig_file, self.orig_config)
+ self.update_config(self.orig_config, self.orig_config+'.new')
+ diff_configs(self.orig_config, self.orig_config+'.new')
- # 2. Apply overrides
- if overrides:
- print "kernel_config: using " + overrides + \
- " to re-configure kernel"
- self.over_config = config_dir + '/config.over'
- overrides_local = self.over_config + '.changes'
- get_file(overrides, overrides_local)
- apply_overrides(self.build_config, overrides_local, self.over_config)
- self.update_config(self.over_config, self.over_config+'.new')
- diff_configs(self.over_config, self.over_config+'.new')
- else:
- self.over_config = self.orig_config
+ # 2. Apply overrides
+ if overrides:
+ print "kernel_config: using " + overrides + \
+ " to re-configure kernel"
+ self.over_config = config_dir + '/config.over'
+ overrides_local = self.over_config + '.changes'
+ get_file(overrides, overrides_local)
+ apply_overrides(self.build_config, overrides_local, self.over_config)
+ self.update_config(self.over_config, self.over_config+'.new')
+ diff_configs(self.over_config, self.over_config+'.new')
+ else:
+ self.over_config = self.orig_config
- def update_config(self, old_config, new_config = 'None'):
- os.chdir(self.build_dir)
- shutil.copyfile(old_config, self.build_config)
- utils.system('yes "" | make oldconfig > /dev/null')
- if new_config:
- shutil.copyfile(self.build_config, new_config)
+ def update_config(self, old_config, new_config = 'None'):
+ os.chdir(self.build_dir)
+ shutil.copyfile(old_config, self.build_config)
+ utils.system('yes "" | make oldconfig > /dev/null')
+ if new_config:
+ shutil.copyfile(self.build_config, new_config)
diff --git a/client/bin/kernel_versions.py b/client/bin/kernel_versions.py
index 3af08fe..9b04beb 100644
--- a/client/bin/kernel_versions.py
+++ b/client/bin/kernel_versions.py
@@ -5,111 +5,109 @@
import sys,re
-#
+#
# Sort key for ordering versions chronologically. The key ordering
# problem is between that introduced by -rcN. These come _before_
# their accompanying version.
-#
+#
# 2.6.0 -> 2.6.1-rc1 -> 2.6.1
-#
+#
# In order to sort them we convert all non-rc releases to a pseudo
# -rc99 release. We also convert all numbers to two digits. The
# result is then sortable textually.
-#
+#
# 02.06.00-rc99 -> 02.06.01-rc01 -> 02.06.01-rc99
-#
+#
encode_sep = re.compile(r'(\D+)')
def version_encode(version):
- bits = encode_sep.split(version)
- n = 9
- if len(bits[0]) == 0:
- n += 2
- if len(bits) == n or (len(bits) > n and bits[n] != '_rc'):
- # Insert missing _rc99 after 2 . 6 . 18 -smp- 220 . 0
- bits.insert(n, '_rc')
- bits.insert(n+1, '99')
- n = 5
- if len(bits[0]) == 0:
- n += 2
- if len(bits) <= n or bits[n] != '-rc':
- bits.insert(n, '-rc')
- bits.insert(n+1, '99')
- for n in range(0, len(bits), 2):
- if len(bits[n]) == 1:
- bits[n] = '0' + bits[n]
+ bits = encode_sep.split(version)
+ n = 9
+ if len(bits[0]) == 0:
+ n += 2
+ if len(bits) == n or (len(bits) > n and bits[n] != '_rc'):
+ # Insert missing _rc99 after 2 . 6 . 18 -smp- 220 . 0
+ bits.insert(n, '_rc')
+ bits.insert(n+1, '99')
+ n = 5
+ if len(bits[0]) == 0:
+ n += 2
+ if len(bits) <= n or bits[n] != '-rc':
+ bits.insert(n, '-rc')
+ bits.insert(n+1, '99')
+ for n in range(0, len(bits), 2):
+ if len(bits[n]) == 1:
+ bits[n] = '0' + bits[n]
- return ''.join(bits)
+ return ''.join(bits)
def version_limit(version, n):
- bits = encode_sep.split(version)
- return ''.join(bits[0:n])
+ bits = encode_sep.split(version)
+ return ''.join(bits[0:n])
def version_len(version):
- return len(encode_sep.split(version))
+ return len(encode_sep.split(version))
#
# Given a list of versions find the nearest version which is deemed
# less than or equal to the target. Versions are in linux order
# as follows:
-#
+#
# 2.6.0 -> 2.6.1 -> 2.6.2-rc1 -> 2.6.2-rc2 -> 2.6.2 -> 2.6.3-rc1
# | |\
# | | 2.6.2-rc1-mm1 -> 2.6.2-rc1-mm2
# | \
# | 2.6.2-rc1-ac1 -> 2.6.2-rc1-ac2
-# \
+# \
# 2.6.1-mm1 -> 2.6.1-mm2
-#
+#
# Note that a 2.6.1-mm1 is not a predecessor of 2.6.2-rc1-mm1.
#
def version_choose_config(version, candidates):
- # Check if we have an exact match ... if so magic
- if version in candidates:
- return version
+ # Check if we have an exact match ... if so magic
+ if version in candidates:
+ return version
- # Sort the search key into the list ordered by 'age'
- deco = [ (version_encode(v), i, v) for i, v in
- enumerate(candidates + [ version ]) ]
- deco.sort()
- versions = [ v for _, _, v in deco ]
+ # Sort the search key into the list ordered by 'age'
+ deco = [ (version_encode(v), i, v) for i, v in
+ enumerate(candidates + [ version ]) ]
+ deco.sort()
+ versions = [ v for _, _, v in deco ]
- # Everything sorted below us is of interst.
- for n in range(len(versions) - 1, -1, -1):
- if versions[n] == version:
- break
- n -= 1
+ # Everything sorted below us is of interst.
+ for n in range(len(versions) - 1, -1, -1):
+ if versions[n] == version:
+ break
+ n -= 1
- # Try ever shorter 'prefixes' 2.6.20-rc3-mm, 2.6.20-rc, 2.6. etc
- # to match against the ordered list newest to oldest.
- length = version_len(version) - 1
- version = version_limit(version, length)
- while length > 1:
- for o in range(n, -1, -1):
- if version_len(versions[o]) == (length + 1) and \
- version_limit(versions[o], length) == version:
- return versions[o]
- length -= 2
- version = version_limit(version, length)
+ # Try ever shorter 'prefixes' 2.6.20-rc3-mm, 2.6.20-rc, 2.6. etc
+ # to match against the ordered list newest to oldest.
+ length = version_len(version) - 1
+ version = version_limit(version, length)
+ while length > 1:
+ for o in range(n, -1, -1):
+ if version_len(versions[o]) == (length + 1) and \
+ version_limit(versions[o], length) == version:
+ return versions[o]
+ length -= 2
+ version = version_limit(version, length)
- return None
+ return None
def is_released_kernel(version):
- # True if version name suggests a released kernel,
- # not some release candidate or experimental kernel name
- # e.g. 2.6.18-smp-200.0 includes no other text, underscores, etc
- version = version.strip('01234567890.-')
- return version in ['', 'smp', 'smpx', 'pae']
+ # True if version name suggests a released kernel,
+ # not some release candidate or experimental kernel name
+ # e.g. 2.6.18-smp-200.0 includes no other text, underscores, etc
+ version = version.strip('01234567890.-')
+ return version in ['', 'smp', 'smpx', 'pae']
def is_release_candidate(version):
- # True if version names a released kernel or release candidate,
- # not some experimental name containing arbitrary text
- # e.g. 2.6.18-smp-220.0_rc3 but not 2.6.18_patched
- version = re.sub(r'[_-]rc\d+', '', version)
- return is_released_kernel(version)
-
-
+ # True if version names a released kernel or release candidate,
+ # not some experimental name containing arbitrary text
+ # e.g. 2.6.18-smp-220.0_rc3 but not 2.6.18_patched
+ version = re.sub(r'[_-]rc\d+', '', version)
+ return is_released_kernel(version)
diff --git a/client/bin/kernel_versions_unittest.py b/client/bin/kernel_versions_unittest.py
index fc21a65..b63ecc8 100755
--- a/client/bin/kernel_versions_unittest.py
+++ b/client/bin/kernel_versions_unittest.py
@@ -5,83 +5,83 @@
class kernel_versions_test(unittest.TestCase):
- def increases(self, kernels):
- for i in xrange(len(kernels)-1):
- k1 = kernels[i]
- k2 = kernels[i+1]
- ek1 = version_encode(k1)
- ek2 = version_encode(k2)
- self.assert_(ek1 < ek2,
- '%s (-> %s) should sort < %s (-> %s)'
- % (k1, ek1, k2, ek2) )
+ def increases(self, kernels):
+ for i in xrange(len(kernels)-1):
+ k1 = kernels[i]
+ k2 = kernels[i+1]
+ ek1 = version_encode(k1)
+ ek2 = version_encode(k2)
+ self.assert_(ek1 < ek2,
+ '%s (-> %s) should sort < %s (-> %s)'
+ % (k1, ek1, k2, ek2) )
- def test_version_encode(self):
- series1 = [
- '2.6',
- '2.6.0',
- '2.6.1-rc1',
- '2.6.1-rc1_fix',
- '2.6.1-rc1_patch',
- '2.6.1-rc9',
- '2.6.1-rc9-mm1',
- '2.6.1-rc9-mm2',
- '2.6.1-rc10',
- '2.6.1-rc98',
- '2.6.1',
- '2.6.1_patch',
- '2.6.9',
- '2.6.10',
- '2.6.99',
- '2.7',
- '2.9.99',
- '2.10.0',
- '99.99.99',
- 'UNKNOWN',
- ]
- self.increases(series1)
- self.increases(['pathX'+k for k in series1])
- series2 = [
- '2.6.18-smp-220',
- '2.6.18-smp-220.0',
- '2.6.18-smp-220.1_rc1',
- '2.6.18-smp-220.1_rc1_fix',
- '2.6.18-smp-220.1_rc1_patch',
- '2.6.18-smp-220.1_rc9',
- '2.6.18-smp-220.1_rc9_mm1',
- '2.6.18-smp-220.1_rc9_mm2',
- '2.6.18-smp-220.1_rc10',
- '2.6.18-smp-220.1_rc98',
- '2.6.18-smp-220.1',
- '2.6.18-smp-220.1_patch',
- '2.6.18-smp-220.9',
- '2.6.18-smp-220.10',
- '2.6.18-smp-220.99',
- '2.6.18-smp-221',
- 'UNKNOWN',
- ]
- self.increases(series2)
- self.increases(['pathX'+k for k in series2])
+ def test_version_encode(self):
+ series1 = [
+ '2.6',
+ '2.6.0',
+ '2.6.1-rc1',
+ '2.6.1-rc1_fix',
+ '2.6.1-rc1_patch',
+ '2.6.1-rc9',
+ '2.6.1-rc9-mm1',
+ '2.6.1-rc9-mm2',
+ '2.6.1-rc10',
+ '2.6.1-rc98',
+ '2.6.1',
+ '2.6.1_patch',
+ '2.6.9',
+ '2.6.10',
+ '2.6.99',
+ '2.7',
+ '2.9.99',
+ '2.10.0',
+ '99.99.99',
+ 'UNKNOWN',
+ ]
+ self.increases(series1)
+ self.increases(['pathX'+k for k in series1])
+ series2 = [
+ '2.6.18-smp-220',
+ '2.6.18-smp-220.0',
+ '2.6.18-smp-220.1_rc1',
+ '2.6.18-smp-220.1_rc1_fix',
+ '2.6.18-smp-220.1_rc1_patch',
+ '2.6.18-smp-220.1_rc9',
+ '2.6.18-smp-220.1_rc9_mm1',
+ '2.6.18-smp-220.1_rc9_mm2',
+ '2.6.18-smp-220.1_rc10',
+ '2.6.18-smp-220.1_rc98',
+ '2.6.18-smp-220.1',
+ '2.6.18-smp-220.1_patch',
+ '2.6.18-smp-220.9',
+ '2.6.18-smp-220.10',
+ '2.6.18-smp-220.99',
+ '2.6.18-smp-221',
+ 'UNKNOWN',
+ ]
+ self.increases(series2)
+ self.increases(['pathX'+k for k in series2])
- releases = ['2.6.1' , '2.6.18-smp-220.0' ]
- candidates = ['2.6.1-rc1' , '2.6.18-smp-220.0_rc1']
- experiments = ['2.6.1-patch', '2.6.1-rc1_patch',
- '2.6.18-smp-220.0_patch', 'UNKNOWN']
+ releases = ['2.6.1' , '2.6.18-smp-220.0' ]
+ candidates = ['2.6.1-rc1' , '2.6.18-smp-220.0_rc1']
+ experiments = ['2.6.1-patch', '2.6.1-rc1_patch',
+ '2.6.18-smp-220.0_patch', 'UNKNOWN']
- def test_is_released_kernel(self):
- for v in self.releases:
- self.assert_( is_released_kernel(v))
- for v in self.candidates + self.experiments:
- self.assert_(not is_released_kernel(v))
+ def test_is_released_kernel(self):
+ for v in self.releases:
+ self.assert_( is_released_kernel(v))
+ for v in self.candidates + self.experiments:
+ self.assert_(not is_released_kernel(v))
- def test_is_release_candidate(self):
- for v in self.releases + self.candidates:
- self.assert_( is_release_candidate(v))
- for v in self.experiments:
- self.assert_(not is_release_candidate(v))
+ def test_is_release_candidate(self):
+ for v in self.releases + self.candidates:
+ self.assert_( is_release_candidate(v))
+ for v in self.experiments:
+ self.assert_(not is_release_candidate(v))
if __name__ == "__main__":
- unittest.main()
+ unittest.main()
diff --git a/client/bin/kernelexpand-test.py b/client/bin/kernelexpand-test.py
index 3c34010..24b4ce0 100755
--- a/client/bin/kernelexpand-test.py
+++ b/client/bin/kernelexpand-test.py
@@ -11,136 +11,136 @@
akpml = 'http://www.example.com/mirror/akpm/'
mirrorA = [
- [ akpm, akpml ],
- [ km, kml ],
+ [ akpm, akpml ],
+ [ km, kml ],
]
class kernelexpandTest(unittest.TestCase):
- def test_decompose_simple(self):
- correct = [
- [ km + 'v2.6/linux-2.6.23.tar.bz2' ]
- ]
- sample = decompose_kernel('2.6.23')
- self.assertEqual(sample, correct)
+ def test_decompose_simple(self):
+ correct = [
+ [ km + 'v2.6/linux-2.6.23.tar.bz2' ]
+ ]
+ sample = decompose_kernel('2.6.23')
+ self.assertEqual(sample, correct)
- def test_decompose_fail(self):
- success = False
- try:
- sample = decompose_kernel('1.0.0.0.0')
- success = True
- except NameError:
- pass
- except Exception, e:
- self.fail('expected NameError, got something else')
+ def test_decompose_fail(self):
+ success = False
+ try:
+ sample = decompose_kernel('1.0.0.0.0')
+ success = True
+ except NameError:
+ pass
+ except Exception, e:
+ self.fail('expected NameError, got something else')
- if success:
- self.fail('expected NameError, was successful')
+ if success:
+ self.fail('expected NameError, was successful')
- def test_decompose_rcN(self):
- correct = [
- [ km + 'v2.6/testing/v2.6.23/linux-2.6.23-rc1.tar.bz2',
- km + 'v2.6/testing/linux-2.6.23-rc1.tar.bz2']
- ]
- sample = decompose_kernel('2.6.23-rc1')
- self.assertEqual(sample, correct)
-
-
- def test_decompose_mmN(self):
- correct = [
- [ km + 'v2.6/linux-2.6.23.tar.bz2' ],
- [ akpm + '2.6/2.6.23/2.6.23-mm1/2.6.23-mm1.bz2' ]
- ]
- sample = decompose_kernel('2.6.23-mm1')
- self.assertEqual(sample, correct)
+ def test_decompose_rcN(self):
+ correct = [
+ [ km + 'v2.6/testing/v2.6.23/linux-2.6.23-rc1.tar.bz2',
+ km + 'v2.6/testing/linux-2.6.23-rc1.tar.bz2']
+ ]
+ sample = decompose_kernel('2.6.23-rc1')
+ self.assertEqual(sample, correct)
- def test_decompose_gitN(self):
- correct = [
- [ km + 'v2.6/linux-2.6.23.tar.bz2' ],
- [ km + 'v2.6/snapshots/old/patch-2.6.23-git1.bz2',
- km + 'v2.6/snapshots/patch-2.6.23-git1.bz2']
- ]
- sample = decompose_kernel('2.6.23-git1')
- self.assertEqual(sample, correct)
+ def test_decompose_mmN(self):
+ correct = [
+ [ km + 'v2.6/linux-2.6.23.tar.bz2' ],
+ [ akpm + '2.6/2.6.23/2.6.23-mm1/2.6.23-mm1.bz2' ]
+ ]
+ sample = decompose_kernel('2.6.23-mm1')
+ self.assertEqual(sample, correct)
- def test_decompose_rcN_mmN(self):
- correct = [
- [ km + 'v2.6/testing/v2.6.23/linux-2.6.23-rc1.tar.bz2',
- km + 'v2.6/testing/linux-2.6.23-rc1.tar.bz2' ],
- [ akpm + '2.6/2.6.23-rc1/2.6.23-rc1-mm1/2.6.23-rc1-mm1.bz2']
- ]
- sample = decompose_kernel('2.6.23-rc1-mm1')
- self.assertEqual(sample, correct)
+ def test_decompose_gitN(self):
+ correct = [
+ [ km + 'v2.6/linux-2.6.23.tar.bz2' ],
+ [ km + 'v2.6/snapshots/old/patch-2.6.23-git1.bz2',
+ km + 'v2.6/snapshots/patch-2.6.23-git1.bz2']
+ ]
+ sample = decompose_kernel('2.6.23-git1')
+ self.assertEqual(sample, correct)
- def test_mirrorA_simple(self):
- correct = [
- [ kml + 'v2.6/linux-2.6.23.tar.bz2',
- km + 'v2.6/linux-2.6.23.tar.bz2' ]
- ]
- sample = decompose_kernel('2.6.23')
- sample = mirror_kernel_components(mirrorA, sample)
-
- self.assertEqual(sample, correct)
+ def test_decompose_rcN_mmN(self):
+ correct = [
+ [ km + 'v2.6/testing/v2.6.23/linux-2.6.23-rc1.tar.bz2',
+ km + 'v2.6/testing/linux-2.6.23-rc1.tar.bz2' ],
+ [ akpm + '2.6/2.6.23-rc1/2.6.23-rc1-mm1/2.6.23-rc1-mm1.bz2']
+ ]
+ sample = decompose_kernel('2.6.23-rc1-mm1')
+ self.assertEqual(sample, correct)
- def test_mirrorA_rcN(self):
- correct = [
- [ kml + 'v2.6/testing/v2.6.23/linux-2.6.23-rc1.tar.bz2',
- kml + 'v2.6/testing/linux-2.6.23-rc1.tar.bz2',
- km + 'v2.6/testing/v2.6.23/linux-2.6.23-rc1.tar.bz2',
- km + 'v2.6/testing/linux-2.6.23-rc1.tar.bz2' ]
- ]
- sample = decompose_kernel('2.6.23-rc1')
- sample = mirror_kernel_components(mirrorA, sample)
- self.assertEqual(sample, correct)
+ def test_mirrorA_simple(self):
+ correct = [
+ [ kml + 'v2.6/linux-2.6.23.tar.bz2',
+ km + 'v2.6/linux-2.6.23.tar.bz2' ]
+ ]
+ sample = decompose_kernel('2.6.23')
+ sample = mirror_kernel_components(mirrorA, sample)
-
- def test_mirrorA_mmN(self):
- correct = [
- [ kml + 'v2.6/linux-2.6.23.tar.bz2',
- km + 'v2.6/linux-2.6.23.tar.bz2'],
- [ akpml + '2.6/2.6.23/2.6.23-mm1/2.6.23-mm1.bz2',
- kml + 'people/akpm/patches/2.6/2.6.23/2.6.23-mm1/2.6.23-mm1.bz2',
- akpm + '2.6/2.6.23/2.6.23-mm1/2.6.23-mm1.bz2' ]
- ]
-
- sample = decompose_kernel('2.6.23-mm1')
- sample = mirror_kernel_components(mirrorA, sample)
- self.assertEqual(sample, correct)
+ self.assertEqual(sample, correct)
- def test_mirrorA_gitN(self):
- correct = [
- [ kml + 'v2.6/linux-2.6.23.tar.bz2',
- km + 'v2.6/linux-2.6.23.tar.bz2'],
- [ kml + 'v2.6/snapshots/old/patch-2.6.23-git1.bz2',
- kml + 'v2.6/snapshots/patch-2.6.23-git1.bz2',
- km + 'v2.6/snapshots/old/patch-2.6.23-git1.bz2',
- km + 'v2.6/snapshots/patch-2.6.23-git1.bz2' ]
- ]
- sample = decompose_kernel('2.6.23-git1')
- sample = mirror_kernel_components(mirrorA, sample)
- self.assertEqual(sample, correct)
+ def test_mirrorA_rcN(self):
+ correct = [
+ [ kml + 'v2.6/testing/v2.6.23/linux-2.6.23-rc1.tar.bz2',
+ kml + 'v2.6/testing/linux-2.6.23-rc1.tar.bz2',
+ km + 'v2.6/testing/v2.6.23/linux-2.6.23-rc1.tar.bz2',
+ km + 'v2.6/testing/linux-2.6.23-rc1.tar.bz2' ]
+ ]
+ sample = decompose_kernel('2.6.23-rc1')
+ sample = mirror_kernel_components(mirrorA, sample)
+ self.assertEqual(sample, correct)
- def test_mirrorA_rcN_mmN(self):
- correct = [
- [ kml + 'v2.6/testing/v2.6.23/linux-2.6.23-rc1.tar.bz2',
- kml + 'v2.6/testing/linux-2.6.23-rc1.tar.bz2',
- km + 'v2.6/testing/v2.6.23/linux-2.6.23-rc1.tar.bz2',
- km + 'v2.6/testing/linux-2.6.23-rc1.tar.bz2'],
- [ akpml + '2.6/2.6.23-rc1/2.6.23-rc1-mm1/2.6.23-rc1-mm1.bz2',
- kml + 'people/akpm/patches/2.6/2.6.23-rc1/2.6.23-rc1-mm1/2.6.23-rc1-mm1.bz2',
- akpm + '2.6/2.6.23-rc1/2.6.23-rc1-mm1/2.6.23-rc1-mm1.bz2' ]
- ]
- sample = decompose_kernel('2.6.23-rc1-mm1')
- sample = mirror_kernel_components(mirrorA, sample)
- self.assertEqual(sample, correct)
+ def test_mirrorA_mmN(self):
+ correct = [
+ [ kml + 'v2.6/linux-2.6.23.tar.bz2',
+ km + 'v2.6/linux-2.6.23.tar.bz2'],
+ [ akpml + '2.6/2.6.23/2.6.23-mm1/2.6.23-mm1.bz2',
+ kml + 'people/akpm/patches/2.6/2.6.23/2.6.23-mm1/2.6.23-mm1.bz2',
+ akpm + '2.6/2.6.23/2.6.23-mm1/2.6.23-mm1.bz2' ]
+ ]
+
+ sample = decompose_kernel('2.6.23-mm1')
+ sample = mirror_kernel_components(mirrorA, sample)
+ self.assertEqual(sample, correct)
+
+
+ def test_mirrorA_gitN(self):
+ correct = [
+ [ kml + 'v2.6/linux-2.6.23.tar.bz2',
+ km + 'v2.6/linux-2.6.23.tar.bz2'],
+ [ kml + 'v2.6/snapshots/old/patch-2.6.23-git1.bz2',
+ kml + 'v2.6/snapshots/patch-2.6.23-git1.bz2',
+ km + 'v2.6/snapshots/old/patch-2.6.23-git1.bz2',
+ km + 'v2.6/snapshots/patch-2.6.23-git1.bz2' ]
+ ]
+ sample = decompose_kernel('2.6.23-git1')
+ sample = mirror_kernel_components(mirrorA, sample)
+ self.assertEqual(sample, correct)
+
+
+ def test_mirrorA_rcN_mmN(self):
+ correct = [
+ [ kml + 'v2.6/testing/v2.6.23/linux-2.6.23-rc1.tar.bz2',
+ kml + 'v2.6/testing/linux-2.6.23-rc1.tar.bz2',
+ km + 'v2.6/testing/v2.6.23/linux-2.6.23-rc1.tar.bz2',
+ km + 'v2.6/testing/linux-2.6.23-rc1.tar.bz2'],
+ [ akpml + '2.6/2.6.23-rc1/2.6.23-rc1-mm1/2.6.23-rc1-mm1.bz2',
+ kml + 'people/akpm/patches/2.6/2.6.23-rc1/2.6.23-rc1-mm1/2.6.23-rc1-mm1.bz2',
+ akpm + '2.6/2.6.23-rc1/2.6.23-rc1-mm1/2.6.23-rc1-mm1.bz2' ]
+ ]
+ sample = decompose_kernel('2.6.23-rc1-mm1')
+ sample = mirror_kernel_components(mirrorA, sample)
+ self.assertEqual(sample, correct)
if __name__ == '__main__':
- unittest.main()
+ unittest.main()
diff --git a/client/bin/kernelexpand.py b/client/bin/kernelexpand.py
index 73028fa..e23b865 100755
--- a/client/bin/kernelexpand.py
+++ b/client/bin/kernelexpand.py
@@ -12,180 +12,180 @@
kernel = 'http://www.kernel.org/pub/linux/kernel/'
mappings = [
- [ r'^\d+\.\d+\.\d+$', '', True, [
- kernel + 'v%(major)s/linux-%(full)s.tar.bz2'
- ]],
- [ r'^\d+\.\d+\.\d+\.\d+$', '', True, [
- kernel + 'v%(major)s/linux-%(full)s.tar.bz2'
- ]],
- [ r'-rc\d+$', '%(minor-prev)s', True, [
- kernel + 'v%(major)s/testing/v%(minor)s/linux-%(full)s.tar.bz2',
- kernel + 'v%(major)s/testing/linux-%(full)s.tar.bz2',
- ]],
- [ r'-(git|bk)\d+$', '%(base)s', False, [
- kernel + 'v%(major)s/snapshots/old/patch-%(full)s.bz2',
- kernel + 'v%(major)s/snapshots/patch-%(full)s.bz2',
- ]],
- [ r'-mm\d+$', '%(base)s', False, [
- kernel + 'people/akpm/patches/' +
- '%(major)s/%(base)s/%(full)s/%(full)s.bz2'
- ]],
- [ r'-mjb\d+$', '%(base)s', False, [
- kernel + 'people/mbligh/%(base)s/patch-%(full)s.bz2'
- ]]
+ [ r'^\d+\.\d+\.\d+$', '', True, [
+ kernel + 'v%(major)s/linux-%(full)s.tar.bz2'
+ ]],
+ [ r'^\d+\.\d+\.\d+\.\d+$', '', True, [
+ kernel + 'v%(major)s/linux-%(full)s.tar.bz2'
+ ]],
+ [ r'-rc\d+$', '%(minor-prev)s', True, [
+ kernel + 'v%(major)s/testing/v%(minor)s/linux-%(full)s.tar.bz2',
+ kernel + 'v%(major)s/testing/linux-%(full)s.tar.bz2',
+ ]],
+ [ r'-(git|bk)\d+$', '%(base)s', False, [
+ kernel + 'v%(major)s/snapshots/old/patch-%(full)s.bz2',
+ kernel + 'v%(major)s/snapshots/patch-%(full)s.bz2',
+ ]],
+ [ r'-mm\d+$', '%(base)s', False, [
+ kernel + 'people/akpm/patches/' +
+ '%(major)s/%(base)s/%(full)s/%(full)s.bz2'
+ ]],
+ [ r'-mjb\d+$', '%(base)s', False, [
+ kernel + 'people/mbligh/%(base)s/patch-%(full)s.bz2'
+ ]]
];
def decompose_kernel_once(kernel):
- ##print "S<" + kernel + ">"
- for mapping in mappings:
- (suffix, becomes, is_full, patch_templates) = mapping
+ ##print "S<" + kernel + ">"
+ for mapping in mappings:
+ (suffix, becomes, is_full, patch_templates) = mapping
- params = {}
+ params = {}
- match = re.search(r'^(.*)' + suffix, kernel)
- if not match:
- continue
+ match = re.search(r'^(.*)' + suffix, kernel)
+ if not match:
+ continue
- # Generate the parameters for the patches:
- # full => full kernel name
- # base => all but the matches suffix
- # minor => 2.n.m
- # major => 2.n
- # minor-prev => 2.n.m-1
- params['full'] = kernel
- params['base'] = match.group(1)
+ # Generate the parameters for the patches:
+ # full => full kernel name
+ # base => all but the matches suffix
+ # minor => 2.n.m
+ # major => 2.n
+ # minor-prev => 2.n.m-1
+ params['full'] = kernel
+ params['base'] = match.group(1)
- match = re.search(r'^((\d+\.\d+)\.(\d+))', kernel)
- if not match:
- raise "unable to determine major/minor version"
- params['minor'] = match.group(1)
- params['major'] = match.group(2)
- params['minor-prev'] = match.group(2) + \
- '.%d' % (int(match.group(3)) - 1)
+ match = re.search(r'^((\d+\.\d+)\.(\d+))', kernel)
+ if not match:
+ raise "unable to determine major/minor version"
+ params['minor'] = match.group(1)
+ params['major'] = match.group(2)
+ params['minor-prev'] = match.group(2) + \
+ '.%d' % (int(match.group(3)) - 1)
- # Build the new kernel and patch list.
- new_kernel = becomes % params
- patch_list = []
- for template in patch_templates:
- patch_list.append(template % params)
-
- return (is_full, new_kernel, patch_list)
+ # Build the new kernel and patch list.
+ new_kernel = becomes % params
+ patch_list = []
+ for template in patch_templates:
+ patch_list.append(template % params)
- return (True, kernel, None)
+ return (is_full, new_kernel, patch_list)
+
+ return (True, kernel, None)
def decompose_kernel(kernel):
- kernel_patches = []
+ kernel_patches = []
- done = False
- while not done:
- (done, kernel, patch_list) = decompose_kernel_once(kernel)
- if patch_list:
- kernel_patches.insert(0, patch_list)
- if not len(kernel_patches):
- raise NameError('kernelexpand: %s: unknown kernel' % (kernel))
+ done = False
+ while not done:
+ (done, kernel, patch_list) = decompose_kernel_once(kernel)
+ if patch_list:
+ kernel_patches.insert(0, patch_list)
+ if not len(kernel_patches):
+ raise NameError('kernelexpand: %s: unknown kernel' % (kernel))
- return kernel_patches
+ return kernel_patches
# Look for and add potential mirrors.
def mirror_kernel_components(mirrors, components):
- new_components = []
- for component in components:
- new_patches = []
- for mirror in mirrors:
- (prefix, local) = mirror
- for patch in component:
- if patch.startswith(prefix):
- new_patch = local + \
- patch[len(prefix):]
- new_patches.append(new_patch)
- for patch in component:
- new_patches.append(patch)
- new_components.append(new_patches)
-
- return new_components
+ new_components = []
+ for component in components:
+ new_patches = []
+ for mirror in mirrors:
+ (prefix, local) = mirror
+ for patch in component:
+ if patch.startswith(prefix):
+ new_patch = local + \
+ patch[len(prefix):]
+ new_patches.append(new_patch)
+ for patch in component:
+ new_patches.append(patch)
+ new_components.append(new_patches)
+
+ return new_components
def url_accessible(url):
- status = os.system("wget --spider -q '%s'" % (url))
- #print url + ": status=%d" % (status)
-
- return status == 0
+ status = os.system("wget --spider -q '%s'" % (url))
+ #print url + ": status=%d" % (status)
+
+ return status == 0
def select_kernel_components(components):
- new_components = []
- for component in components:
- new_patches = []
- for patch in component:
- if url_accessible(patch):
- new_patches.append(patch)
- break
- if not len(new_patches):
- new_patches.append(component[-1])
- new_components.append(new_patches)
- return new_components
+ new_components = []
+ for component in components:
+ new_patches = []
+ for patch in component:
+ if url_accessible(patch):
+ new_patches.append(patch)
+ break
+ if not len(new_patches):
+ new_patches.append(component[-1])
+ new_components.append(new_patches)
+ return new_components
def expand_classic(kernel, mirrors):
- components = decompose_kernel(kernel)
- if mirrors:
- components = mirror_kernel_components(mirrors, components)
- components = select_kernel_components(components)
+ components = decompose_kernel(kernel)
+ if mirrors:
+ components = mirror_kernel_components(mirrors, components)
+ components = select_kernel_components(components)
- patches = []
- for component in components:
- patches.append(component[0])
+ patches = []
+ for component in components:
+ patches.append(component[0])
- return patches
+ return patches
if __name__ == '__main__':
- from optparse import OptionParser
+ from optparse import OptionParser
- parser = OptionParser()
+ parser = OptionParser()
- parser.add_option("-m", "--mirror",
- type="string", dest="mirror", action="append", nargs=2,
- help="mirror prefix")
- parser.add_option("-v", "--no-validate", dest="validate",
- action="store_false", default=True,
- help="prune invalid entries")
+ parser.add_option("-m", "--mirror",
+ type="string", dest="mirror", action="append", nargs=2,
+ help="mirror prefix")
+ parser.add_option("-v", "--no-validate", dest="validate",
+ action="store_false", default=True,
+ help="prune invalid entries")
- def usage():
- parser.print_help()
- sys.exit(1)
+ def usage():
+ parser.print_help()
+ sys.exit(1)
- options, args = parser.parse_args()
+ options, args = parser.parse_args()
- # Check for a kernel version
- if len(args) != 1:
- usage()
- kernel = args[0]
+ # Check for a kernel version
+ if len(args) != 1:
+ usage()
+ kernel = args[0]
- #mirrors = [
- # [ 'http://www.kernel.org/pub/linux/kernel/v2.4',
- # 'http://kernel.beaverton.ibm.com/mirror/v2.4' ],
- # [ 'http://www.kernel.org/pub/linux/kernel/v2.6',
- # 'http://kernel.beaverton.ibm.com/mirror/v2.6' ],
- # [ 'http://www.kernel.org/pub/linux/kernel/people/akpm/patches',
- # 'http://kernel.beaverton.ibm.com/mirror/akpm' ],
- #]
- mirrors = options.mirror
+ #mirrors = [
+ # [ 'http://www.kernel.org/pub/linux/kernel/v2.4',
+ # 'http://kernel.beaverton.ibm.com/mirror/v2.4' ],
+ # [ 'http://www.kernel.org/pub/linux/kernel/v2.6',
+ # 'http://kernel.beaverton.ibm.com/mirror/v2.6' ],
+ # [ 'http://www.kernel.org/pub/linux/kernel/people/akpm/patches',
+ # 'http://kernel.beaverton.ibm.com/mirror/akpm' ],
+ #]
+ mirrors = options.mirror
- try:
- components = decompose_kernel(kernel)
- except NameError, e:
- sys.stderr.write(e.args[0] + "\n")
- sys.exit(1)
+ try:
+ components = decompose_kernel(kernel)
+ except NameError, e:
+ sys.stderr.write(e.args[0] + "\n")
+ sys.exit(1)
- if mirrors:
- components = mirror_kernel_components(mirrors, components)
+ if mirrors:
+ components = mirror_kernel_components(mirrors, components)
- if options.validate:
- components = select_kernel_components(components)
+ if options.validate:
+ components = select_kernel_components(components)
- # Dump them out.
- for component in components:
- print " ".join(component)
+ # Dump them out.
+ for component in components:
+ print " ".join(component)
diff --git a/client/bin/os_dep.py b/client/bin/os_dep.py
index f61d46d..9022913 100644
--- a/client/bin/os_dep.py
+++ b/client/bin/os_dep.py
@@ -8,30 +8,30 @@
"""
def command(cmd):
- # this could use '/usr/bin/which', I suppose. But this seems simpler
- for dir in os.environ['PATH'].split(':'):
- file = os.path.join(dir, cmd)
- if os.path.exists(file):
- return file
- raise ValueError('Missing command: %s' % cmd)
+ # this could use '/usr/bin/which', I suppose. But this seems simpler
+ for dir in os.environ['PATH'].split(':'):
+ file = os.path.join(dir, cmd)
+ if os.path.exists(file):
+ return file
+ raise ValueError('Missing command: %s' % cmd)
def commands(*cmds):
- results = []
- for cmd in cmds:
- results.append(command(cmd))
+ results = []
+ for cmd in cmds:
+ results.append(command(cmd))
def library(lib):
- lddirs = [x.rstrip() for x in open('/etc/ld.so.conf', 'r').readlines()]
- for dir in ['/lib', '/usr/lib'] + lddirs:
- file = os.path.join(dir, lib)
- if os.path.exists(file):
- return file
- raise ValueError('Missing library: %s' % lib)
+ lddirs = [x.rstrip() for x in open('/etc/ld.so.conf', 'r').readlines()]
+ for dir in ['/lib', '/usr/lib'] + lddirs:
+ file = os.path.join(dir, lib)
+ if os.path.exists(file):
+ return file
+ raise ValueError('Missing library: %s' % lib)
def libraries(*libs):
- results = []
- for lib in libs:
- results.append(library(lib))
+ results = []
+ for lib in libs:
+ results.append(library(lib))
diff --git a/client/bin/package.py b/client/bin/package.py
index c889a9c..42a8604 100644
--- a/client/bin/package.py
+++ b/client/bin/package.py
@@ -1,5 +1,5 @@
"""
-Functions to handle software packages. The functions covered here aim to be
+Functions to handle software packages. The functions covered here aim to be
generic, with implementations that deal with different package managers, such
as dpkg and rpm.
"""
@@ -15,277 +15,277 @@
def __rpm_info(rpm_package):
- """\
- Private function that returns a dictionary with information about an
- RPM package file
- - type: Package management program that handles the file
- - system_support: If the package management program is installed on the
- system or not
- - source: If it is a source (True) our binary (False) package
- - version: The package version (or name), that is used to check against the
- package manager if the package is installed
- - arch: The architecture for which a binary package was built
- - installed: Whether the package is installed (True) on the system or not
- (False)
- """
- # We will make good use of what the file command has to tell us about the
- # package :)
- file_result = utils.system_output('file ' + rpm_package)
- package_info = {}
- package_info['type'] = 'rpm'
- try:
- os_dep.command('rpm')
- # Build the command strings that will be used to get package info
- # s_cmd - Command to determine if package is a source package
- # a_cmd - Command to determine package architecture
- # v_cmd - Command to determine package version
- # i_cmd - Command to determiine if package is installed
- s_cmd = 'rpm -qp --qf %{SOURCE} ' + rpm_package + ' 2>/dev/null'
- a_cmd = 'rpm -qp --qf %{ARCH} ' + rpm_package + ' 2>/dev/null'
- v_cmd = 'rpm -qp ' + rpm_package + ' 2>/dev/null'
- i_cmd = 'rpm -q ' + utils.system_output(v_cmd) + ' 2>&1 >/dev/null'
+ """\
+ Private function that returns a dictionary with information about an
+ RPM package file
+ - type: Package management program that handles the file
+ - system_support: If the package management program is installed on the
+ system or not
+ - source: If it is a source (True) our binary (False) package
+ - version: The package version (or name), that is used to check against the
+ package manager if the package is installed
+ - arch: The architecture for which a binary package was built
+ - installed: Whether the package is installed (True) on the system or not
+ (False)
+ """
+ # We will make good use of what the file command has to tell us about the
+ # package :)
+ file_result = utils.system_output('file ' + rpm_package)
+ package_info = {}
+ package_info['type'] = 'rpm'
+ try:
+ os_dep.command('rpm')
+ # Build the command strings that will be used to get package info
+ # s_cmd - Command to determine if package is a source package
+ # a_cmd - Command to determine package architecture
+ # v_cmd - Command to determine package version
+ # i_cmd - Command to determiine if package is installed
+ s_cmd = 'rpm -qp --qf %{SOURCE} ' + rpm_package + ' 2>/dev/null'
+ a_cmd = 'rpm -qp --qf %{ARCH} ' + rpm_package + ' 2>/dev/null'
+ v_cmd = 'rpm -qp ' + rpm_package + ' 2>/dev/null'
+ i_cmd = 'rpm -q ' + utils.system_output(v_cmd) + ' 2>&1 >/dev/null'
- package_info['system_support'] = True
- # Checking whether this is a source or src package
- source = utils.system_output(s_cmd)
- if source == '(none)':
- package_info['source'] = False
- else:
- package_info['source'] = True
- package_info['version'] = utils.system_output(v_cmd)
- package_info['arch'] = utils.system_output(a_cmd)
- # Checking if package is installed
- try:
- utils.system(i_cmd)
- package_info['installed'] = True
- except:
- package_info['installed'] = False
+ package_info['system_support'] = True
+ # Checking whether this is a source or src package
+ source = utils.system_output(s_cmd)
+ if source == '(none)':
+ package_info['source'] = False
+ else:
+ package_info['source'] = True
+ package_info['version'] = utils.system_output(v_cmd)
+ package_info['arch'] = utils.system_output(a_cmd)
+ # Checking if package is installed
+ try:
+ utils.system(i_cmd)
+ package_info['installed'] = True
+ except:
+ package_info['installed'] = False
- except:
- package_info['system_support'] = False
- package_info['installed'] = False
- # File gives a wealth of information about rpm packages.
- # However, we can't trust all this info, as incorrectly
- # packaged rpms can report some wrong values.
- # It's better than nothing though :)
- if len(file_result.split(' ')) == 6:
- # Figure if package is a source package
- if file_result.split(' ')[3] == 'src':
- package_info['source'] = True
- elif file_result.split(' ')[3] == 'bin':
- package_info['source'] = False
- else:
- package_info['source'] = False
- # Get architecture
- package_info['arch'] = file_result.split(' ')[4]
- # Get version
- package_info['version'] = file_result.split(' ')[5]
- elif len(file_result.split(' ')) == 5:
- # Figure if package is a source package
- if file_result.split(' ')[3] == 'src':
- package_info['source'] = True
- elif file_result.split(' ')[3] == 'bin':
- package_info['source'] = False
- else:
- package_info['source'] = False
- # When the arch param is missing on file, we assume noarch
- package_info['arch'] = 'noarch'
- # Get version
- package_info['version'] = file_result.split(' ')[4]
- else:
- # If everything else fails...
- package_info['source'] = False
- package_info['arch'] = 'Not Available'
- package_info['version'] = 'Not Available'
- return package_info
+ except:
+ package_info['system_support'] = False
+ package_info['installed'] = False
+ # File gives a wealth of information about rpm packages.
+ # However, we can't trust all this info, as incorrectly
+ # packaged rpms can report some wrong values.
+ # It's better than nothing though :)
+ if len(file_result.split(' ')) == 6:
+ # Figure if package is a source package
+ if file_result.split(' ')[3] == 'src':
+ package_info['source'] = True
+ elif file_result.split(' ')[3] == 'bin':
+ package_info['source'] = False
+ else:
+ package_info['source'] = False
+ # Get architecture
+ package_info['arch'] = file_result.split(' ')[4]
+ # Get version
+ package_info['version'] = file_result.split(' ')[5]
+ elif len(file_result.split(' ')) == 5:
+ # Figure if package is a source package
+ if file_result.split(' ')[3] == 'src':
+ package_info['source'] = True
+ elif file_result.split(' ')[3] == 'bin':
+ package_info['source'] = False
+ else:
+ package_info['source'] = False
+ # When the arch param is missing on file, we assume noarch
+ package_info['arch'] = 'noarch'
+ # Get version
+ package_info['version'] = file_result.split(' ')[4]
+ else:
+ # If everything else fails...
+ package_info['source'] = False
+ package_info['arch'] = 'Not Available'
+ package_info['version'] = 'Not Available'
+ return package_info
def __dpkg_info(dpkg_package):
- """\
- Private function that returns a dictionary with information about a
- dpkg package file
- - type: Package management program that handles the file
- - system_support: If the package management program is installed on the
- system or not
- - source: If it is a source (True) our binary (False) package
- - version: The package version (or name), that is used to check against the
- package manager if the package is installed
- - arch: The architecture for which a binary package was built
- - installed: Whether the package is installed (True) on the system or not
- (False)
- """
- # We will make good use of what the file command has to tell us about the
- # package :)
- file_result = utils.system_output('file ' + dpkg_package)
- package_info = {}
- package_info['type'] = 'dpkg'
- # There's no single debian source package as is the case
- # with RPM
- package_info['source'] = False
- try:
- os_dep.command('dpkg')
- # Build the command strings that will be used to get package info
- # a_cmd - Command to determine package architecture
- # v_cmd - Command to determine package version
- # i_cmd - Command to determiine if package is installed
- a_cmd = 'dpkg -f ' + dpkg_package + ' Architecture 2>/dev/null'
- v_cmd = 'dpkg -f ' + dpkg_package + ' Package 2>/dev/null'
- i_cmd = 'dpkg -s ' + utils.system_output(v_cmd) + ' 2>/dev/null'
+ """\
+ Private function that returns a dictionary with information about a
+ dpkg package file
+ - type: Package management program that handles the file
+ - system_support: If the package management program is installed on the
+ system or not
+ - source: If it is a source (True) our binary (False) package
+ - version: The package version (or name), that is used to check against the
+ package manager if the package is installed
+ - arch: The architecture for which a binary package was built
+ - installed: Whether the package is installed (True) on the system or not
+ (False)
+ """
+ # We will make good use of what the file command has to tell us about the
+ # package :)
+ file_result = utils.system_output('file ' + dpkg_package)
+ package_info = {}
+ package_info['type'] = 'dpkg'
+ # There's no single debian source package as is the case
+ # with RPM
+ package_info['source'] = False
+ try:
+ os_dep.command('dpkg')
+ # Build the command strings that will be used to get package info
+ # a_cmd - Command to determine package architecture
+ # v_cmd - Command to determine package version
+ # i_cmd - Command to determiine if package is installed
+ a_cmd = 'dpkg -f ' + dpkg_package + ' Architecture 2>/dev/null'
+ v_cmd = 'dpkg -f ' + dpkg_package + ' Package 2>/dev/null'
+ i_cmd = 'dpkg -s ' + utils.system_output(v_cmd) + ' 2>/dev/null'
- package_info['system_support'] = True
- package_info['version'] = utils.system_output(v_cmd)
- package_info['arch'] = utils.system_output(a_cmd)
- # Checking if package is installed
- package_status = utils.system_output(i_cmd, ignore_status=True)
- not_inst_pattern = re.compile('not-installed', re.IGNORECASE)
- dpkg_not_installed = re.search(not_inst_pattern, package_status)
- if dpkg_not_installed:
- package_info['installed'] = False
- else:
- package_info['installed'] = True
+ package_info['system_support'] = True
+ package_info['version'] = utils.system_output(v_cmd)
+ package_info['arch'] = utils.system_output(a_cmd)
+ # Checking if package is installed
+ package_status = utils.system_output(i_cmd, ignore_status=True)
+ not_inst_pattern = re.compile('not-installed', re.IGNORECASE)
+ dpkg_not_installed = re.search(not_inst_pattern, package_status)
+ if dpkg_not_installed:
+ package_info['installed'] = False
+ else:
+ package_info['installed'] = True
- except:
- package_info['system_support'] = False
- package_info['installed'] = False
- # The output of file is not as generous for dpkg files as
- # it is with rpm files
- package_info['arch'] = 'Not Available'
- package_info['version'] = 'Not Available'
+ except:
+ package_info['system_support'] = False
+ package_info['installed'] = False
+ # The output of file is not as generous for dpkg files as
+ # it is with rpm files
+ package_info['arch'] = 'Not Available'
+ package_info['version'] = 'Not Available'
- return package_info
+ return package_info
def info(package):
- """\
- Returns a dictionary with package information about a given package file:
- - type: Package management program that handles the file
- - system_support: If the package management program is installed on the
- system or not
- - source: If it is a source (True) our binary (False) package
- - version: The package version (or name), that is used to check against the
- package manager if the package is installed
- - arch: The architecture for which a binary package was built
- - installed: Whether the package is installed (True) on the system or not
- (False)
+ """\
+ Returns a dictionary with package information about a given package file:
+ - type: Package management program that handles the file
+ - system_support: If the package management program is installed on the
+ system or not
+ - source: If it is a source (True) our binary (False) package
+ - version: The package version (or name), that is used to check against the
+ package manager if the package is installed
+ - arch: The architecture for which a binary package was built
+ - installed: Whether the package is installed (True) on the system or not
+ (False)
- Implemented package types:
- - 'dpkg' - dpkg (debian, ubuntu) package files
- - 'rpm' - rpm (red hat, suse) package files
- Raises an exception if the package type is not one of the implemented
- package types.
- """
- if not os.path.isfile(package):
- raise ValueError('invalid file %s to verify' % package)
- # Use file and libmagic to determine the actual package file type.
- file_result = utils.system_output('file ' + package)
- for package_manager in KNOWN_PACKAGE_MANAGERS:
- if package_manager == 'rpm':
- package_pattern = re.compile('RPM', re.IGNORECASE)
- elif package_manager == 'dpkg':
- package_pattern = re.compile('Debian', re.IGNORECASE)
+ Implemented package types:
+ - 'dpkg' - dpkg (debian, ubuntu) package files
+ - 'rpm' - rpm (red hat, suse) package files
+ Raises an exception if the package type is not one of the implemented
+ package types.
+ """
+ if not os.path.isfile(package):
+ raise ValueError('invalid file %s to verify' % package)
+ # Use file and libmagic to determine the actual package file type.
+ file_result = utils.system_output('file ' + package)
+ for package_manager in KNOWN_PACKAGE_MANAGERS:
+ if package_manager == 'rpm':
+ package_pattern = re.compile('RPM', re.IGNORECASE)
+ elif package_manager == 'dpkg':
+ package_pattern = re.compile('Debian', re.IGNORECASE)
- result = re.search(package_pattern, file_result)
+ result = re.search(package_pattern, file_result)
- if result and package_manager == 'rpm':
- return __rpm_info(package)
- elif result and package_manager == 'dpkg':
- return __dpkg_info(package)
+ if result and package_manager == 'rpm':
+ return __rpm_info(package)
+ elif result and package_manager == 'dpkg':
+ return __dpkg_info(package)
- # If it's not one of the implemented package manager methods, there's
- # not much that can be done, hence we throw an exception.
- raise error.PackageError('Unknown package type %s' % file_result)
+ # If it's not one of the implemented package manager methods, there's
+ # not much that can be done, hence we throw an exception.
+ raise error.PackageError('Unknown package type %s' % file_result)
def install(package, nodeps = False):
- """\
- Tries to install a package file. If the package is already installed,
- it prints a message to the user and ends gracefully. If nodeps is set to
- true, it will ignore package dependencies.
- """
- my_package_info = info(package)
- type = my_package_info['type']
- system_support = my_package_info['system_support']
- source = my_package_info['source']
- installed = my_package_info['installed']
+ """\
+ Tries to install a package file. If the package is already installed,
+ it prints a message to the user and ends gracefully. If nodeps is set to
+ true, it will ignore package dependencies.
+ """
+ my_package_info = info(package)
+ type = my_package_info['type']
+ system_support = my_package_info['system_support']
+ source = my_package_info['source']
+ installed = my_package_info['installed']
- if not system_support:
- e_msg = 'Client does not have package manager %s to handle %s install' \
- % (type, package)
- raise error.PackageError(e_msg)
+ if not system_support:
+ e_msg = 'Client does not have package manager %s to handle %s install' \
+ % (type, package)
+ raise error.PackageError(e_msg)
- opt_args = ''
- if type == 'rpm':
- if nodeps:
- opt_args = opt_args + '--nodeps'
- install_command = 'rpm %s -U %s' % (opt_args, package)
- if type == 'dpkg':
- if nodeps:
- opt_args = opt_args + '--force-depends'
- install_command = 'dpkg %s -i %s' % (opt_args, package)
+ opt_args = ''
+ if type == 'rpm':
+ if nodeps:
+ opt_args = opt_args + '--nodeps'
+ install_command = 'rpm %s -U %s' % (opt_args, package)
+ if type == 'dpkg':
+ if nodeps:
+ opt_args = opt_args + '--force-depends'
+ install_command = 'dpkg %s -i %s' % (opt_args, package)
- # RPM source packages can be installed along with the binary versions
- # with this check
- if installed and not source:
- return 'Package %s is already installed' % package
+ # RPM source packages can be installed along with the binary versions
+ # with this check
+ if installed and not source:
+ return 'Package %s is already installed' % package
- # At this point, the most likely thing to go wrong is that there are
- # unmet dependencies for the package. We won't cover this case, at
- # least for now.
- utils.system(install_command)
- return 'Package %s was installed successfuly' % package
+ # At this point, the most likely thing to go wrong is that there are
+ # unmet dependencies for the package. We won't cover this case, at
+ # least for now.
+ utils.system(install_command)
+ return 'Package %s was installed successfuly' % package
def convert(package, destination_format):
- """\
- Convert packages with the 'alien' utility. If alien is not installed, it
- throws a NotImplementedError exception.
- returns: filename of the package generated.
- """
- try:
- os_dep.command('alien')
- except:
- e_msg = 'Cannot convert to %s, alien not installed' % destination_format
- raise error.TestError(e_msg)
+ """\
+ Convert packages with the 'alien' utility. If alien is not installed, it
+ throws a NotImplementedError exception.
+ returns: filename of the package generated.
+ """
+ try:
+ os_dep.command('alien')
+ except:
+ e_msg = 'Cannot convert to %s, alien not installed' % destination_format
+ raise error.TestError(e_msg)
- # alien supports converting to many formats, but its interesting to map
- # convertions only for the implemented package types.
- if destination_format == 'dpkg':
- deb_pattern = re.compile('[A-Za-z0-9_.-]*[.][d][e][b]')
- conv_output = utils.system_output('alien --to-deb %s 2>/dev/null' % package)
- converted_package = re.findall(deb_pattern, conv_output)[0]
- elif destination_format == 'rpm':
- rpm_pattern = re.compile('[A-Za-z0-9_.-]*[.][r][p][m]')
- conv_output = utils.system_output('alien --to-rpm %s 2>/dev/null' % package)
- converted_package = re.findall(rpm_pattern, conv_output)[0]
- else:
- e_msg = 'Convertion to format %s not implemented' % destination_format
- raise NotImplementedError(e_msg)
+ # alien supports converting to many formats, but its interesting to map
+ # convertions only for the implemented package types.
+ if destination_format == 'dpkg':
+ deb_pattern = re.compile('[A-Za-z0-9_.-]*[.][d][e][b]')
+ conv_output = utils.system_output('alien --to-deb %s 2>/dev/null' % package)
+ converted_package = re.findall(deb_pattern, conv_output)[0]
+ elif destination_format == 'rpm':
+ rpm_pattern = re.compile('[A-Za-z0-9_.-]*[.][r][p][m]')
+ conv_output = utils.system_output('alien --to-rpm %s 2>/dev/null' % package)
+ converted_package = re.findall(rpm_pattern, conv_output)[0]
+ else:
+ e_msg = 'Convertion to format %s not implemented' % destination_format
+ raise NotImplementedError(e_msg)
- print 'Package %s successfuly converted to %s' % \
- (os.path.basename(package), os.path.basename(converted_package))
- return os.path.abspath(converted_package)
+ print 'Package %s successfuly converted to %s' % \
+ (os.path.basename(package), os.path.basename(converted_package))
+ return os.path.abspath(converted_package)
def os_support():
- """\
- Returns a dictionary with host os package support info:
- - rpm: True if system supports rpm packages, False otherwise
- - dpkg: True if system supports dpkg packages, False otherwise
- - conversion: True if the system can convert packages (alien installed),
- or False otherwise
- """
- support_info = {}
- for package_manager in KNOWN_PACKAGE_MANAGERS:
- try:
- os_dep.command(package_manager)
- support_info[package_manager] = True
- except:
- support_info[package_manager] = False
+ """\
+ Returns a dictionary with host os package support info:
+ - rpm: True if system supports rpm packages, False otherwise
+ - dpkg: True if system supports dpkg packages, False otherwise
+ - conversion: True if the system can convert packages (alien installed),
+ or False otherwise
+ """
+ support_info = {}
+ for package_manager in KNOWN_PACKAGE_MANAGERS:
+ try:
+ os_dep.command(package_manager)
+ support_info[package_manager] = True
+ except:
+ support_info[package_manager] = False
- try:
- os_dep.command('alien')
- support_info['conversion'] = True
- except:
- support_info['conversion'] = False
+ try:
+ os_dep.command('alien')
+ support_info['conversion'] = True
+ except:
+ support_info['conversion'] = False
- return support_info
+ return support_info
diff --git a/client/bin/parallel.py b/client/bin/parallel.py
index a95b643..13a8b51 100644
--- a/client/bin/parallel.py
+++ b/client/bin/parallel.py
@@ -6,42 +6,42 @@
from autotest_lib.client.common_lib import error
def fork_start(tmp, l):
- sys.stdout.flush()
- sys.stderr.flush()
- pid = os.fork()
- if pid:
- # Parent
- return pid
+ sys.stdout.flush()
+ sys.stderr.flush()
+ pid = os.fork()
+ if pid:
+ # Parent
+ return pid
- try:
- try:
- l()
+ try:
+ try:
+ l()
- except error.AutotestError:
- raise
+ except error.AutotestError:
+ raise
- except:
- raise error.UnhandledError("test failed and threw:\n")
+ except:
+ raise error.UnhandledError("test failed and threw:\n")
- except Exception, detail:
- ename = tmp + "/debug/error-%d" % (os.getpid())
- pickle.dump(detail, open(ename, "w"))
+ except Exception, detail:
+ ename = tmp + "/debug/error-%d" % (os.getpid())
+ pickle.dump(detail, open(ename, "w"))
- sys.stdout.flush()
- sys.stderr.flush()
- os._exit(1)
+ sys.stdout.flush()
+ sys.stderr.flush()
+ os._exit(1)
- sys.stdout.flush()
- sys.stderr.flush()
- os._exit(0)
+ sys.stdout.flush()
+ sys.stderr.flush()
+ os._exit(0)
def fork_waitfor(tmp, pid):
- (pid, status) = os.waitpid(pid, 0)
+ (pid, status) = os.waitpid(pid, 0)
- ename = tmp + "/debug/error-%d" % pid
- if (os.path.exists(ename)):
- raise pickle.load(file(ename, 'r'))
+ ename = tmp + "/debug/error-%d" % pid
+ if (os.path.exists(ename)):
+ raise pickle.load(file(ename, 'r'))
- if (status != 0):
- raise error.TestError("test failed rc=%d" % (status))
+ if (status != 0):
+ raise error.TestError("test failed rc=%d" % (status))
diff --git a/client/bin/profiler.py b/client/bin/profiler.py
index b919542..ff821f9 100755
--- a/client/bin/profiler.py
+++ b/client/bin/profiler.py
@@ -1,25 +1,24 @@
class profiler:
- preserve_srcdir = False
+ preserve_srcdir = False
- def __init__(self, job):
- self.job = job
+ def __init__(self, job):
+ self.job = job
- def setup(self, *args):
- return
+ def setup(self, *args):
+ return
- def initialize(self, *args):
- return
+ def initialize(self, *args):
+ return
- def start(self, test):
- return
+ def start(self, test):
+ return
- def stop(self, test):
- return
+ def stop(self, test):
+ return
- def report(self, test):
- return
-
+ def report(self, test):
+ return
diff --git a/client/bin/profilers.py b/client/bin/profilers.py
index c080021..298cba9 100755
--- a/client/bin/profilers.py
+++ b/client/bin/profilers.py
@@ -4,71 +4,71 @@
class profilers:
- def __init__(self, job):
- self.job = job
- self.list = []
- self.profdir = job.autodir + '/profilers'
- self.tmpdir = job.tmpdir
- self.profile_run_only = False
+ def __init__(self, job):
+ self.job = job
+ self.list = []
+ self.profdir = job.autodir + '/profilers'
+ self.tmpdir = job.tmpdir
+ self.profile_run_only = False
- # add a profiler
- def add(self, profiler, *args, **dargs):
- try:
- sys.path.insert(0, self.job.profdir + '/' + profiler)
- exec 'import ' + profiler
- exec 'newprofiler = %s.%s(self)' % (profiler, profiler)
- finally:
- sys.path.pop(0)
- newprofiler.name = profiler
- newprofiler.bindir = self.profdir + '/' + profiler
- newprofiler.srcdir = newprofiler.bindir + '/src'
- newprofiler.tmpdir = self.tmpdir + '/' + profiler
- utils.update_version(newprofiler.srcdir, newprofiler.preserve_srcdir,
- newprofiler.version, newprofiler.setup,
- *args, **dargs)
- newprofiler.initialize(*args, **dargs)
- self.list.append(newprofiler)
+ # add a profiler
+ def add(self, profiler, *args, **dargs):
+ try:
+ sys.path.insert(0, self.job.profdir + '/' + profiler)
+ exec 'import ' + profiler
+ exec 'newprofiler = %s.%s(self)' % (profiler, profiler)
+ finally:
+ sys.path.pop(0)
+ newprofiler.name = profiler
+ newprofiler.bindir = self.profdir + '/' + profiler
+ newprofiler.srcdir = newprofiler.bindir + '/src'
+ newprofiler.tmpdir = self.tmpdir + '/' + profiler
+ utils.update_version(newprofiler.srcdir, newprofiler.preserve_srcdir,
+ newprofiler.version, newprofiler.setup,
+ *args, **dargs)
+ newprofiler.initialize(*args, **dargs)
+ self.list.append(newprofiler)
- # remove a profiler
- def delete(self, profiler):
- nukeme = None
- for p in self.list:
- if (p.name == profiler):
- nukeme = p
- self.list.remove(p)
+ # remove a profiler
+ def delete(self, profiler):
+ nukeme = None
+ for p in self.list:
+ if (p.name == profiler):
+ nukeme = p
+ self.list.remove(p)
- # are any profilers enabled ?
- def present(self):
- if self.list:
- return 1
- else:
- return 0
+ # are any profilers enabled ?
+ def present(self):
+ if self.list:
+ return 1
+ else:
+ return 0
- # Returns True if job is supposed to be run only with profiling turned
- # on, False otherwise
- def only(self):
- return self.profile_run_only
+ # Returns True if job is supposed to be run only with profiling turned
+ # on, False otherwise
+ def only(self):
+ return self.profile_run_only
- # Changes the flag which determines whether or not the job is to be
- # run without profilers at all
- def set_only(self, value):
- self.profile_run_only = value
+ # Changes the flag which determines whether or not the job is to be
+ # run without profilers at all
+ def set_only(self, value):
+ self.profile_run_only = value
- # Start all enabled profilers
- def start(self, test):
- for p in self.list:
- p.start(test)
+ # Start all enabled profilers
+ def start(self, test):
+ for p in self.list:
+ p.start(test)
- # Stop all enabled profilers
- def stop(self, test):
- for p in self.list:
- p.stop(test)
+ # Stop all enabled profilers
+ def stop(self, test):
+ for p in self.list:
+ p.stop(test)
- # Report on all enabled profilers
- def report(self, test):
- for p in self.list:
- p.report(test)
+ # Report on all enabled profilers
+ def report(self, test):
+ for p in self.list:
+ p.report(test)
diff --git a/client/bin/sysinfo.py b/client/bin/sysinfo.py
index 2f605ea..107229c 100755
--- a/client/bin/sysinfo.py
+++ b/client/bin/sysinfo.py
@@ -6,14 +6,14 @@
from autotest_lib.client.common_lib import utils
try:
- from autotest_lib.client.bin import site_sysinfo
- local = True
+ from autotest_lib.client.bin import site_sysinfo
+ local = True
except ImportError:
- local = False
+ local = False
# stuff to log per reboot
-files = ['/proc/pci', '/proc/meminfo', '/proc/slabinfo', '/proc/version',
- '/proc/cpuinfo', '/proc/cmdline', '/proc/modules']
+files = ['/proc/pci', '/proc/meminfo', '/proc/slabinfo', '/proc/version',
+ '/proc/cpuinfo', '/proc/cmdline', '/proc/modules']
# commands = ['lshw'] # this causes problems triggering CDROM drives
commands = ['uname -a', 'lspci -vvn', 'gcc --version', 'ld --version',
'mount', 'hostname']
@@ -21,84 +21,84 @@
def run_command(command, output):
- parts = command.split(None, 1)
- cmd = parts[0]
- if len(parts) > 1:
- args = parts[1]
- else:
- args = ''
- for dir in path:
- pathname = dir + '/' + cmd
- if not os.path.exists(pathname):
- continue
- tmp_cmd = "%s %s > %s 2> /dev/null" % (pathname, args, output)
- utils.system(tmp_cmd)
+ parts = command.split(None, 1)
+ cmd = parts[0]
+ if len(parts) > 1:
+ args = parts[1]
+ else:
+ args = ''
+ for dir in path:
+ pathname = dir + '/' + cmd
+ if not os.path.exists(pathname):
+ continue
+ tmp_cmd = "%s %s > %s 2> /dev/null" % (pathname, args, output)
+ utils.system(tmp_cmd)
def reboot_count():
- if not glob.glob('*'):
- return -1 # No reboots, initial data not logged
- else:
- return len(glob.glob('reboot*'))
-
-
+ if not glob.glob('*'):
+ return -1 # No reboots, initial data not logged
+ else:
+ return len(glob.glob('reboot*'))
+
+
def boot_subdir(reboot_count):
- """subdir of job sysinfo"""
- if reboot_count == 0:
- return '.'
- else:
- return 'reboot%d' % reboot_count
+ """subdir of job sysinfo"""
+ if reboot_count == 0:
+ return '.'
+ else:
+ return 'reboot%d' % reboot_count
def log_per_reboot_data(sysinfo_dir):
- """we log this data when the job starts, and again after any reboot"""
- pwd = os.getcwd()
- try:
- os.chdir(sysinfo_dir)
- subdir = boot_subdir(reboot_count() + 1)
- if not os.path.exists(subdir):
- os.mkdir(subdir)
- os.chdir(os.path.join(sysinfo_dir, subdir))
- _log_per_reboot_data()
- finally:
- os.chdir(pwd)
+ """we log this data when the job starts, and again after any reboot"""
+ pwd = os.getcwd()
+ try:
+ os.chdir(sysinfo_dir)
+ subdir = boot_subdir(reboot_count() + 1)
+ if not os.path.exists(subdir):
+ os.mkdir(subdir)
+ os.chdir(os.path.join(sysinfo_dir, subdir))
+ _log_per_reboot_data()
+ finally:
+ os.chdir(pwd)
def _log_per_reboot_data():
- """system info to log before each step of the job"""
- for command in commands:
- run_command(command, re.sub(r'\s', '_', command))
+ """system info to log before each step of the job"""
+ for command in commands:
+ run_command(command, re.sub(r'\s', '_', command))
- for file in files:
- if (os.path.exists(file)):
- shutil.copyfile(file, os.path.basename(file))
+ for file in files:
+ if (os.path.exists(file)):
+ shutil.copyfile(file, os.path.basename(file))
- utils.system('dmesg -c > dmesg', ignore_status=True)
- utils.system('df -mP > df', ignore_status=True)
- if local:
- site_sysinfo.log_per_reboot_data()
+ utils.system('dmesg -c > dmesg', ignore_status=True)
+ utils.system('df -mP > df', ignore_status=True)
+ if local:
+ site_sysinfo.log_per_reboot_data()
def log_after_each_test(test_sysinfo_dir, job_sysinfo_dir):
- """log things that change after each test (called from test.py)"""
- pwd = os.getcwd()
- try:
- os.chdir(job_sysinfo_dir)
- reboot_subdir = boot_subdir(reboot_count())
- reboot_dir = os.path.join(job_sysinfo_dir, reboot_subdir)
- assert os.path.exists(reboot_dir)
+ """log things that change after each test (called from test.py)"""
+ pwd = os.getcwd()
+ try:
+ os.chdir(job_sysinfo_dir)
+ reboot_subdir = boot_subdir(reboot_count())
+ reboot_dir = os.path.join(job_sysinfo_dir, reboot_subdir)
+ assert os.path.exists(reboot_dir)
- os.makedirs(test_sysinfo_dir)
- os.chdir(test_sysinfo_dir)
- utils.system('ln -s %s reboot_current' % reboot_dir)
+ os.makedirs(test_sysinfo_dir)
+ os.chdir(test_sysinfo_dir)
+ utils.system('ln -s %s reboot_current' % reboot_dir)
- utils.system('dmesg -c > dmesg', ignore_status=True)
- utils.system('df -mP > df', ignore_status=True)
- if local:
- site_sysinfo.log_after_each_test()
- finally:
- os.chdir(pwd)
-
-
+ utils.system('dmesg -c > dmesg', ignore_status=True)
+ utils.system('df -mP > df', ignore_status=True)
+ if local:
+ site_sysinfo.log_after_each_test()
+ finally:
+ os.chdir(pwd)
+
+
if __name__ == '__main__':
- log_per_reboot_data()
+ log_per_reboot_data()
diff --git a/client/bin/test.py b/client/bin/test.py
index 78dbdd1..5d5d4f9 100755
--- a/client/bin/test.py
+++ b/client/bin/test.py
@@ -3,20 +3,20 @@
# Shell class for a test, inherited by all individual tests
#
# Methods:
-# __init__ initialise
-# initialize run once for each job
-# setup run once for each new version of the test installed
-# run run the test (wrapped by job.run_test())
+# __init__ initialise
+# initialize run once for each job
+# setup run once for each new version of the test installed
+# run run the test (wrapped by job.run_test())
#
# Data:
-# job backreference to the job this test instance is part of
-# outputdir eg. results/<job>/<testname.tag>
-# resultsdir eg. results/<job>/<testname.tag>/results
-# profdir eg. results/<job>/<testname.tag>/profiling
-# debugdir eg. results/<job>/<testname.tag>/debug
-# bindir eg. tests/<test>
-# src eg. tests/<test>/src
-# tmpdir eg. tmp/<testname.tag>
+# job backreference to the job this test instance is part of
+# outputdir eg. results/<job>/<testname.tag>
+# resultsdir eg. results/<job>/<testname.tag>/results
+# profdir eg. results/<job>/<testname.tag>/profiling
+# debugdir eg. results/<job>/<testname.tag>/debug
+# bindir eg. tests/<test>
+# src eg. tests/<test>/src
+# tmpdir eg. tmp/<testname.tag>
import os, traceback
@@ -26,23 +26,23 @@
class test(common_test.base_test):
- pass
+ pass
testname = common_test.testname
def _grab_sysinfo(mytest):
- try:
- sysinfo_dir = os.path.join(mytest.outputdir, 'sysinfo')
- sysinfo.log_after_each_test(sysinfo_dir, mytest.job.sysinfodir)
- if os.path.exists(mytest.tmpdir):
- utils.system('rm -rf ' + mytest.tmpdir)
- except:
- print 'after-test error:'
- traceback.print_exc(file=sys.stdout)
+ try:
+ sysinfo_dir = os.path.join(mytest.outputdir, 'sysinfo')
+ sysinfo.log_after_each_test(sysinfo_dir, mytest.job.sysinfodir)
+ if os.path.exists(mytest.tmpdir):
+ utils.system('rm -rf ' + mytest.tmpdir)
+ except:
+ print 'after-test error:'
+ traceback.print_exc(file=sys.stdout)
def runtest(job, url, tag, args, dargs):
- common_test.runtest(job, url, tag, args, dargs,
- locals(), globals(), _grab_sysinfo)
+ common_test.runtest(job, url, tag, args, dargs,
+ locals(), globals(), _grab_sysinfo)
diff --git a/client/bin/test_config.py b/client/bin/test_config.py
index e926813..d8a5337 100644
--- a/client/bin/test_config.py
+++ b/client/bin/test_config.py
@@ -13,79 +13,79 @@
__all__ = ['config_loader']
class config_loader:
- """Base class of the configuration parser"""
- def __init__(self, cfg, tmpdir = '/tmp'):
- """\
- Instantiate ConfigParser and provide the file like object that we'll
- use to read configuration data from.
- Args:
- * cfg: Where we'll get configuration data. It can be either:
- * A URL containing the file
- * A valid file path inside the filesystem
- * A string containing configuration data
- * tmpdir: Where we'll dump the temporary conf files. The default
- is the /tmp directory.
- """
- # Base Parser
- self.parser = ConfigParser()
- # File is already a file like object
- if hasattr(cfg, 'read'):
- self.cfg = cfg
- self.parser.readfp(self.cfg)
- elif isinstance(cfg, types.StringTypes):
- # Config file is a URL. Download it to a temp dir
- if cfg.startswith('http') or cfg.startswith('ftp'):
- self.cfg = path.join(tmpdir, path.basename(cfg))
- utils.urlretrieve(cfg, self.cfg)
- self.parser.read(self.cfg)
- # Config is a valid filesystem path to a file.
- elif path.exists(path.abspath(cfg)):
- if path.isfile(cfg):
- self.cfg = path.abspath(cfg)
- self.parser.read(self.cfg)
- else:
- e_msg = 'Invalid config file path: %s' % cfg
- raise IOError(e_msg)
- # Config file is just a string, convert it to a python file like
- # object using StringIO
- else:
- self.cfg = StringIO(cfg)
- self.parser.readfp(self.cfg)
+ """Base class of the configuration parser"""
+ def __init__(self, cfg, tmpdir = '/tmp'):
+ """\
+ Instantiate ConfigParser and provide the file like object that we'll
+ use to read configuration data from.
+ Args:
+ * cfg: Where we'll get configuration data. It can be either:
+ * A URL containing the file
+ * A valid file path inside the filesystem
+ * A string containing configuration data
+ * tmpdir: Where we'll dump the temporary conf files. The default
+ is the /tmp directory.
+ """
+ # Base Parser
+ self.parser = ConfigParser()
+ # File is already a file like object
+ if hasattr(cfg, 'read'):
+ self.cfg = cfg
+ self.parser.readfp(self.cfg)
+ elif isinstance(cfg, types.StringTypes):
+ # Config file is a URL. Download it to a temp dir
+ if cfg.startswith('http') or cfg.startswith('ftp'):
+ self.cfg = path.join(tmpdir, path.basename(cfg))
+ utils.urlretrieve(cfg, self.cfg)
+ self.parser.read(self.cfg)
+ # Config is a valid filesystem path to a file.
+ elif path.exists(path.abspath(cfg)):
+ if path.isfile(cfg):
+ self.cfg = path.abspath(cfg)
+ self.parser.read(self.cfg)
+ else:
+ e_msg = 'Invalid config file path: %s' % cfg
+ raise IOError(e_msg)
+ # Config file is just a string, convert it to a python file like
+ # object using StringIO
+ else:
+ self.cfg = StringIO(cfg)
+ self.parser.readfp(self.cfg)
- def get(self, section, name, default=None):
- """Get the value of a option.
+ def get(self, section, name, default=None):
+ """Get the value of a option.
- Section of the config file and the option name.
- You can pass a default value if the option doesn't exist.
- """
- if not self.parser.has_option(section, name):
- return default
- return self.parser.get(section, name)
+ Section of the config file and the option name.
+ You can pass a default value if the option doesn't exist.
+ """
+ if not self.parser.has_option(section, name):
+ return default
+ return self.parser.get(section, name)
- def set(self, section, option, value):
- """Set an option.
+ def set(self, section, option, value):
+ """Set an option.
- This change is not persistent unless saved with 'save()'.
- """
- if not self.parser.has_section(section):
- self.parser.add_section(section)
- return self.parser.set(section, name, value)
+ This change is not persistent unless saved with 'save()'.
+ """
+ if not self.parser.has_section(section):
+ self.parser.add_section(section)
+ return self.parser.set(section, name, value)
- def remove(self, section, name):
- """Remove an option."""
- if self.parser.has_section(section):
- self.parser.remove_option(section, name)
+ def remove(self, section, name):
+ """Remove an option."""
+ if self.parser.has_section(section):
+ self.parser.remove_option(section, name)
- def save(self):
- """Save the configuration file with all modifications"""
- if not self.filename:
- return
- fileobj = file(self.filename, 'w')
- try:
- self.parser.write(fileobj)
- finally:
- fileobj.close()
+ def save(self):
+ """Save the configuration file with all modifications"""
+ if not self.filename:
+ return
+ fileobj = file(self.filename, 'w')
+ try:
+ self.parser.write(fileobj)
+ finally:
+ fileobj.close()
diff --git a/client/bin/xen.py b/client/bin/xen.py
index d69b2c5..a792427 100644
--- a/client/bin/xen.py
+++ b/client/bin/xen.py
@@ -8,201 +8,201 @@
class xen(kernel.kernel):
- def log(self, msg):
- print msg
- self.logfile.write('%s\n' % msg)
+ def log(self, msg):
+ print msg
+ self.logfile.write('%s\n' % msg)
- def __init__(self, job, base_tree, results_dir, tmp_dir, build_dir, \
- leave = False, kjob = None):
- # call base-class
- kernel.kernel.__init__(self, job, base_tree, results_dir, \
- tmp_dir, build_dir, leave)
- self.kjob = kjob
+ def __init__(self, job, base_tree, results_dir, tmp_dir, build_dir, \
+ leave = False, kjob = None):
+ # call base-class
+ kernel.kernel.__init__(self, job, base_tree, results_dir, \
+ tmp_dir, build_dir, leave)
+ self.kjob = kjob
- def config(self, config_file, config_list = None):
- raise NotImplementedError('config() not implemented for xen')
+ def config(self, config_file, config_list = None):
+ raise NotImplementedError('config() not implemented for xen')
- def build(self, make_opts = '', logfile = '', extraversion='autotest'):
- """build xen
+ def build(self, make_opts = '', logfile = '', extraversion='autotest'):
+ """build xen
- make_opts
- additional options to make, if any
- """
- self.log('running build')
- os_dep.commands('gcc', 'make')
- # build xen with extraversion flag
- os.environ['XEN_EXTRAVERSION'] = '-unstable-%s'% extraversion
- if logfile == '':
- logfile = os.path.join(self.log_dir, 'xen_build')
- os.chdir(self.build_dir)
- self.log('log_dir: %s ' % os.path.join(self.log_dir, 'stdout'))
- self.job.stdout.tee_redirect(logfile + '.stdout')
- self.job.stderr.tee_redirect(logfile + '.stderr')
+ make_opts
+ additional options to make, if any
+ """
+ self.log('running build')
+ os_dep.commands('gcc', 'make')
+ # build xen with extraversion flag
+ os.environ['XEN_EXTRAVERSION'] = '-unstable-%s'% extraversion
+ if logfile == '':
+ logfile = os.path.join(self.log_dir, 'xen_build')
+ os.chdir(self.build_dir)
+ self.log('log_dir: %s ' % os.path.join(self.log_dir, 'stdout'))
+ self.job.stdout.tee_redirect(logfile + '.stdout')
+ self.job.stderr.tee_redirect(logfile + '.stderr')
- # build xen hypervisor and user-space tools
- targets = ['xen', 'tools']
- threads = 2 * autotest_utils.count_cpus()
- for t in targets:
- build_string = 'make -j %d %s %s' % (threads, make_opts, t)
- self.log('build_string: %s' % build_string)
- system(build_string)
+ # build xen hypervisor and user-space tools
+ targets = ['xen', 'tools']
+ threads = 2 * autotest_utils.count_cpus()
+ for t in targets:
+ build_string = 'make -j %d %s %s' % (threads, make_opts, t)
+ self.log('build_string: %s' % build_string)
+ system(build_string)
- # make a kernel job out of the kernel from the xen src if one isn't provided
- if self.kjob == None:
- # get xen kernel tree ready
- self.log("prep-ing xen'ified kernel source tree")
- system('make prep-kernels')
+ # make a kernel job out of the kernel from the xen src if one isn't provided
+ if self.kjob == None:
+ # get xen kernel tree ready
+ self.log("prep-ing xen'ified kernel source tree")
+ system('make prep-kernels')
- v = self.get_xen_kernel_build_ver()
- self.log('building xen kernel version: %s' % v)
+ v = self.get_xen_kernel_build_ver()
+ self.log('building xen kernel version: %s' % v)
- # build xen-ified kernel in xen tree
- kernel_base_tree = os.path.join(self.build_dir, \
- 'linux-%s' % self.get_xen_kernel_build_ver())
+ # build xen-ified kernel in xen tree
+ kernel_base_tree = os.path.join(self.build_dir, \
+ 'linux-%s' % self.get_xen_kernel_build_ver())
- self.log('kernel_base_tree = %s' % kernel_base_tree)
- # fix up XENGUEST value in EXTRAVERSION; we can't have
- # files with '$(XENGEUST)' in the name, =(
- self.fix_up_xen_kernel_makefile(kernel_base_tree)
+ self.log('kernel_base_tree = %s' % kernel_base_tree)
+ # fix up XENGUEST value in EXTRAVERSION; we can't have
+ # files with '$(XENGEUST)' in the name, =(
+ self.fix_up_xen_kernel_makefile(kernel_base_tree)
- # make the kernel job
- self.kjob = self.job.kernel(kernel_base_tree)
+ # make the kernel job
+ self.kjob = self.job.kernel(kernel_base_tree)
- # hardcoding dom0 config (no modules for testing, yay!)
- # FIXME: probe host to determine which config to pick
- c = self.build_dir + '/buildconfigs/linux-defconfig_xen0_x86_32'
- self.log('using kernel config: %s ' % c)
- self.kjob.config(c)
+ # hardcoding dom0 config (no modules for testing, yay!)
+ # FIXME: probe host to determine which config to pick
+ c = self.build_dir + '/buildconfigs/linux-defconfig_xen0_x86_32'
+ self.log('using kernel config: %s ' % c)
+ self.kjob.config(c)
- # Xen's kernel tree sucks; doesn't use bzImage, but vmlinux
- self.kjob.set_build_target('vmlinuz')
+ # Xen's kernel tree sucks; doesn't use bzImage, but vmlinux
+ self.kjob.set_build_target('vmlinuz')
- # also, the vmlinuz is not out in arch/*/boot, ARGH! more hackery
- self.kjob.set_build_image(self.job.tmpdir + '/build/linux/vmlinuz')
+ # also, the vmlinuz is not out in arch/*/boot, ARGH! more hackery
+ self.kjob.set_build_image(self.job.tmpdir + '/build/linux/vmlinuz')
- self.kjob.build()
+ self.kjob.build()
- self.job.stdout.restore()
- self.job.stderr.restore()
+ self.job.stdout.restore()
+ self.job.stderr.restore()
- xen_version = self.get_xen_build_ver()
- self.log('BUILD VERSION: Xen: %s Kernel:%s' % \
- (xen_version, self.kjob.get_kernel_build_ver()))
+ xen_version = self.get_xen_build_ver()
+ self.log('BUILD VERSION: Xen: %s Kernel:%s' % \
+ (xen_version, self.kjob.get_kernel_build_ver()))
- def build_timed(self, *args, **kwds):
- raise NotImplementedError('build_timed() not implemented')
+ def build_timed(self, *args, **kwds):
+ raise NotImplementedError('build_timed() not implemented')
- def install(self, tag='', prefix = '/', extraversion='autotest'):
- """make install in the kernel tree"""
- self.log('Installing ...')
+ def install(self, tag='', prefix = '/', extraversion='autotest'):
+ """make install in the kernel tree"""
+ self.log('Installing ...')
- os.chdir(self.build_dir)
+ os.chdir(self.build_dir)
- if not os.path.isdir(prefix):
- os.mkdir(prefix)
- self.boot_dir = os.path.join(prefix, 'boot')
- if not os.path.isdir(self.boot_dir):
- os.mkdir(self.boot_dir)
+ if not os.path.isdir(prefix):
+ os.mkdir(prefix)
+ self.boot_dir = os.path.join(prefix, 'boot')
+ if not os.path.isdir(self.boot_dir):
+ os.mkdir(self.boot_dir)
- # remember what we are going to install
- xen_version = '%s-%s' % (self.get_xen_build_ver(), extraversion)
- self.xen_image = self.boot_dir + '/xen-' + xen_version + '.gz'
- self.xen_syms = self.boot_dir + '/xen-syms-' + xen_version
+ # remember what we are going to install
+ xen_version = '%s-%s' % (self.get_xen_build_ver(), extraversion)
+ self.xen_image = self.boot_dir + '/xen-' + xen_version + '.gz'
+ self.xen_syms = self.boot_dir + '/xen-syms-' + xen_version
- self.log('Installing Xen ...')
- os.environ['XEN_EXTRAVERSION'] = '-unstable-%s'% extraversion
+ self.log('Installing Xen ...')
+ os.environ['XEN_EXTRAVERSION'] = '-unstable-%s'% extraversion
- # install xen
- system('make DESTDIR=%s -C xen install' % prefix)
+ # install xen
+ system('make DESTDIR=%s -C xen install' % prefix)
- # install tools
- system('make DESTDIR=%s -C tools install' % prefix)
+ # install tools
+ system('make DESTDIR=%s -C tools install' % prefix)
- # install kernel
- ktag = self.kjob.get_kernel_build_ver()
- kprefix = prefix
- self.kjob.install(tag=ktag, prefix=kprefix)
+ # install kernel
+ ktag = self.kjob.get_kernel_build_ver()
+ kprefix = prefix
+ self.kjob.install(tag=ktag, prefix=kprefix)
- def add_to_bootloader(self, tag='autotest', args=''):
- """ add this kernel to bootloader, taking an
- optional parameter of space separated parameters
- e.g.: kernel.add_to_bootloader('mykernel', 'ro acpi=off')
- """
+ def add_to_bootloader(self, tag='autotest', args=''):
+ """ add this kernel to bootloader, taking an
+ optional parameter of space separated parameters
+ e.g.: kernel.add_to_bootloader('mykernel', 'ro acpi=off')
+ """
- # turn on xen mode
- self.job.bootloader.enable_xen_mode()
+ # turn on xen mode
+ self.job.bootloader.enable_xen_mode()
- # remove existing entry if present
- self.job.bootloader.remove_kernel(tag)
+ # remove existing entry if present
+ self.job.bootloader.remove_kernel(tag)
- # add xen and xen kernel
- self.job.bootloader.add_kernel(self.kjob.image, tag, \
- self.kjob.initrd, self.xen_image)
+ # add xen and xen kernel
+ self.job.bootloader.add_kernel(self.kjob.image, tag, \
+ self.kjob.initrd, self.xen_image)
- # if no args passed, populate from /proc/cmdline
- if not args:
- args = open('/proc/cmdline', 'r').readline().strip()
+ # if no args passed, populate from /proc/cmdline
+ if not args:
+ args = open('/proc/cmdline', 'r').readline().strip()
- # add args to entry one at a time
- for a in args.split(' '):
- self.job.bootloader.add_args(tag, a)
+ # add args to entry one at a time
+ for a in args.split(' '):
+ self.job.bootloader.add_args(tag, a)
- # turn off xen mode
- self.job.bootloader.disable_xen_mode()
+ # turn off xen mode
+ self.job.bootloader.disable_xen_mode()
- def get_xen_kernel_build_ver(self):
- """Check xen buildconfig for current kernel version"""
- version = patchlevel = sublevel = ''
- extraversion = localversion = ''
+ def get_xen_kernel_build_ver(self):
+ """Check xen buildconfig for current kernel version"""
+ version = patchlevel = sublevel = ''
+ extraversion = localversion = ''
- version_file = self.build_dir + '/buildconfigs/mk.linux-2.6-xen'
+ version_file = self.build_dir + '/buildconfigs/mk.linux-2.6-xen'
- for line in open(version_file, 'r').readlines():
- if line.startswith('LINUX_VER'):
- start = line.index('=') + 1
- version = line[start:].strip() + "-xen"
- break
+ for line in open(version_file, 'r').readlines():
+ if line.startswith('LINUX_VER'):
+ start = line.index('=') + 1
+ version = line[start:].strip() + "-xen"
+ break
- return version
+ return version
- def fix_up_xen_kernel_makefile(self, kernel_dir):
- """Fix up broken EXTRAVERSION in xen-ified Linux kernel Makefile"""
- xenguest = ''
- makefile = kernel_dir + '/Makefile'
+ def fix_up_xen_kernel_makefile(self, kernel_dir):
+ """Fix up broken EXTRAVERSION in xen-ified Linux kernel Makefile"""
+ xenguest = ''
+ makefile = kernel_dir + '/Makefile'
- for line in open(makefile, 'r').readlines():
- if line.startswith('XENGUEST'):
- start = line.index('=') + 1
- xenguest = line[start:].strip()
- break;
+ for line in open(makefile, 'r').readlines():
+ if line.startswith('XENGUEST'):
+ start = line.index('=') + 1
+ xenguest = line[start:].strip()
+ break;
- # change out $XENGUEST in EXTRAVERSION line
- system('sed -i.old "s,\$(XENGUEST),%s," %s' % \
- (xenguest, makefile))
+ # change out $XENGUEST in EXTRAVERSION line
+ system('sed -i.old "s,\$(XENGUEST),%s," %s' % \
+ (xenguest, makefile))
- def get_xen_build_ver(self):
- """Check Makefile and .config to return kernel version"""
- version = patchlevel = sublevel = ''
- extraversion = localversion = ''
+ def get_xen_build_ver(self):
+ """Check Makefile and .config to return kernel version"""
+ version = patchlevel = sublevel = ''
+ extraversion = localversion = ''
- for line in open(self.build_dir + '/xen/Makefile', 'r').readlines():
- if line.startswith('export XEN_VERSION'):
- start = line.index('=') + 1
- version = line[start:].strip()
- if line.startswith('export XEN_SUBVERSION'):
- start = line.index('=') + 1
- sublevel = line[start:].strip()
- if line.startswith('export XEN_EXTRAVERSION'):
- start = line.index('=') + 1
- extraversion = line[start:].strip()
+ for line in open(self.build_dir + '/xen/Makefile', 'r').readlines():
+ if line.startswith('export XEN_VERSION'):
+ start = line.index('=') + 1
+ version = line[start:].strip()
+ if line.startswith('export XEN_SUBVERSION'):
+ start = line.index('=') + 1
+ sublevel = line[start:].strip()
+ if line.startswith('export XEN_EXTRAVERSION'):
+ start = line.index('=') + 1
+ extraversion = line[start:].strip()
- return "%s.%s%s" % (version, sublevel, extraversion)
+ return "%s.%s%s" % (version, sublevel, extraversion)
diff --git a/client/common_lib/barrier.py b/client/common_lib/barrier.py
index bc6bb61..d9da330 100755
--- a/client/common_lib/barrier.py
+++ b/client/common_lib/barrier.py
@@ -6,462 +6,462 @@
class BarrierError(error.JobError):
- pass
+ pass
class barrier:
- """ Multi-machine barrier support
+ """ Multi-machine barrier support
- Provides multi-machine barrier mechanism. Execution
- stopping until all members arrive at the barrier.
+ Provides multi-machine barrier mechanism. Execution
+ stopping until all members arrive at the barrier.
- When a barrier is forming the master node (first in sort
- order) in the set accepts connections from each member
- of the set. As they arrive they indicate the barrier
- they are joining and their identifier (their hostname
- or IP address and optional tag). They are then asked
- to wait. When all members are present the master node
- then checks that each member is still responding via a
- ping/pong exchange. If this is successful then everyone
- has checked in at the barrier. We then tell everyone
- they may continue via a rlse message.
+ When a barrier is forming the master node (first in sort
+ order) in the set accepts connections from each member
+ of the set. As they arrive they indicate the barrier
+ they are joining and their identifier (their hostname
+ or IP address and optional tag). They are then asked
+ to wait. When all members are present the master node
+ then checks that each member is still responding via a
+ ping/pong exchange. If this is successful then everyone
+ has checked in at the barrier. We then tell everyone
+ they may continue via a rlse message.
- Where the master is not the first to reach the barrier
- the client connects will fail. Client will retry until
- they either succeed in connecting to master or the overal
- timeout is exceeded.
+ Where the master is not the first to reach the barrier
+ the client connects will fail. Client will retry until
+ they either succeed in connecting to master or the overal
+ timeout is exceeded.
- As an example here is the exchange for a three node
- barrier called 'TAG'
+ As an example here is the exchange for a three node
+ barrier called 'TAG'
- MASTER CLIENT1 CLIENT2
- <-------------TAG C1-------------
- --------------wait-------------->
- [...]
- <-------------TAG C2-----------------------------
- --------------wait------------------------------>
- [...]
- --------------ping-------------->
- <-------------pong---------------
- --------------ping------------------------------>
- <-------------pong-------------------------------
- ----- BARRIER conditions MET -----
- --------------rlse-------------->
- --------------rlse------------------------------>
+ MASTER CLIENT1 CLIENT2
+ <-------------TAG C1-------------
+ --------------wait-------------->
+ [...]
+ <-------------TAG C2-----------------------------
+ --------------wait------------------------------>
+ [...]
+ --------------ping-------------->
+ <-------------pong---------------
+ --------------ping------------------------------>
+ <-------------pong-------------------------------
+ ----- BARRIER conditions MET -----
+ --------------rlse-------------->
+ --------------rlse------------------------------>
- Note that once the last client has responded to pong the
- barrier is implicitly deemed satisifed, they have all
- acknowledged their presence. If we fail to send any
- of the rlse messages the barrier is still a success,
- the failed host has effectively broken 'right at the
- beginning' of the post barrier execution window.
+ Note that once the last client has responded to pong the
+ barrier is implicitly deemed satisifed, they have all
+ acknowledged their presence. If we fail to send any
+ of the rlse messages the barrier is still a success,
+ the failed host has effectively broken 'right at the
+ beginning' of the post barrier execution window.
- In addition, there is another rendevous, that makes each slave a server
- and the master a client. The connection process and usage is still the
- same but allows barriers from machines that only have a one-way
- connection initiation. This is called rendevous_servers.
+ In addition, there is another rendevous, that makes each slave a server
+ and the master a client. The connection process and usage is still the
+ same but allows barriers from machines that only have a one-way
+ connection initiation. This is called rendevous_servers.
- For example:
- if ME == SERVER:
- server start
+ For example:
+ if ME == SERVER:
+ server start
- b = job.barrier(ME, 'server-up', 120)
- b.rendevous(CLIENT, SERVER)
+ b = job.barrier(ME, 'server-up', 120)
+ b.rendevous(CLIENT, SERVER)
- if ME == CLIENT:
- client run
+ if ME == CLIENT:
+ client run
- b = job.barrier(ME, 'test-complete', 3600)
- b.rendevous(CLIENT, SERVER)
+ b = job.barrier(ME, 'test-complete', 3600)
+ b.rendevous(CLIENT, SERVER)
- if ME == SERVER:
- server stop
+ if ME == SERVER:
+ server stop
- Properties:
- hostid
- My hostname/IP address + optional tag
- tag
- Symbolic name of the barrier in progress
- port
- TCP port used for this barrier
- timeout
- Maximum time to wait for a the barrier to meet
- start
- Timestamp when we started waiting
- members
- All members we expect to find in the barrier
- seen
- Number of clients seen (should be the length of waiting)
- waiting
- Clients who have checked in and are waiting (master)
- masterid
- Hostname/IP address + optional tag of selected master
- """
+ Properties:
+ hostid
+ My hostname/IP address + optional tag
+ tag
+ Symbolic name of the barrier in progress
+ port
+ TCP port used for this barrier
+ timeout
+ Maximum time to wait for a the barrier to meet
+ start
+ Timestamp when we started waiting
+ members
+ All members we expect to find in the barrier
+ seen
+ Number of clients seen (should be the length of waiting)
+ waiting
+ Clients who have checked in and are waiting (master)
+ masterid
+ Hostname/IP address + optional tag of selected master
+ """
- def __init__(self, hostid, tag, timeout, port=63000):
- self.hostid = hostid
- self.tag = tag
- self.port = port
- self.timeout = timeout
+ def __init__(self, hostid, tag, timeout, port=63000):
+ self.hostid = hostid
+ self.tag = tag
+ self.port = port
+ self.timeout = timeout
- self.report("tag=%s port=%d timeout=%d" \
- % (self.tag, self.port, self.timeout))
+ self.report("tag=%s port=%d timeout=%d" \
+ % (self.tag, self.port, self.timeout))
- def get_host_from_id(self, id):
- # Remove any trailing local identifier following a #.
- # This allows multiple members per host which is particularly
- # helpful in testing.
- return id.split('#')[0]
+ def get_host_from_id(self, id):
+ # Remove any trailing local identifier following a #.
+ # This allows multiple members per host which is particularly
+ # helpful in testing.
+ return id.split('#')[0]
- def report(self, out):
- print "barrier:", self.hostid, out
- sys.stdout.flush()
+ def report(self, out):
+ print "barrier:", self.hostid, out
+ sys.stdout.flush()
- def update_timeout(self, timeout):
- self.timeout = (time() - self.start) + timeout
+ def update_timeout(self, timeout):
+ self.timeout = (time() - self.start) + timeout
- def remaining(self):
- timeout = self.timeout - (time() - self.start)
- if (timeout <= 0):
- raise BarrierError("timeout waiting for barrier")
+ def remaining(self):
+ timeout = self.timeout - (time() - self.start)
+ if (timeout <= 0):
+ raise BarrierError("timeout waiting for barrier")
- self.report("remaining: %d" % (timeout))
- return timeout
+ self.report("remaining: %d" % (timeout))
+ return timeout
- def master_welcome(self, connection):
- (client, addr) = connection
- name = None
+ def master_welcome(self, connection):
+ (client, addr) = connection
+ name = None
- client.settimeout(5)
- try:
- # Get the clients name.
- intro = client.recv(1024)
- intro = intro.strip("\r\n")
+ client.settimeout(5)
+ try:
+ # Get the clients name.
+ intro = client.recv(1024)
+ intro = intro.strip("\r\n")
- (tag, name) = intro.split(' ')
+ (tag, name) = intro.split(' ')
- self.report("new client tag=%s, name=%s" % (tag, name))
+ self.report("new client tag=%s, name=%s" % (tag, name))
- # Ok, we know who is trying to attach. Confirm that
- # they are coming to the same meeting. Also, everyone
- # should be using a unique handle (their IP address).
- # If we see a duplicate, something _bad_ has happened
- # so drop them now.
- if self.tag != tag:
- self.report("client arriving for the " \
- "wrong barrier")
- client.settimeout(5)
- client.send("!tag")
- client.close()
- return
- elif name in self.waiting:
- self.report("duplicate client")
- client.settimeout(5)
- client.send("!dup")
- client.close()
- return
-
- # Acknowledge the client
- client.send("wait")
+ # Ok, we know who is trying to attach. Confirm that
+ # they are coming to the same meeting. Also, everyone
+ # should be using a unique handle (their IP address).
+ # If we see a duplicate, something _bad_ has happened
+ # so drop them now.
+ if self.tag != tag:
+ self.report("client arriving for the " \
+ "wrong barrier")
+ client.settimeout(5)
+ client.send("!tag")
+ client.close()
+ return
+ elif name in self.waiting:
+ self.report("duplicate client")
+ client.settimeout(5)
+ client.send("!dup")
+ client.close()
+ return
- except socket.timeout:
- # This is nominally an error, but as we do not know
- # who that was we cannot do anything sane other
- # than report it and let the normal timeout kill
- # us when thats appropriate.
- self.report("client handshake timeout: (%s:%d)" %\
- (addr[0], addr[1]))
- client.close()
- return
+ # Acknowledge the client
+ client.send("wait")
- self.report("client now waiting: %s (%s:%d)" % \
- (name, addr[0], addr[1]))
+ except socket.timeout:
+ # This is nominally an error, but as we do not know
+ # who that was we cannot do anything sane other
+ # than report it and let the normal timeout kill
+ # us when thats appropriate.
+ self.report("client handshake timeout: (%s:%d)" %\
+ (addr[0], addr[1]))
+ client.close()
+ return
- # They seem to be valid record them.
- self.waiting[name] = connection
- self.seen += 1
+ self.report("client now waiting: %s (%s:%d)" % \
+ (name, addr[0], addr[1]))
+ # They seem to be valid record them.
+ self.waiting[name] = connection
+ self.seen += 1
- def slave_hello(self, connection):
- (client, addr) = connection
- name = None
- client.settimeout(5)
- try:
- client.send(self.tag + " " + self.hostid)
+ def slave_hello(self, connection):
+ (client, addr) = connection
+ name = None
- reply = client.recv(4)
- reply = reply.strip("\r\n")
- self.report("master said: " + reply)
+ client.settimeout(5)
+ try:
+ client.send(self.tag + " " + self.hostid)
- # Confirm the master accepted the connection.
- if reply != "wait":
- self.report("Bad connection request to master")
- client.close()
- return
+ reply = client.recv(4)
+ reply = reply.strip("\r\n")
+ self.report("master said: " + reply)
- except socket.timeout:
- # This is nominally an error, but as we do not know
- # who that was we cannot do anything sane other
- # than report it and let the normal timeout kill
- # us when thats appropriate.
- self.report("master handshake timeout: (%s:%d)" %\
- (addr[0], addr[1]))
- client.close()
- return
+ # Confirm the master accepted the connection.
+ if reply != "wait":
+ self.report("Bad connection request to master")
+ client.close()
+ return
- self.report("slave now waiting: (%s:%d)" % \
- (addr[0], addr[1]))
+ except socket.timeout:
+ # This is nominally an error, but as we do not know
+ # who that was we cannot do anything sane other
+ # than report it and let the normal timeout kill
+ # us when thats appropriate.
+ self.report("master handshake timeout: (%s:%d)" %\
+ (addr[0], addr[1]))
+ client.close()
+ return
- # They seem to be valid record them.
- self.waiting[self.hostid] = connection
- self.seen = 1
+ self.report("slave now waiting: (%s:%d)" % \
+ (addr[0], addr[1]))
+ # They seem to be valid record them.
+ self.waiting[self.hostid] = connection
+ self.seen = 1
- def master_release(self):
- # Check everyone is still there, that they have not
- # crashed or disconnected in the meantime.
- allpresent = 1
- for name in self.waiting:
- (client, addr) = self.waiting[name]
- self.report("checking client present: " + name)
+ def master_release(self):
+ # Check everyone is still there, that they have not
+ # crashed or disconnected in the meantime.
+ allpresent = 1
+ for name in self.waiting:
+ (client, addr) = self.waiting[name]
- client.settimeout(5)
- reply = 'none'
- try:
- client.send("ping")
- reply = client.recv(1024)
- except socket.timeout:
- self.report("ping/pong timeout: " + name)
- pass
+ self.report("checking client present: " + name)
+
+ client.settimeout(5)
+ reply = 'none'
+ try:
+ client.send("ping")
+ reply = client.recv(1024)
+ except socket.timeout:
+ self.report("ping/pong timeout: " + name)
+ pass
+
+ if reply != "pong":
+ allpresent = 0
+
+ if not allpresent:
+ raise BarrierError("master lost client")
- if reply != "pong":
- allpresent = 0
+ # If every ones checks in then commit the release.
+ for name in self.waiting:
+ (client, addr) = self.waiting[name]
- if not allpresent:
- raise BarrierError("master lost client")
-
- # If every ones checks in then commit the release.
- for name in self.waiting:
- (client, addr) = self.waiting[name]
+ self.report("releasing client: " + name)
- self.report("releasing client: " + name)
+ client.settimeout(5)
+ try:
+ client.send("rlse")
+ except socket.timeout:
+ self.report("release timeout: " + name)
+ pass
- client.settimeout(5)
- try:
- client.send("rlse")
- except socket.timeout:
- self.report("release timeout: " + name)
- pass
+ def waiting_close(self):
+ # Either way, close out all the clients. If we have
+ # not released them then they know to abort.
+ for name in self.waiting:
+ (client, addr) = self.waiting[name]
- def waiting_close(self):
- # Either way, close out all the clients. If we have
- # not released them then they know to abort.
- for name in self.waiting:
- (client, addr) = self.waiting[name]
+ self.report("closing client: " + name)
- self.report("closing client: " + name)
-
- try:
- client.close()
- except:
- pass
+ try:
+ client.close()
+ except:
+ pass
- def run_server(self, is_master):
- self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- self.server.setsockopt(socket.SOL_SOCKET,
- socket.SO_REUSEADDR, 1)
- self.server.bind(('', self.port))
- self.server.listen(10)
+ def run_server(self, is_master):
+ self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ self.server.setsockopt(socket.SOL_SOCKET,
+ socket.SO_REUSEADDR, 1)
+ self.server.bind(('', self.port))
+ self.server.listen(10)
- failed = 0
- try:
- while 1:
- try:
- # Wait for callers welcoming each.
- self.server.settimeout(self.remaining())
- connection = self.server.accept()
- if is_master:
- self.master_welcome(connection)
- else:
- self.slave_hello(connection)
- except socket.timeout:
- self.report("timeout waiting for " +
- "remaining clients")
- pass
+ failed = 0
+ try:
+ while 1:
+ try:
+ # Wait for callers welcoming each.
+ self.server.settimeout(self.remaining())
+ connection = self.server.accept()
+ if is_master:
+ self.master_welcome(connection)
+ else:
+ self.slave_hello(connection)
+ except socket.timeout:
+ self.report("timeout waiting for " +
+ "remaining clients")
+ pass
- if is_master:
- # Check if everyone is here.
- self.report("master seen %d of %d" % \
- (self.seen, len(self.members)))
- if self.seen == len(self.members):
- self.master_release()
- break
- else:
- # Check if master connected.
- if self.seen:
- self.report("slave connected " +
- "to master")
- self.slave_wait()
- break
+ if is_master:
+ # Check if everyone is here.
+ self.report("master seen %d of %d" % \
+ (self.seen, len(self.members)))
+ if self.seen == len(self.members):
+ self.master_release()
+ break
+ else:
+ # Check if master connected.
+ if self.seen:
+ self.report("slave connected " +
+ "to master")
+ self.slave_wait()
+ break
- self.waiting_close()
- self.server.close()
- except:
- self.waiting_close()
- self.server.close()
- raise
+ self.waiting_close()
+ self.server.close()
+ except:
+ self.waiting_close()
+ self.server.close()
+ raise
- def run_client(self, is_master):
- while self.remaining() > 0:
- try:
- remote = socket.socket(socket.AF_INET,
- socket.SOCK_STREAM)
- remote.settimeout(30)
- if is_master:
- # Connect to all slaves.
- host = self.get_host_from_id(
- self.members[self.seen])
- self.report("calling slave: %s" % host)
- connection = (remote, (host, self.port))
- remote.connect(connection[1])
- self.master_welcome(connection)
- else:
- # Just connect to the master.
- host = self.get_host_from_id(
- self.masterid)
- self.report("calling master")
- connection = (remote, (host, self.port))
- remote.connect(connection[1])
- self.slave_hello(connection)
- except socket.timeout:
- self.report("timeout calling host, retry")
- sleep(10)
- pass
- except socket.error, err:
- (code, str) = err
- if (code != errno.ECONNREFUSED):
- raise
- sleep(10)
+ def run_client(self, is_master):
+ while self.remaining() > 0:
+ try:
+ remote = socket.socket(socket.AF_INET,
+ socket.SOCK_STREAM)
+ remote.settimeout(30)
+ if is_master:
+ # Connect to all slaves.
+ host = self.get_host_from_id(
+ self.members[self.seen])
+ self.report("calling slave: %s" % host)
+ connection = (remote, (host, self.port))
+ remote.connect(connection[1])
+ self.master_welcome(connection)
+ else:
+ # Just connect to the master.
+ host = self.get_host_from_id(
+ self.masterid)
+ self.report("calling master")
+ connection = (remote, (host, self.port))
+ remote.connect(connection[1])
+ self.slave_hello(connection)
+ except socket.timeout:
+ self.report("timeout calling host, retry")
+ sleep(10)
+ pass
+ except socket.error, err:
+ (code, str) = err
+ if (code != errno.ECONNREFUSED):
+ raise
+ sleep(10)
- if is_master:
- # Check if everyone is here.
- self.report("master seen %d of %d" % \
- (self.seen, len(self.members)))
- if self.seen == len(self.members):
- self.master_release()
- break
- else:
- # Check if master connected.
- if self.seen:
- self.report("slave connected " +
- "to master")
- self.slave_wait()
- break
+ if is_master:
+ # Check if everyone is here.
+ self.report("master seen %d of %d" % \
+ (self.seen, len(self.members)))
+ if self.seen == len(self.members):
+ self.master_release()
+ break
+ else:
+ # Check if master connected.
+ if self.seen:
+ self.report("slave connected " +
+ "to master")
+ self.slave_wait()
+ break
- self.waiting_close()
+ self.waiting_close()
- def slave_wait(self):
- remote = self.waiting[self.hostid][0]
- mode = "wait"
- while 1:
- # All control messages are the same size to allow
- # us to split individual messages easily.
- remote.settimeout(self.remaining())
- reply = remote.recv(4)
- if not reply:
- break
+ def slave_wait(self):
+ remote = self.waiting[self.hostid][0]
+ mode = "wait"
+ while 1:
+ # All control messages are the same size to allow
+ # us to split individual messages easily.
+ remote.settimeout(self.remaining())
+ reply = remote.recv(4)
+ if not reply:
+ break
- reply = reply.strip("\r\n")
- self.report("master said: " + reply)
+ reply = reply.strip("\r\n")
+ self.report("master said: " + reply)
- mode = reply
- if reply == "ping":
- # Ensure we have sufficient time for the
- # ping/pong/rlse cyle to complete normally.
- self.update_timeout(10 + 10 * len(self.members))
+ mode = reply
+ if reply == "ping":
+ # Ensure we have sufficient time for the
+ # ping/pong/rlse cyle to complete normally.
+ self.update_timeout(10 + 10 * len(self.members))
- self.report("pong")
- remote.settimeout(self.remaining())
- remote.send("pong")
+ self.report("pong")
+ remote.settimeout(self.remaining())
+ remote.send("pong")
- elif reply == "rlse":
- # Ensure we have sufficient time for the
- # ping/pong/rlse cyle to complete normally.
- self.update_timeout(10 + 10 * len(self.members))
+ elif reply == "rlse":
+ # Ensure we have sufficient time for the
+ # ping/pong/rlse cyle to complete normally.
+ self.update_timeout(10 + 10 * len(self.members))
- self.report("was released, waiting for close")
+ self.report("was released, waiting for close")
- if mode == "rlse":
- pass
- elif mode == "wait":
- raise BarrierError("master abort -- barrier timeout")
- elif mode == "ping":
- raise BarrierError("master abort -- client lost")
- elif mode == "!tag":
- raise BarrierError("master abort -- incorrect tag")
- elif mode == "!dup":
- raise BarrierError("master abort -- duplicate client")
- else:
- raise BarrierError("master handshake failure: " + mode)
+ if mode == "rlse":
+ pass
+ elif mode == "wait":
+ raise BarrierError("master abort -- barrier timeout")
+ elif mode == "ping":
+ raise BarrierError("master abort -- client lost")
+ elif mode == "!tag":
+ raise BarrierError("master abort -- incorrect tag")
+ elif mode == "!dup":
+ raise BarrierError("master abort -- duplicate client")
+ else:
+ raise BarrierError("master handshake failure: " + mode)
- def rendevous(self, *hosts):
- self.start = time()
- self.members = list(hosts)
- self.members.sort()
- self.masterid = self.members.pop(0)
+ def rendevous(self, *hosts):
+ self.start = time()
+ self.members = list(hosts)
+ self.members.sort()
+ self.masterid = self.members.pop(0)
- self.report("masterid: %s" % self.masterid)
- if not len(self.members):
- self.report("No other members listed.")
- return
- self.report("members: %s" % ",".join(self.members))
+ self.report("masterid: %s" % self.masterid)
+ if not len(self.members):
+ self.report("No other members listed.")
+ return
+ self.report("members: %s" % ",".join(self.members))
- self.seen = 0
- self.waiting = {}
+ self.seen = 0
+ self.waiting = {}
- # Figure out who is the master in this barrier.
- if self.hostid == self.masterid:
- self.report("selected as master")
- self.run_server(is_master=True)
- else:
- self.report("selected as slave")
- self.run_client(is_master=False)
+ # Figure out who is the master in this barrier.
+ if self.hostid == self.masterid:
+ self.report("selected as master")
+ self.run_server(is_master=True)
+ else:
+ self.report("selected as slave")
+ self.run_client(is_master=False)
- def rendevous_servers(self, masterid, *hosts):
- self.start = time()
- self.members = list(hosts)
- self.members.sort()
- self.masterid = masterid
+ def rendevous_servers(self, masterid, *hosts):
+ self.start = time()
+ self.members = list(hosts)
+ self.members.sort()
+ self.masterid = masterid
- self.report("masterid: %s" % self.masterid)
- if not len(self.members):
- self.report("No other members listed.")
- return
- self.report("members: %s" % ",".join(self.members))
+ self.report("masterid: %s" % self.masterid)
+ if not len(self.members):
+ self.report("No other members listed.")
+ return
+ self.report("members: %s" % ",".join(self.members))
- self.seen = 0
- self.waiting = {}
+ self.seen = 0
+ self.waiting = {}
- # Figure out who is the master in this barrier.
- if self.hostid == self.masterid:
- self.report("selected as master")
- self.run_client(is_master=True)
- else:
- self.report("selected as slave")
- self.run_server(is_master=False)
+ # Figure out who is the master in this barrier.
+ if self.hostid == self.masterid:
+ self.report("selected as master")
+ self.run_client(is_master=True)
+ else:
+ self.report("selected as slave")
+ self.run_server(is_master=False)
#
# TESTING -- direct test harness.
@@ -472,28 +472,28 @@
# python bin/barrier.py 3 meeting
#
if __name__ == "__main__":
- barrier = barrier('127.0.0.1#' + sys.argv[1], sys.argv[2], 60)
+ barrier = barrier('127.0.0.1#' + sys.argv[1], sys.argv[2], 60)
- try:
- all = [ '127.0.0.1#2', '127.0.0.1#1', '127.0.0.1#3' ]
- barrier.rendevous(*all)
- except BarrierError, err:
- print "barrier: 127.0.0.1#" + sys.argv[1] + \
- ": barrier failed:", err
- sys.exit(1)
- else:
- print "barrier: 127.0.0.1#" + sys.argv[1] + \
- ": all present and accounted for"
+ try:
+ all = [ '127.0.0.1#2', '127.0.0.1#1', '127.0.0.1#3' ]
+ barrier.rendevous(*all)
+ except BarrierError, err:
+ print "barrier: 127.0.0.1#" + sys.argv[1] + \
+ ": barrier failed:", err
+ sys.exit(1)
+ else:
+ print "barrier: 127.0.0.1#" + sys.argv[1] + \
+ ": all present and accounted for"
- try:
- all = [ '127.0.0.1#2', '127.0.0.1#1' ]
- if 1 <= int(sys.argv[1]) <= 2:
- barrier.rendevous_servers(*all)
- except BarrierError, err:
- print "barrier: 127.0.0.1#" + sys.argv[1] + \
- ": barrier failed:", err
- sys.exit(1)
- else:
- print "barrier: 127.0.0.1#" + sys.argv[1] + \
- ": all present and accounted for"
- sys.exit(0)
+ try:
+ all = [ '127.0.0.1#2', '127.0.0.1#1' ]
+ if 1 <= int(sys.argv[1]) <= 2:
+ barrier.rendevous_servers(*all)
+ except BarrierError, err:
+ print "barrier: 127.0.0.1#" + sys.argv[1] + \
+ ": barrier failed:", err
+ sys.exit(1)
+ else:
+ print "barrier: 127.0.0.1#" + sys.argv[1] + \
+ ": all present and accounted for"
+ sys.exit(0)
diff --git a/client/common_lib/check_version.py b/client/common_lib/check_version.py
index 981a21c..6971049 100755
--- a/client/common_lib/check_version.py
+++ b/client/common_lib/check_version.py
@@ -2,41 +2,41 @@
def extract_version(path):
- match = re.search(r'/python(\d+)\.(\d+)$', path)
- if match:
- return (int(match.group(1)), int(match.group(2)))
- else:
- return None
+ match = re.search(r'/python(\d+)\.(\d+)$', path)
+ if match:
+ return (int(match.group(1)), int(match.group(2)))
+ else:
+ return None
def find_newest_python():
- pythons = []
- pythons.extend(glob.glob('/usr/bin/python*'))
- pythons.extend(glob.glob('/usr/local/bin/python*'))
+ pythons = []
+ pythons.extend(glob.glob('/usr/bin/python*'))
+ pythons.extend(glob.glob('/usr/local/bin/python*'))
- best_python = (0, 0), ''
- for python in pythons:
- version = extract_version(python)
- if version > best_python[0] and version >= (2, 4):
- best_python = version, python
+ best_python = (0, 0), ''
+ for python in pythons:
+ version = extract_version(python)
+ if version > best_python[0] and version >= (2, 4):
+ best_python = version, python
- if best_python[0] == (0, 0):
- raise ValueError('Python 2.4 or newer is needed')
- return best_python[1]
-
+ if best_python[0] == (0, 0):
+ raise ValueError('Python 2.4 or newer is needed')
+ return best_python[1]
+
def restart():
- python = find_newest_python()
- sys.argv.insert(0, '-u')
- sys.argv.insert(0, python)
- os.execv(sys.argv[0], sys.argv)
+ python = find_newest_python()
+ sys.argv.insert(0, '-u')
+ sys.argv.insert(0, python)
+ os.execv(sys.argv[0], sys.argv)
def check_python_version():
- version = None
- try:
- version = sys.version_info[0:2]
- except AttributeError:
- pass # pre 2.0, no neat way to get the exact number
- if not version or version < (2, 4):
- restart()
+ version = None
+ try:
+ version = sys.version_info[0:2]
+ except AttributeError:
+ pass # pre 2.0, no neat way to get the exact number
+ if not version or version < (2, 4):
+ restart()
diff --git a/client/common_lib/error.py b/client/common_lib/error.py
index 6eeb3fe..1fe3370 100644
--- a/client/common_lib/error.py
+++ b/client/common_lib/error.py
@@ -6,137 +6,137 @@
from traceback import format_exception
def format_error():
- t, o, tb = sys.exc_info()
- trace = format_exception(t, o, tb)
- # Clear the backtrace to prevent a circular reference
- # in the heap -- as per tutorial
- tb = ''
+ t, o, tb = sys.exc_info()
+ trace = format_exception(t, o, tb)
+ # Clear the backtrace to prevent a circular reference
+ # in the heap -- as per tutorial
+ tb = ''
- return ''.join(trace)
+ return ''.join(trace)
class JobContinue(SystemExit):
- """Allow us to bail out requesting continuance."""
- pass
+ """Allow us to bail out requesting continuance."""
+ pass
class JobComplete(SystemExit):
- """Allow us to bail out indicating continuation not required."""
- pass
+ """Allow us to bail out indicating continuation not required."""
+ pass
class AutotestError(Exception):
- """The parent of all errors deliberatly thrown within the client code."""
- pass
+ """The parent of all errors deliberatly thrown within the client code."""
+ pass
class JobError(AutotestError):
- """Indicates an error which terminates and fails the whole job."""
- pass
+ """Indicates an error which terminates and fails the whole job."""
+ pass
class TestError(AutotestError):
- """Indicates an error which terminates and fails the test."""
- pass
+ """Indicates an error which terminates and fails the test."""
+ pass
class TestNAError(AutotestError):
- """Indictates that the test is Not Applicable. Should be thrown
- when various conditions are such that the test is inappropriate"""
- pass
+ """Indictates that the test is Not Applicable. Should be thrown
+ when various conditions are such that the test is inappropriate"""
+ pass
class CmdError(TestError):
- """\
- Indicates that a command failed, is fatal to the test unless caught.
- """
- def __init__(self, command, result_obj, additional_text=None):
- TestError.__init__(self, command, result_obj, additional_text)
+ """\
+ Indicates that a command failed, is fatal to the test unless caught.
+ """
+ def __init__(self, command, result_obj, additional_text=None):
+ TestError.__init__(self, command, result_obj, additional_text)
- def __str__(self):
- msg = "Command <%s> failed, rc=%d" % (self.args[0],
- self.args[1].exit_status)
- if self.args[2]:
- msg += ", " + self.args[2]
- return msg
+ def __str__(self):
+ msg = "Command <%s> failed, rc=%d" % (self.args[0],
+ self.args[1].exit_status)
+ if self.args[2]:
+ msg += ", " + self.args[2]
+ return msg
class PackageError(TestError):
- """Indicates an error trying to perform a package operation."""
- pass
+ """Indicates an error trying to perform a package operation."""
+ pass
class UnhandledError(TestError):
- """Indicates an unhandled exception in a test."""
- def __init__(self, prefix):
- msg = prefix + format_error()
- TestError.__init__(self, msg)
+ """Indicates an unhandled exception in a test."""
+ def __init__(self, prefix):
+ msg = prefix + format_error()
+ TestError.__init__(self, msg)
class InstallError(JobError):
- """Indicates an installation error which Terminates and fails the job."""
- pass
+ """Indicates an installation error which Terminates and fails the job."""
+ pass
class AutotestRunError(AutotestError):
- pass
+ pass
class AutotestTimeoutError(AutotestError):
- """This exception is raised when an autotest test exceeds the timeout
- parameter passed to run_timed_test and is killed.
- """
+ """This exception is raised when an autotest test exceeds the timeout
+ parameter passed to run_timed_test and is killed.
+ """
# server-specific errors
class AutoservError(Exception):
- pass
+ pass
class AutoservSSHTimeout(AutoservError):
- """SSH experienced a connection timeout"""
- pass
+ """SSH experienced a connection timeout"""
+ pass
class AutoservRunError(AutoservError):
- """\
- Errors raised by one of the run functions. Should always be
- constructed with a tuple of two args (error description (str),
- run result object).
- """
- def __init__(self, description, result_obj):
- AutoservError.__init__(self, description, result_obj)
+ """\
+ Errors raised by one of the run functions. Should always be
+ constructed with a tuple of two args (error description (str),
+ run result object).
+ """
+ def __init__(self, description, result_obj):
+ AutoservError.__init__(self, description, result_obj)
class AutoservVirtError(AutoservError):
- """Vitualization related error"""
- pass
+ """Vitualization related error"""
+ pass
class AutoservUnsupportedError(AutoservError):
- """Error raised when you try to use an unsupported optional feature"""
- pass
+ """Error raised when you try to use an unsupported optional feature"""
+ pass
class AutoservHostError(AutoservError):
- """Error reaching a host"""
- pass
+ """Error reaching a host"""
+ pass
class AutoservRebootError(AutoservError):
- """Error occured while rebooting a machine"""
- pass
+ """Error occured while rebooting a machine"""
+ pass
class AutoservSubcommandError(AutoservError):
- """Indicates an error while executing a (forked) subcommand"""
- def __init__(self, func, exit_code):
- AutoservError.__init__(self, func, exit_code)
- self.func = func
- self.exit_code = exit_code
+ """Indicates an error while executing a (forked) subcommand"""
+ def __init__(self, func, exit_code):
+ AutoservError.__init__(self, func, exit_code)
+ self.func = func
+ self.exit_code = exit_code
- def __str__(self):
- return ("Subcommand %s failed with exit code %d" %
- (self.func, self.exit_code))
+ def __str__(self):
+ return ("Subcommand %s failed with exit code %d" %
+ (self.func, self.exit_code))
diff --git a/client/common_lib/global_config.py b/client/common_lib/global_config.py
index 1036e3d..8db02e1 100644
--- a/client/common_lib/global_config.py
+++ b/client/common_lib/global_config.py
@@ -11,115 +11,115 @@
import error
dirname = os.path.dirname(sys.modules[__name__].__file__)
-DEFAULT_CONFIG_FILE = os.path.abspath(os.path.join(dirname,
- "../../global_config.ini"))
-DEFAULT_SHADOW_FILE = os.path.abspath(os.path.join(dirname,
- "../../shadow_config.ini"))
-
+DEFAULT_CONFIG_FILE = os.path.abspath(os.path.join(dirname,
+ "../../global_config.ini"))
+DEFAULT_SHADOW_FILE = os.path.abspath(os.path.join(dirname,
+ "../../shadow_config.ini"))
+
class ConfigError(error.AutotestError):
- pass
+ pass
class ConfigValueError(ConfigError):
- pass
+ pass
class global_config(object):
- config = None
- config_file = DEFAULT_CONFIG_FILE
- shadow_file = DEFAULT_SHADOW_FILE
-
-
- def set_config_files(self, config_file=DEFAULT_CONFIG_FILE,
- shadow_file=DEFAULT_SHADOW_FILE):
- self.config_file = config_file
- self.shadow_file = shadow_file
- self.config = None
+ config = None
+ config_file = DEFAULT_CONFIG_FILE
+ shadow_file = DEFAULT_SHADOW_FILE
- def get_config_value(self, section, key, type=str, default=None):
- if self.config == None:
- self.parse_config_file()
-
- try:
- val = self.config.get(section, key)
- except:
- if default == None:
- msg = ("Value '%s' not found in section '%s'" %
- (key, section))
- raise ConfigError(msg)
- else:
- return default
-
- return self.convert_value(key, section, val, type, default)
+ def set_config_files(self, config_file=DEFAULT_CONFIG_FILE,
+ shadow_file=DEFAULT_SHADOW_FILE):
+ self.config_file = config_file
+ self.shadow_file = shadow_file
+ self.config = None
- def merge_configs(self, shadow_config):
- # overwrite whats in config with whats in shadow_config
- sections = shadow_config.sections()
- for section in sections:
- # add the section if need be
- if not self.config.has_section(section):
- self.config.add_section(section)
- # now run through all options and set them
- options = shadow_config.options(section)
- for option in options:
- val = shadow_config.get(section, option)
- self.config.set(section, option, val)
+ def get_config_value(self, section, key, type=str, default=None):
+ if self.config == None:
+ self.parse_config_file()
+
+ try:
+ val = self.config.get(section, key)
+ except:
+ if default == None:
+ msg = ("Value '%s' not found in section '%s'" %
+ (key, section))
+ raise ConfigError(msg)
+ else:
+ return default
+
+ return self.convert_value(key, section, val, type, default)
- def parse_config_file(self):
- if not os.path.exists(self.config_file):
- raise ConfigError('%s not found' % (self.config_file))
- self.config = ConfigParser.ConfigParser()
- self.config.read(self.config_file)
+ def merge_configs(self, shadow_config):
+ # overwrite whats in config with whats in shadow_config
+ sections = shadow_config.sections()
+ for section in sections:
+ # add the section if need be
+ if not self.config.has_section(section):
+ self.config.add_section(section)
+ # now run through all options and set them
+ options = shadow_config.options(section)
+ for option in options:
+ val = shadow_config.get(section, option)
+ self.config.set(section, option, val)
- # now also read the shadow file if there is one
- # this will overwrite anything that is found in the
- # other config
- if os.path.exists(self.shadow_file):
- shadow_config = ConfigParser.ConfigParser()
- shadow_config.read(self.shadow_file)
- # now we merge shadow into global
- self.merge_configs(shadow_config)
-
-
- # the values that are pulled from ini
- # are strings. But we should attempt to
- # convert them to other types if needed.
- def convert_value(self, key, section, value, type, default):
- # strip off leading and trailing white space
- sval = value.strip()
-
- # if length of string is zero then return None
- if len(sval) == 0:
- if type == str:
- return ""
- elif type == bool:
- return False
- elif type == int:
- return 0
- elif type == float:
- return 0.0
- else:
- return None
-
- if type == bool:
- if sval.lower() == "false":
- return False
- else:
- return True
-
- try:
- conv_val = type(sval)
- return conv_val
- except:
- msg = ("Could not covert %s in section %s" %
- (key, section))
- raise ConfigValueError(msg)
-
-# insure the class is a singleton. Now the symbol global_config
+ def parse_config_file(self):
+ if not os.path.exists(self.config_file):
+ raise ConfigError('%s not found' % (self.config_file))
+ self.config = ConfigParser.ConfigParser()
+ self.config.read(self.config_file)
+
+ # now also read the shadow file if there is one
+ # this will overwrite anything that is found in the
+ # other config
+ if os.path.exists(self.shadow_file):
+ shadow_config = ConfigParser.ConfigParser()
+ shadow_config.read(self.shadow_file)
+ # now we merge shadow into global
+ self.merge_configs(shadow_config)
+
+
+ # the values that are pulled from ini
+ # are strings. But we should attempt to
+ # convert them to other types if needed.
+ def convert_value(self, key, section, value, type, default):
+ # strip off leading and trailing white space
+ sval = value.strip()
+
+ # if length of string is zero then return None
+ if len(sval) == 0:
+ if type == str:
+ return ""
+ elif type == bool:
+ return False
+ elif type == int:
+ return 0
+ elif type == float:
+ return 0.0
+ else:
+ return None
+
+ if type == bool:
+ if sval.lower() == "false":
+ return False
+ else:
+ return True
+
+ try:
+ conv_val = type(sval)
+ return conv_val
+ except:
+ msg = ("Could not covert %s in section %s" %
+ (key, section))
+ raise ConfigValueError(msg)
+
+
+# insure the class is a singleton. Now the symbol global_config
# will point to the one and only one instace of the class
global_config = global_config()
diff --git a/client/common_lib/global_config_unittest.py b/client/common_lib/global_config_unittest.py
index e41d709..dab6825 100644
--- a/client/common_lib/global_config_unittest.py
+++ b/client/common_lib/global_config_unittest.py
@@ -30,90 +30,90 @@
def create_config_files():
- (fp, global_file) = tempfile.mkstemp(".ini", text=True)
- os.write(fp, global_config_ini_contents)
- os.close(fp)
-
- (fp, shadow_file) = tempfile.mkstemp(".ini", text=True)
- os.write(fp, shadow_config_ini_contents)
- os.close(fp)
-
- return (global_file, shadow_file)
+ (fp, global_file) = tempfile.mkstemp(".ini", text=True)
+ os.write(fp, global_config_ini_contents)
+ os.close(fp)
+
+ (fp, shadow_file) = tempfile.mkstemp(".ini", text=True)
+ os.write(fp, shadow_config_ini_contents)
+ os.close(fp)
+
+ return (global_file, shadow_file)
class global_config_test(unittest.TestCase):
- # grab the singelton
- conf = global_config.global_config
-
- def setUp(self):
- # set the config files to our test files
- (self.global_file, self.shadow_file) = create_config_files()
- self.conf.set_config_files(self.global_file, self.shadow_file)
+ # grab the singelton
+ conf = global_config.global_config
-
- def tearDown(self):
- os.remove(self.global_file)
- os.remove(self.shadow_file)
- self.conf.set_config_files(global_config.DEFAULT_CONFIG_FILE,
- global_config.DEFAULT_SHADOW_FILE)
+ def setUp(self):
+ # set the config files to our test files
+ (self.global_file, self.shadow_file) = create_config_files()
+ self.conf.set_config_files(self.global_file, self.shadow_file)
- def testFloat(self):
- val = self.conf.get_config_value("SECTION_A", "value_1", float)
- self.assertEquals(type(val), types.FloatType)
- self.assertEquals(val, 6.0)
-
-
- def testInt(self):
- val = self.conf.get_config_value("SECTION_B", "value_1", int)
- self.assertEquals(type(val), types.IntType)
- self.assertTrue(val < 0)
- val = self.conf.get_config_value("SECTION_B", "value_3", int)
- self.assertEquals(val, 0)
- val = self.conf.get_config_value("SECTION_B", "value_4", int)
- self.assertTrue(val > 0)
+ def tearDown(self):
+ os.remove(self.global_file)
+ os.remove(self.shadow_file)
+ self.conf.set_config_files(global_config.DEFAULT_CONFIG_FILE,
+ global_config.DEFAULT_SHADOW_FILE)
- def testString(self):
- val = self.conf.get_config_value("SECTION_A", "value_2")
- self.assertEquals(type(val),types.StringType)
- self.assertEquals(val, "hello")
-
-
- def testOverride(self):
- val = self.conf.get_config_value("SECTION_C", "value_1")
- self.assertEquals(val, "somebody@remotehost")
+ def testFloat(self):
+ val = self.conf.get_config_value("SECTION_A", "value_1", float)
+ self.assertEquals(type(val), types.FloatType)
+ self.assertEquals(val, 6.0)
- def testException(self):
- error = 0
- try:
- val = self.conf.get_config_value("SECTION_B",
- "value_2", int)
- except:
- error = 1
- self.assertEquals(error, 1)
+ def testInt(self):
+ val = self.conf.get_config_value("SECTION_B", "value_1", int)
+ self.assertEquals(type(val), types.IntType)
+ self.assertTrue(val < 0)
+ val = self.conf.get_config_value("SECTION_B", "value_3", int)
+ self.assertEquals(val, 0)
+ val = self.conf.get_config_value("SECTION_B", "value_4", int)
+ self.assertTrue(val > 0)
- def testBoolean(self):
- val = self.conf.get_config_value("SECTION_A", "value_3", bool)
- self.assertEquals(val, True)
- val = self.conf.get_config_value("SECTION_A", "value_4", bool)
- self.assertEquals(val, False)
- val = self.conf.get_config_value("SECTION_A", "value_5", bool)
- self.assertEquals(val, True)
- val = self.conf.get_config_value("SECTION_A", "value_6", bool)
- self.assertEquals(val, False)
+ def testString(self):
+ val = self.conf.get_config_value("SECTION_A", "value_2")
+ self.assertEquals(type(val),types.StringType)
+ self.assertEquals(val, "hello")
- def testDefaults(self):
- val = self.conf.get_config_value("MISSING", "foo", float, 3.6)
- self.assertEquals(val, 3.6)
- val = self.conf.get_config_value("SECTION_A", "novalue", str,
- "default")
- self.assertEquals(val, "default")
+ def testOverride(self):
+ val = self.conf.get_config_value("SECTION_C", "value_1")
+ self.assertEquals(val, "somebody@remotehost")
+
+
+ def testException(self):
+ error = 0
+ try:
+ val = self.conf.get_config_value("SECTION_B",
+ "value_2", int)
+ except:
+ error = 1
+ self.assertEquals(error, 1)
+
+
+ def testBoolean(self):
+ val = self.conf.get_config_value("SECTION_A", "value_3", bool)
+ self.assertEquals(val, True)
+ val = self.conf.get_config_value("SECTION_A", "value_4", bool)
+ self.assertEquals(val, False)
+ val = self.conf.get_config_value("SECTION_A", "value_5", bool)
+ self.assertEquals(val, True)
+ val = self.conf.get_config_value("SECTION_A", "value_6", bool)
+ self.assertEquals(val, False)
+
+
+ def testDefaults(self):
+ val = self.conf.get_config_value("MISSING", "foo", float, 3.6)
+ self.assertEquals(val, 3.6)
+ val = self.conf.get_config_value("SECTION_A", "novalue", str,
+ "default")
+ self.assertEquals(val, "default")
# this is so the test can be run in standalone mode
if __name__ == '__main__':
- unittest.main()
+ unittest.main()
diff --git a/client/common_lib/logging.py b/client/common_lib/logging.py
index 004975a..64b8086 100644
--- a/client/common_lib/logging.py
+++ b/client/common_lib/logging.py
@@ -8,48 +8,48 @@
job_statuses = ["TEST_NA", "ABORT", "ERROR", "FAIL", "WARN", "GOOD", "ALERT",
- "NOSTATUS"]
+ "NOSTATUS"]
def is_valid_status(status):
- if not re.match(r'(START|(END )?(GOOD|WARN|FAIL|ABORT|TEST_NA))$',
- status):
- return False
- else:
- return True
+ if not re.match(r'(START|(END )?(GOOD|WARN|FAIL|ABORT|TEST_NA))$',
+ status):
+ return False
+ else:
+ return True
def record(fn):
- """
- Generic method decorator for logging calls under the
- assumption that return=GOOD, exception=FAIL. The method
- determines parameters as:
- subdir = self.subdir if it exists, or None
- operation = "class name"."method name"
- status = None on GOOD, str(exception) on FAIL
- The object using this method must have a job attribute
- for the logging to actually occur, otherwise the logging
- will silently fail.
+ """
+ Generic method decorator for logging calls under the
+ assumption that return=GOOD, exception=FAIL. The method
+ determines parameters as:
+ subdir = self.subdir if it exists, or None
+ operation = "class name"."method name"
+ status = None on GOOD, str(exception) on FAIL
+ The object using this method must have a job attribute
+ for the logging to actually occur, otherwise the logging
+ will silently fail.
- Logging can explicitly be disabled for a call by passing
- a logged=False parameter
- """
- def recorded_func(self, *args, **dargs):
- logged = dargs.pop('logged', True)
- job = getattr(self, 'job', None)
- # if logging is disabled/unavailable, just
- # call the method
- if not logged or job is None:
- return fn(self, *args, **dargs)
- # logging is available, so wrap the method call
- # in success/failure logging
- subdir = getattr(self, 'subdir', None)
- operation = '%s.%s' % (self.__class__.__name__,
- fn.__name__)
- try:
- result = fn(self, *args, **dargs)
- job.record('GOOD', subdir, operation)
- except Exception, detail:
- job.record('FAIL', subdir, operation, str(detail))
- raise
- return result
- return recorded_func
+ Logging can explicitly be disabled for a call by passing
+ a logged=False parameter
+ """
+ def recorded_func(self, *args, **dargs):
+ logged = dargs.pop('logged', True)
+ job = getattr(self, 'job', None)
+ # if logging is disabled/unavailable, just
+ # call the method
+ if not logged or job is None:
+ return fn(self, *args, **dargs)
+ # logging is available, so wrap the method call
+ # in success/failure logging
+ subdir = getattr(self, 'subdir', None)
+ operation = '%s.%s' % (self.__class__.__name__,
+ fn.__name__)
+ try:
+ result = fn(self, *args, **dargs)
+ job.record('GOOD', subdir, operation)
+ except Exception, detail:
+ job.record('FAIL', subdir, operation, str(detail))
+ raise
+ return result
+ return recorded_func
diff --git a/client/common_lib/mail.py b/client/common_lib/mail.py
index 697c209..0c641dd 100644
--- a/client/common_lib/mail.py
+++ b/client/common_lib/mail.py
@@ -1,39 +1,39 @@
-import os, email.Message, smtplib
+import os, email.Message, smtplib
def send(from_address, to_addresses, cc_addresses, subject, message_body):
- """
- Send out a plain old text email. It uses sendmail by default, but
- if that fails then it falls back to using smtplib.
+ """
+ Send out a plain old text email. It uses sendmail by default, but
+ if that fails then it falls back to using smtplib.
- Args:
- from_address: the email address to put in the "From:" field
- to_addresses: either a single string or an iterable of
- strings to put in the "To:" field of the email
- cc_addresses: either a single string of an iterable of
- strings to put in the "Cc:" field of the email
- subject: the email subject
- message_body: the body of the email. there's no special
- handling of encoding here, so it's safest to
- stick to 7-bit ASCII text
- """
- # addresses can be a tuple or a single string, so make them tuples
- if isinstance(to_addresses, str):
- to_addresses = [to_addresses]
- else:
- to_addresses = list(to_addresses)
- if isinstance(cc_addresses, str):
- cc_addresses = [cc_addresses]
- else:
- cc_addresses = list(cc_addresses)
+ Args:
+ from_address: the email address to put in the "From:" field
+ to_addresses: either a single string or an iterable of
+ strings to put in the "To:" field of the email
+ cc_addresses: either a single string of an iterable of
+ strings to put in the "Cc:" field of the email
+ subject: the email subject
+ message_body: the body of the email. there's no special
+ handling of encoding here, so it's safest to
+ stick to 7-bit ASCII text
+ """
+ # addresses can be a tuple or a single string, so make them tuples
+ if isinstance(to_addresses, str):
+ to_addresses = [to_addresses]
+ else:
+ to_addresses = list(to_addresses)
+ if isinstance(cc_addresses, str):
+ cc_addresses = [cc_addresses]
+ else:
+ cc_addresses = list(cc_addresses)
- message = email.Message.Message()
- message["To"] = ", ".join(to_addresses)
- message["Cc"] = ", ".join(cc_addresses)
- message["From"] = from_address
- message["Subject"] = subject
- message.set_payload(message_body)
+ message = email.Message.Message()
+ message["To"] = ", ".join(to_addresses)
+ message["Cc"] = ", ".join(cc_addresses)
+ message["From"] = from_address
+ message["Subject"] = subject
+ message.set_payload(message_body)
- server = smtplib.SMTP("localhost")
- server.sendmail(from_address, to_addresses + cc_addresses, message.as_string())
- server.quit()
+ server = smtplib.SMTP("localhost")
+ server.sendmail(from_address, to_addresses + cc_addresses, message.as_string())
+ server.quit()
diff --git a/client/common_lib/mail_unittest.py b/client/common_lib/mail_unittest.py
index 95de782..cac7ee6 100644
--- a/client/common_lib/mail_unittest.py
+++ b/client/common_lib/mail_unittest.py
@@ -4,66 +4,66 @@
import mail, email.Message
class test_data:
- mail_host = None
- mail_port = None
- mail_connect = False
- mail_from_address = None
- mail_to_address = None
- mail_message = None
+ mail_host = None
+ mail_port = None
+ mail_connect = False
+ mail_from_address = None
+ mail_to_address = None
+ mail_message = None
# we define our needed mock SMTP
class SMTP:
- def __init__(self, host=None, port=25):
- test_data.mail_host = host
- test_data.mail_port = port
-
- if test_data.mail_host:
- self.connect(test_data.mail_host, test_data.mail_port)
-
-
- def connect(self, host, port):
- test_data.mail_connect = True
+ def __init__(self, host=None, port=25):
+ test_data.mail_host = host
+ test_data.mail_port = port
+
+ if test_data.mail_host:
+ self.connect(test_data.mail_host, test_data.mail_port)
- def quit(self):
- test_data.mail_connect = False
+ def connect(self, host, port):
+ test_data.mail_connect = True
- def sendmail(self, from_address, to_address, message):
- test_data.mail_from_address = from_address
- test_data.mail_to_address = to_address
- test_data.mail_message = message
+ def quit(self):
+ test_data.mail_connect = False
+
+
+ def sendmail(self, from_address, to_address, message):
+ test_data.mail_from_address = from_address
+ test_data.mail_to_address = to_address
+ test_data.mail_message = message
class mail_test(unittest.TestCase):
- cached_SMTP = None
+ cached_SMTP = None
- def setUp(self):
- # now perform the slip
- self.cached_SMTP = mail.smtplib.SMTP
- mail.smtplib.SMTP = SMTP
-
-
- def tearDown(self):
- # now put things back
- mail.smtplib.SMTP = self.cached_SMTP
+ def setUp(self):
+ # now perform the slip
+ self.cached_SMTP = mail.smtplib.SMTP
+ mail.smtplib.SMTP = SMTP
- def test_send_message(self):
- message = email.Message.Message()
- message["To"] = "you"
- message["Cc"] = "them"
- message["From"] = "me"
- message["Subject"] = "hello"
- message.set_payload("Hello everybody!")
-
- mail.send("me", "you", "them", "hello", "Hello everybody!")
- self.assertEquals("me", test_data.mail_from_address)
- self.assertEquals(["you","them"], test_data.mail_to_address)
- self.assertEquals(message.as_string(), test_data.mail_message)
-
+ def tearDown(self):
+ # now put things back
+ mail.smtplib.SMTP = self.cached_SMTP
+
+
+ def test_send_message(self):
+ message = email.Message.Message()
+ message["To"] = "you"
+ message["Cc"] = "them"
+ message["From"] = "me"
+ message["Subject"] = "hello"
+ message.set_payload("Hello everybody!")
+
+ mail.send("me", "you", "them", "hello", "Hello everybody!")
+ self.assertEquals("me", test_data.mail_from_address)
+ self.assertEquals(["you","them"], test_data.mail_to_address)
+ self.assertEquals(message.as_string(), test_data.mail_message)
+
# this is so the test can be run in standalone mode
if __name__ == '__main__':
- unittest.main()
+ unittest.main()
diff --git a/client/common_lib/pexpect.py b/client/common_lib/pexpect.py
index 67c6389..260c1d8 100644
--- a/client/common_lib/pexpect.py
+++ b/client/common_lib/pexpect.py
@@ -336,12 +336,12 @@
the input from the child and output sent to the child. Sometimes you
don't want to see everything you write to the child. You only want to
log what the child sends back. For example::
-
+
child = pexpect.spawn('some_command')
child.logfile_read = sys.stdout
To separately log output sent to the child use logfile_send::
-
+
self.logfile_send = fout
The delaybeforesend helps overcome a weird behavior that many users
@@ -704,7 +704,7 @@
if timeout == -1:
timeout = self.timeout
if timeout is not None:
- end_time = time.time() + timeout
+ end_time = time.time() + timeout
while True:
if not self.getecho():
return True
@@ -1355,7 +1355,7 @@
if timeout == -1:
timeout = self.timeout
if timeout is not None:
- end_time = time.time() + timeout
+ end_time = time.time() + timeout
if searchwindowsize == -1:
searchwindowsize = self.searchwindowsize
@@ -1653,7 +1653,7 @@
# rescanning until we've read three more bytes.
#
# Sadly, I don't know enough about this interesting topic. /grahn
-
+
for index, s in self._strings:
if searchwindowsize is None:
# the match, if any, can only be in the fresh data,
@@ -1732,7 +1732,7 @@
'buffer' which have not been searched before.
See class spawn for the 'searchwindowsize' argument.
-
+
If there is a match this returns the index of that string, and sets
'start', 'end' and 'match'. Otherwise, returns -1."""
diff --git a/client/common_lib/pxssh.py b/client/common_lib/pxssh.py
index f8add8d..1e5a6a4 100644
--- a/client/common_lib/pxssh.py
+++ b/client/common_lib/pxssh.py
@@ -30,10 +30,10 @@
shells.
Example that runs a few commands on a remote server and prints the result::
-
+
import pxssh
import getpass
- try:
+ try:
s = pxssh.pxssh()
hostname = raw_input('hostname: ')
username = raw_input('username: ')
@@ -74,7 +74,7 @@
spawn.__init__(self, None, timeout=timeout, maxread=maxread, searchwindowsize=searchwindowsize, logfile=logfile, cwd=cwd, env=env)
self.name = '<pxssh>'
-
+
#SUBTLE HACK ALERT! Note that the command to set the prompt uses a
#slightly different string than the regular expression to match it. This
#is because when you set the prompt the command will echo back, but we
@@ -98,7 +98,7 @@
# Unsetting SSH_ASKPASS on the remote side doesn't disable it! Annoying!
#self.SSH_OPTS = "-x -o'RSAAuthentication=no' -o 'PubkeyAuthentication=no'"
self.force_password = False
- self.auto_prompt_reset = True
+ self.auto_prompt_reset = True
def levenshtein_distance(self, a,b):
@@ -173,7 +173,7 @@
to guess when we have reached the prompt. Then we hope for the best and
blindly try to reset the prompt to something more unique. If that fails
then login() raises an ExceptionPxssh exception.
-
+
In some situations it is not possible or desirable to reset the
original prompt. In this case, set 'auto_prompt_reset' to False to
inhibit setting the prompt to the UNIQUE_PROMPT. Remember that pxssh
@@ -194,7 +194,7 @@
i = self.expect(["(?i)are you sure you want to continue connecting", original_prompt, "(?i)(?:password)|(?:passphrase for key)", "(?i)permission denied", "(?i)terminal type", TIMEOUT, "(?i)connection closed by remote host"], timeout=login_timeout)
# First phase
- if i==0:
+ if i==0:
# New certificate -- always accept it.
# This is what you get if SSH does not have the remote host's
# public key stored in the 'known_hosts' cache.
@@ -212,14 +212,14 @@
# This is weird. This should not happen twice in a row.
self.close()
raise ExceptionPxssh ('Weird error. Got "are you sure" prompt twice.')
- elif i==1: # can occur if you have a public key pair set to authenticate.
+ elif i==1: # can occur if you have a public key pair set to authenticate.
### TODO: May NOT be OK if expect() got tricked and matched a false prompt.
pass
elif i==2: # password prompt again
# For incorrect passwords, some ssh servers will
# ask for the password again, others return 'denied' right away.
# If we get the password prompt again then this means
- # we didn't get the password right the first time.
+ # we didn't get the password right the first time.
self.close()
raise ExceptionPxssh ('password refused')
elif i==3: # permission denied -- password was bad.
@@ -239,7 +239,7 @@
elif i==6: # Connection closed by remote host
self.close()
raise ExceptionPxssh ('connection closed')
- else: # Unexpected
+ else: # Unexpected
self.close()
raise ExceptionPxssh ('unexpected login response')
if not self.synch_original_prompt():
@@ -278,7 +278,7 @@
if i==1:
return False
return True
-
+
def set_unique_prompt (self):
"""This sets the remote prompt to something more unique than # or $.
diff --git a/client/common_lib/test.py b/client/common_lib/test.py
index 52aaf54..dc79371 100644
--- a/client/common_lib/test.py
+++ b/client/common_lib/test.py
@@ -1,20 +1,20 @@
# Shell class for a test, inherited by all individual tests
#
# Methods:
-# __init__ initialise
-# initialize run once for each job
-# setup run once for each new version of the test installed
-# run run the test (wrapped by job.run_test())
+# __init__ initialise
+# initialize run once for each job
+# setup run once for each new version of the test installed
+# run run the test (wrapped by job.run_test())
#
# Data:
-# job backreference to the job this test instance is part of
-# outputdir eg. results/<job>/<testname.tag>
-# resultsdir eg. results/<job>/<testname.tag>/results
-# profdir eg. results/<job>/<testname.tag>/profiling
-# debugdir eg. results/<job>/<testname.tag>/debug
-# bindir eg. tests/<test>
-# src eg. tests/<test>/src
-# tmpdir eg. tmp/<testname.tag>
+# job backreference to the job this test instance is part of
+# outputdir eg. results/<job>/<testname.tag>
+# resultsdir eg. results/<job>/<testname.tag>/results
+# profdir eg. results/<job>/<testname.tag>/profiling
+# debugdir eg. results/<job>/<testname.tag>/debug
+# bindir eg. tests/<test>
+# src eg. tests/<test>/src
+# tmpdir eg. tmp/<testname.tag>
import os, sys, re, fcntl, shutil, tarfile, warnings
@@ -22,243 +22,243 @@
class base_test:
- preserve_srcdir = False
+ preserve_srcdir = False
- def __init__(self, job, bindir, outputdir):
- self.job = job
- self.autodir = job.autodir
+ def __init__(self, job, bindir, outputdir):
+ self.job = job
+ self.autodir = job.autodir
- self.outputdir = outputdir
- tagged_testname = os.path.basename(self.outputdir)
- self.resultsdir = os.path.join(self.outputdir, 'results')
- os.mkdir(self.resultsdir)
- self.profdir = os.path.join(self.outputdir, 'profiling')
- os.mkdir(self.profdir)
- self.debugdir = os.path.join(self.outputdir, 'debug')
- os.mkdir(self.debugdir)
- self.bindir = bindir
- if hasattr(job, 'libdir'):
- self.libdir = job.libdir
- self.srcdir = os.path.join(self.bindir, 'src')
+ self.outputdir = outputdir
+ tagged_testname = os.path.basename(self.outputdir)
+ self.resultsdir = os.path.join(self.outputdir, 'results')
+ os.mkdir(self.resultsdir)
+ self.profdir = os.path.join(self.outputdir, 'profiling')
+ os.mkdir(self.profdir)
+ self.debugdir = os.path.join(self.outputdir, 'debug')
+ os.mkdir(self.debugdir)
+ self.bindir = bindir
+ if hasattr(job, 'libdir'):
+ self.libdir = job.libdir
+ self.srcdir = os.path.join(self.bindir, 'src')
- self.tmpdir = os.path.join(job.tmpdir, tagged_testname)
+ self.tmpdir = os.path.join(job.tmpdir, tagged_testname)
- if os.path.exists(self.tmpdir):
- shutil.rmtree(self.tmpdir)
- os.mkdir(self.tmpdir)
+ if os.path.exists(self.tmpdir):
+ shutil.rmtree(self.tmpdir)
+ os.mkdir(self.tmpdir)
- self.job.stdout.tee_redirect(
- os.path.join(self.debugdir, 'stdout'))
- self.job.stderr.tee_redirect(
- os.path.join(self.debugdir, 'stderr'))
- try:
- self.initialize()
- # compile and install the test, if needed.
- utils.update_version(self.srcdir, self.preserve_srcdir,
- self.version, self.setup)
- finally:
- self.job.stderr.restore()
- self.job.stdout.restore()
+ self.job.stdout.tee_redirect(
+ os.path.join(self.debugdir, 'stdout'))
+ self.job.stderr.tee_redirect(
+ os.path.join(self.debugdir, 'stderr'))
+ try:
+ self.initialize()
+ # compile and install the test, if needed.
+ utils.update_version(self.srcdir, self.preserve_srcdir,
+ self.version, self.setup)
+ finally:
+ self.job.stderr.restore()
+ self.job.stdout.restore()
- def assert_(self, expr, msg='Assertion failed.'):
- if not expr:
- raise error.TestError(msg)
+ def assert_(self, expr, msg='Assertion failed.'):
+ if not expr:
+ raise error.TestError(msg)
- def write_test_keyval(self, attr_dict):
- utils.write_keyval(self.outputdir, attr_dict)
+ def write_test_keyval(self, attr_dict):
+ utils.write_keyval(self.outputdir, attr_dict)
- @staticmethod
- def _append_type_to_keys(dictionary, typename):
- new_dict = {}
- for key, value in dictionary.iteritems():
- new_key = "%s{%s}" % (key, typename)
- new_dict[new_key] = value
- return new_dict
+ @staticmethod
+ def _append_type_to_keys(dictionary, typename):
+ new_dict = {}
+ for key, value in dictionary.iteritems():
+ new_key = "%s{%s}" % (key, typename)
+ new_dict[new_key] = value
+ return new_dict
- def write_iteration_keyval(self, attr_dict, perf_dict):
- attr_dict = self._append_type_to_keys(attr_dict, "attr")
- perf_dict = self._append_type_to_keys(perf_dict, "perf")
+ def write_iteration_keyval(self, attr_dict, perf_dict):
+ attr_dict = self._append_type_to_keys(attr_dict, "attr")
+ perf_dict = self._append_type_to_keys(perf_dict, "perf")
- utils.write_keyval(self.resultsdir, attr_dict,
- type_tag="attr")
- utils.write_keyval(self.resultsdir, perf_dict,
- type_tag="perf")
+ utils.write_keyval(self.resultsdir, attr_dict,
+ type_tag="attr")
+ utils.write_keyval(self.resultsdir, perf_dict,
+ type_tag="perf")
- keyval_path = os.path.join(self.resultsdir, "keyval")
- print >> open(keyval_path, "a"), ""
+ keyval_path = os.path.join(self.resultsdir, "keyval")
+ print >> open(keyval_path, "a"), ""
- # TODO: deprecate, remove from code in favour of
- # the write_*_keyval methods
- def write_keyval(self, dictionary):
- warnings.warn("test.write_keyval is deprecated, use "
- "test.write_test_keyval or "
- "test.write_iteration_keyval instead",
- DeprecationWarning)
- self.write_iteration_keyval({}, dictionary)
+ # TODO: deprecate, remove from code in favour of
+ # the write_*_keyval methods
+ def write_keyval(self, dictionary):
+ warnings.warn("test.write_keyval is deprecated, use "
+ "test.write_test_keyval or "
+ "test.write_iteration_keyval instead",
+ DeprecationWarning)
+ self.write_iteration_keyval({}, dictionary)
- def initialize(self):
- pass
+ def initialize(self):
+ pass
- def setup(self):
- pass
+ def setup(self):
+ pass
- def cleanup(self):
- pass
+ def cleanup(self):
+ pass
- def _exec(self, args, dargs):
- try:
- self.job.stdout.tee_redirect(
- os.path.join(self.debugdir, 'stdout'))
- self.job.stderr.tee_redirect(
- os.path.join(self.debugdir, 'stderr'))
+ def _exec(self, args, dargs):
+ try:
+ self.job.stdout.tee_redirect(
+ os.path.join(self.debugdir, 'stdout'))
+ self.job.stderr.tee_redirect(
+ os.path.join(self.debugdir, 'stderr'))
- try:
- os.chdir(self.outputdir)
- dargs = dargs.copy()
- keyvals = dargs.pop('test_attributes', dict())
- keyvals = keyvals.copy()
- keyvals['version'] = self.version
- self.write_test_keyval(keyvals)
+ try:
+ os.chdir(self.outputdir)
+ dargs = dargs.copy()
+ keyvals = dargs.pop('test_attributes', dict())
+ keyvals = keyvals.copy()
+ keyvals['version'] = self.version
+ self.write_test_keyval(keyvals)
- self.execute(*args, **dargs)
- finally:
- self.cleanup()
- self.job.stderr.restore()
- self.job.stdout.restore()
- except error.AutotestError:
- raise
- except Exception, e:
- msg = "Unhandled %s error occured during test\n"
- msg %= str(e.__class__.__name__)
- raise error.UnhandledError(msg)
+ self.execute(*args, **dargs)
+ finally:
+ self.cleanup()
+ self.job.stderr.restore()
+ self.job.stdout.restore()
+ except error.AutotestError:
+ raise
+ except Exception, e:
+ msg = "Unhandled %s error occured during test\n"
+ msg %= str(e.__class__.__name__)
+ raise error.UnhandledError(msg)
def testname(url):
- # Extract the testname from the test url.
- match = re.match('[^:]+://(.*)/([^/]*)$', url)
- if not match:
- return ('', url)
- (group, filename) = match.groups()
+ # Extract the testname from the test url.
+ match = re.match('[^:]+://(.*)/([^/]*)$', url)
+ if not match:
+ return ('', url)
+ (group, filename) = match.groups()
- # Generate the group prefix.
- group = re.sub(r'\W', '_', group)
+ # Generate the group prefix.
+ group = re.sub(r'\W', '_', group)
- # Drop the extension to get the raw test name.
- testname = re.sub(r'\.tgz', '', filename)
+ # Drop the extension to get the raw test name.
+ testname = re.sub(r'\.tgz', '', filename)
- return (group, testname)
+ return (group, testname)
def _installtest(job, url):
- (group, name) = testname(url)
+ (group, name) = testname(url)
- # Bail if the test is already installed
- group_dir = os.path.join(job.testdir, "download", group)
- if os.path.exists(os.path.join(group_dir, name)):
- return (group, name)
+ # Bail if the test is already installed
+ group_dir = os.path.join(job.testdir, "download", group)
+ if os.path.exists(os.path.join(group_dir, name)):
+ return (group, name)
- # If the group directory is missing create it and add
- # an empty __init__.py so that sub-directories are
- # considered for import.
- if not os.path.exists(group_dir):
- os.mkdir(group_dir)
- f = file(os.path.join(group_dir, '__init__.py'), 'w+')
- f.close()
+ # If the group directory is missing create it and add
+ # an empty __init__.py so that sub-directories are
+ # considered for import.
+ if not os.path.exists(group_dir):
+ os.mkdir(group_dir)
+ f = file(os.path.join(group_dir, '__init__.py'), 'w+')
+ f.close()
- print name + ": installing test url=" + url
- get_file(url, os.path.join(group_dir, 'test.tgz'))
- old_wd = os.getcwd()
- os.chdir(group_dir)
- tar = tarfile.open('test.tgz')
- for member in tar.getmembers():
- tar.extract(member)
- tar.close()
- os.chdir(old_wd)
- os.remove(os.path.join(group_dir, 'test.tgz'))
+ print name + ": installing test url=" + url
+ get_file(url, os.path.join(group_dir, 'test.tgz'))
+ old_wd = os.getcwd()
+ os.chdir(group_dir)
+ tar = tarfile.open('test.tgz')
+ for member in tar.getmembers():
+ tar.extract(member)
+ tar.close()
+ os.chdir(old_wd)
+ os.remove(os.path.join(group_dir, 'test.tgz'))
- # For this 'sub-object' to be importable via the name
- # 'group.name' we need to provide an __init__.py,
- # so link the main entry point to this.
- os.symlink(name + '.py', os.path.join(group_dir, name,
- '__init__.py'))
+ # For this 'sub-object' to be importable via the name
+ # 'group.name' we need to provide an __init__.py,
+ # so link the main entry point to this.
+ os.symlink(name + '.py', os.path.join(group_dir, name,
+ '__init__.py'))
- # The test is now installed.
- return (group, name)
+ # The test is now installed.
+ return (group, name)
def runtest(job, url, tag, args, dargs,
- local_namespace={}, global_namespace={}, after_test_hook=None):
- local_namespace = local_namespace.copy()
- global_namespace = global_namespace.copy()
+ local_namespace={}, global_namespace={}, after_test_hook=None):
+ local_namespace = local_namespace.copy()
+ global_namespace = global_namespace.copy()
- # if this is not a plain test name then download and install the
- # specified test
- if utils.is_url(url):
- (group, testname) = _installtest(job, url)
- bindir = os.path.join(job.testdir, 'download', group, testname)
- site_bindir = None
- else:
- # if the test is local, it can be found in either testdir
- # or site_testdir. tests in site_testdir override tests
- # defined in testdir
- (group, testname) = ('', url)
- bindir = os.path.join(job.testdir, group, testname)
- if hasattr(job, 'site_testdir'):
- site_bindir = os.path.join(job.site_testdir,
- group, testname)
- else:
- site_bindir = None
+ # if this is not a plain test name then download and install the
+ # specified test
+ if utils.is_url(url):
+ (group, testname) = _installtest(job, url)
+ bindir = os.path.join(job.testdir, 'download', group, testname)
+ site_bindir = None
+ else:
+ # if the test is local, it can be found in either testdir
+ # or site_testdir. tests in site_testdir override tests
+ # defined in testdir
+ (group, testname) = ('', url)
+ bindir = os.path.join(job.testdir, group, testname)
+ if hasattr(job, 'site_testdir'):
+ site_bindir = os.path.join(job.site_testdir,
+ group, testname)
+ else:
+ site_bindir = None
- outputdir = os.path.join(job.resultdir, testname)
- if tag:
- outputdir += '.' + tag
+ outputdir = os.path.join(job.resultdir, testname)
+ if tag:
+ outputdir += '.' + tag
- # if we can find the test in site_bindir, use this version
- if site_bindir and os.path.exists(site_bindir):
- bindir = site_bindir
- testdir = job.site_testdir
- elif os.path.exists(bindir):
- testdir = job.testdir
- elif not os.path.exists(bindir):
- raise error.TestError(testname + ': test does not exist')
+ # if we can find the test in site_bindir, use this version
+ if site_bindir and os.path.exists(site_bindir):
+ bindir = site_bindir
+ testdir = job.site_testdir
+ elif os.path.exists(bindir):
+ testdir = job.testdir
+ elif not os.path.exists(bindir):
+ raise error.TestError(testname + ': test does not exist')
- if group:
- sys.path.insert(0, os.path.join(testdir, 'download'))
- group += '.'
- else:
- sys.path.insert(0, os.path.join(testdir, testname))
+ if group:
+ sys.path.insert(0, os.path.join(testdir, 'download'))
+ group += '.'
+ else:
+ sys.path.insert(0, os.path.join(testdir, testname))
- local_namespace['job'] = job
- local_namespace['bindir'] = bindir
- local_namespace['outputdir'] = outputdir
+ local_namespace['job'] = job
+ local_namespace['bindir'] = bindir
+ local_namespace['outputdir'] = outputdir
- lockfile = open(os.path.join(job.tmpdir, '.testlock'), 'w')
- try:
- fcntl.flock(lockfile, fcntl.LOCK_EX)
- exec ("import %s%s" % (group, testname),
- local_namespace, global_namespace)
- exec ("mytest = %s%s.%s(job, bindir, outputdir)" %
- (group, testname, testname),
- local_namespace, global_namespace)
- finally:
- fcntl.flock(lockfile, fcntl.LOCK_UN)
- lockfile.close()
- sys.path.pop(0)
+ lockfile = open(os.path.join(job.tmpdir, '.testlock'), 'w')
+ try:
+ fcntl.flock(lockfile, fcntl.LOCK_EX)
+ exec ("import %s%s" % (group, testname),
+ local_namespace, global_namespace)
+ exec ("mytest = %s%s.%s(job, bindir, outputdir)" %
+ (group, testname, testname),
+ local_namespace, global_namespace)
+ finally:
+ fcntl.flock(lockfile, fcntl.LOCK_UN)
+ lockfile.close()
+ sys.path.pop(0)
- pwd = os.getcwd()
- os.chdir(outputdir)
- try:
- mytest = global_namespace['mytest']
- mytest._exec(args, dargs)
- finally:
- if after_test_hook:
- after_test_hook(mytest)
+ pwd = os.getcwd()
+ os.chdir(outputdir)
+ try:
+ mytest = global_namespace['mytest']
+ mytest._exec(args, dargs)
+ finally:
+ if after_test_hook:
+ after_test_hook(mytest)
diff --git a/client/common_lib/test_utils/mock.py b/client/common_lib/test_utils/mock.py
index 556e165..9906c94 100644
--- a/client/common_lib/test_utils/mock.py
+++ b/client/common_lib/test_utils/mock.py
@@ -5,426 +5,426 @@
class CheckPlaybackError(Exception):
- 'Raised when mock playback does not match recorded calls.'
+ 'Raised when mock playback does not match recorded calls.'
class ExitException(Exception):
- 'Raised when the mocked sys.exit() is called'
- pass
+ 'Raised when the mocked sys.exit() is called'
+ pass
class argument_comparator(object):
- def is_satisfied_by(self, parameter):
- raise NotImplementedError
+ def is_satisfied_by(self, parameter):
+ raise NotImplementedError
class equality_comparator(argument_comparator):
- def __init__(self, value):
- self.value = value
+ def __init__(self, value):
+ self.value = value
- def is_satisfied_by(self, parameter):
- return parameter == self.value
+ def is_satisfied_by(self, parameter):
+ return parameter == self.value
- def __str__(self):
- return repr(self.value)
+ def __str__(self):
+ return repr(self.value)
class regex_comparator(argument_comparator):
- def __init__(self, pattern, flags=0):
- self.regex = re.compile(pattern, flags)
+ def __init__(self, pattern, flags=0):
+ self.regex = re.compile(pattern, flags)
- def is_satisfied_by(self, parameter):
- return self.regex.search(parameter) is not None
+ def is_satisfied_by(self, parameter):
+ return self.regex.search(parameter) is not None
- def __str__(self):
- return self.regex.pattern
+ def __str__(self):
+ return self.regex.pattern
class is_string_comparator(argument_comparator):
- def is_satisfied_by(self, parameter):
- return isinstance(parameter, basestring)
+ def is_satisfied_by(self, parameter):
+ return isinstance(parameter, basestring)
- def __str__(self):
- return "a string"
+ def __str__(self):
+ return "a string"
class is_instance_comparator(argument_comparator):
- def __init__(self, cls):
- self.cls = cls
+ def __init__(self, cls):
+ self.cls = cls
- def is_satisfied_by(self, parameter):
- return isinstance(parameter, self.cls)
+ def is_satisfied_by(self, parameter):
+ return isinstance(parameter, self.cls)
- def __str__(self):
- return "is a %s" % self.cls
+ def __str__(self):
+ return "is a %s" % self.cls
class function_map(object):
- def __init__(self, symbol, return_val, *args, **dargs):
- self.return_val = return_val
- self.args = []
- self.symbol = symbol
- for arg in args:
- if isinstance(arg, argument_comparator):
- self.args.append(arg)
- else:
- self.args.append(equality_comparator(arg))
+ def __init__(self, symbol, return_val, *args, **dargs):
+ self.return_val = return_val
+ self.args = []
+ self.symbol = symbol
+ for arg in args:
+ if isinstance(arg, argument_comparator):
+ self.args.append(arg)
+ else:
+ self.args.append(equality_comparator(arg))
- self.dargs = dargs
- self.error = None
+ self.dargs = dargs
+ self.error = None
- def and_return(self, return_val):
- self.return_val = return_val
+ def and_return(self, return_val):
+ self.return_val = return_val
- def and_raises(self, error):
- self.error = error
+ def and_raises(self, error):
+ self.error = error
- def match(self, *args, **dargs):
- if len(args) != len(self.args) or len(dargs) != len(self.dargs):
- return False
+ def match(self, *args, **dargs):
+ if len(args) != len(self.args) or len(dargs) != len(self.dargs):
+ return False
- for i, expected_arg in enumerate(self.args):
- if not expected_arg.is_satisfied_by(args[i]):
- return False
+ for i, expected_arg in enumerate(self.args):
+ if not expected_arg.is_satisfied_by(args[i]):
+ return False
- if self.dargs != dargs:
- return False
+ if self.dargs != dargs:
+ return False
- return True
+ return True
- def __str__(self):
- return _dump_function_call(self.symbol, self.args, self.dargs)
+ def __str__(self):
+ return _dump_function_call(self.symbol, self.args, self.dargs)
class mock_function(object):
- def __init__(self, symbol, default_return_val=None,
- record=None, playback=None):
- self.default_return_val = default_return_val
- self.num_calls = 0
- self.args = []
- self.dargs = []
- self.symbol = symbol
- self.record = record
- self.playback = playback
- self.__name__ = symbol
+ def __init__(self, symbol, default_return_val=None,
+ record=None, playback=None):
+ self.default_return_val = default_return_val
+ self.num_calls = 0
+ self.args = []
+ self.dargs = []
+ self.symbol = symbol
+ self.record = record
+ self.playback = playback
+ self.__name__ = symbol
- def __call__(self, *args, **dargs):
- self.num_calls += 1
- self.args.append(args)
- self.dargs.append(dargs)
- if self.playback:
- return self.playback(self.symbol, *args, **dargs)
- else:
- return self.default_return_val
+ def __call__(self, *args, **dargs):
+ self.num_calls += 1
+ self.args.append(args)
+ self.dargs.append(dargs)
+ if self.playback:
+ return self.playback(self.symbol, *args, **dargs)
+ else:
+ return self.default_return_val
- def expect_call(self, *args, **dargs):
- mapping = function_map(self.symbol, None, *args, **dargs)
- if self.record:
- self.record(mapping)
+ def expect_call(self, *args, **dargs):
+ mapping = function_map(self.symbol, None, *args, **dargs)
+ if self.record:
+ self.record(mapping)
- return mapping
+ return mapping
class mask_function(mock_function):
- def __init__(self, symbol, original_function, default_return_val=None,
- record=None, playback=None):
- super(mask_function, self).__init__(symbol,
- default_return_val,
- record, playback)
- self.original_function = original_function
+ def __init__(self, symbol, original_function, default_return_val=None,
+ record=None, playback=None):
+ super(mask_function, self).__init__(symbol,
+ default_return_val,
+ record, playback)
+ self.original_function = original_function
- def run_original_function(self, *args, **dargs):
- return self.original_function(*args, **dargs)
+ def run_original_function(self, *args, **dargs):
+ return self.original_function(*args, **dargs)
class mock_class(object):
- def __init__(self, cls, name, default_ret_val=None,
- record=None, playback=None):
- self.errors = []
- self.name = name
- self.record = record
- self.playback = playback
+ def __init__(self, cls, name, default_ret_val=None,
+ record=None, playback=None):
+ self.errors = []
+ self.name = name
+ self.record = record
+ self.playback = playback
- for symbol in dir(cls):
- if symbol.startswith("_"):
- continue
+ for symbol in dir(cls):
+ if symbol.startswith("_"):
+ continue
- orig_symbol = getattr(cls, symbol)
- if callable(orig_symbol):
- f_name = "%s.%s" % (self.name, symbol)
- func = mock_function(f_name, default_ret_val,
- self.record, self.playback)
- setattr(self, symbol, func)
- else:
- setattr(self, symbol, orig_symbol)
+ orig_symbol = getattr(cls, symbol)
+ if callable(orig_symbol):
+ f_name = "%s.%s" % (self.name, symbol)
+ func = mock_function(f_name, default_ret_val,
+ self.record, self.playback)
+ setattr(self, symbol, func)
+ else:
+ setattr(self, symbol, orig_symbol)
class mock_god:
- NONEXISTENT_ATTRIBUTE = object()
+ NONEXISTENT_ATTRIBUTE = object()
- def __init__(self, debug=False):
- """
- With debug=True, all recorded method calls will be printed as
- they happen.
- """
- self.recording = collections.deque()
- self.errors = []
- self._stubs = []
- self._debug = debug
+ def __init__(self, debug=False):
+ """
+ With debug=True, all recorded method calls will be printed as
+ they happen.
+ """
+ self.recording = collections.deque()
+ self.errors = []
+ self._stubs = []
+ self._debug = debug
- def create_mock_class_obj(self, cls, name, default_ret_val=None):
- record = self.__record_call
- playback = self.__method_playback
- errors = self.errors
+ def create_mock_class_obj(self, cls, name, default_ret_val=None):
+ record = self.__record_call
+ playback = self.__method_playback
+ errors = self.errors
- class cls_sub(cls):
- cls_count = 0
- creations = collections.deque()
+ class cls_sub(cls):
+ cls_count = 0
+ creations = collections.deque()
- # overwrite the initializer
- def __init__(self, *args, **dargs):
- pass
+ # overwrite the initializer
+ def __init__(self, *args, **dargs):
+ pass
- @classmethod
- def expect_new(typ, *args, **dargs):
- obj = typ.make_new(*args, **dargs)
- typ.creations.append(obj)
- return obj
+ @classmethod
+ def expect_new(typ, *args, **dargs):
+ obj = typ.make_new(*args, **dargs)
+ typ.creations.append(obj)
+ return obj
- def __new__(typ, *args, **dargs):
- if len(typ.creations) == 0:
- msg = ("not expecting call to %s "
- "constructor" % (name))
- errors.append(msg)
- return None
- else:
- return typ.creations.popleft()
+ def __new__(typ, *args, **dargs):
+ if len(typ.creations) == 0:
+ msg = ("not expecting call to %s "
+ "constructor" % (name))
+ errors.append(msg)
+ return None
+ else:
+ return typ.creations.popleft()
- @classmethod
- def make_new(typ, *args, **dargs):
- obj = super(cls_sub, typ).__new__(typ, *args,
- **dargs)
+ @classmethod
+ def make_new(typ, *args, **dargs):
+ obj = super(cls_sub, typ).__new__(typ, *args,
+ **dargs)
- typ.cls_count += 1
- obj_name = "%s_%s" % (name, typ.cls_count)
- for symbol in dir(obj):
- if (symbol.startswith("__") and
- symbol.endswith("__")):
- continue
+ typ.cls_count += 1
+ obj_name = "%s_%s" % (name, typ.cls_count)
+ for symbol in dir(obj):
+ if (symbol.startswith("__") and
+ symbol.endswith("__")):
+ continue
- orig_symbol = getattr(obj, symbol)
- if callable(orig_symbol):
- f_name = ("%s.%s" %
- (obj_name, symbol))
- func = mock_function(f_name,
- default_ret_val,
- record,
- playback)
- setattr(obj, symbol, func)
- else:
- setattr(obj, symbol,
- orig_symbol)
+ orig_symbol = getattr(obj, symbol)
+ if callable(orig_symbol):
+ f_name = ("%s.%s" %
+ (obj_name, symbol))
+ func = mock_function(f_name,
+ default_ret_val,
+ record,
+ playback)
+ setattr(obj, symbol, func)
+ else:
+ setattr(obj, symbol,
+ orig_symbol)
- return obj
+ return obj
- return cls_sub
+ return cls_sub
- def create_mock_class(self, cls, name, default_ret_val=None):
- """
- Given something that defines a namespace cls (class, object,
- module), and a (hopefully unique) name, will create a
- mock_class object with that name and that possessess all
- the public attributes of cls. default_ret_val sets the
- default_ret_val on all methods of the cls mock.
- """
- return mock_class(cls, name, default_ret_val,
- self.__record_call, self.__method_playback)
+ def create_mock_class(self, cls, name, default_ret_val=None):
+ """
+ Given something that defines a namespace cls (class, object,
+ module), and a (hopefully unique) name, will create a
+ mock_class object with that name and that possessess all
+ the public attributes of cls. default_ret_val sets the
+ default_ret_val on all methods of the cls mock.
+ """
+ return mock_class(cls, name, default_ret_val,
+ self.__record_call, self.__method_playback)
- def create_mock_function(self, symbol, default_return_val=None):
- """
- create a mock_function with name symbol and default return
- value of default_ret_val.
- """
- return mock_function(symbol, default_return_val,
- self.__record_call, self.__method_playback)
+ def create_mock_function(self, symbol, default_return_val=None):
+ """
+ create a mock_function with name symbol and default return
+ value of default_ret_val.
+ """
+ return mock_function(symbol, default_return_val,
+ self.__record_call, self.__method_playback)
- def mock_up(self, obj, name, default_ret_val=None):
- """
- Given an object (class instance or module) and a registration
- name, then replace all its methods with mock function objects
- (passing the orignal functions to the mock functions).
- """
- for symbol in dir(obj):
- if symbol.startswith("__"):
- continue
+ def mock_up(self, obj, name, default_ret_val=None):
+ """
+ Given an object (class instance or module) and a registration
+ name, then replace all its methods with mock function objects
+ (passing the orignal functions to the mock functions).
+ """
+ for symbol in dir(obj):
+ if symbol.startswith("__"):
+ continue
- orig_symbol = getattr(obj, symbol)
- if callable(orig_symbol):
- f_name = "%s.%s" % (name, symbol)
- func = mask_function(f_name, orig_symbol,
- default_ret_val,
- self.__record_call,
- self.__method_playback)
- setattr(obj, symbol, func)
+ orig_symbol = getattr(obj, symbol)
+ if callable(orig_symbol):
+ f_name = "%s.%s" % (name, symbol)
+ func = mask_function(f_name, orig_symbol,
+ default_ret_val,
+ self.__record_call,
+ self.__method_playback)
+ setattr(obj, symbol, func)
- def stub_with(self, namespace, symbol, new_attribute):
- original_attribute = getattr(namespace, symbol,
- self.NONEXISTENT_ATTRIBUTE)
- self._stubs.append((namespace, symbol, original_attribute))
- setattr(namespace, symbol, new_attribute)
+ def stub_with(self, namespace, symbol, new_attribute):
+ original_attribute = getattr(namespace, symbol,
+ self.NONEXISTENT_ATTRIBUTE)
+ self._stubs.append((namespace, symbol, original_attribute))
+ setattr(namespace, symbol, new_attribute)
- def stub_function(self, namespace, symbol):
- mock_attribute = self.create_mock_function(symbol)
- self.stub_with(namespace, symbol, mock_attribute)
+ def stub_function(self, namespace, symbol):
+ mock_attribute = self.create_mock_function(symbol)
+ self.stub_with(namespace, symbol, mock_attribute)
- def stub_class_method(self, cls, symbol):
- mock_attribute = self.create_mock_function(symbol)
- self.stub_with(cls, symbol, staticmethod(mock_attribute))
+ def stub_class_method(self, cls, symbol):
+ mock_attribute = self.create_mock_function(symbol)
+ self.stub_with(cls, symbol, staticmethod(mock_attribute))
- def unstub_all(self):
- self._stubs.reverse()
- for namespace, symbol, original_attribute in self._stubs:
- if original_attribute == self.NONEXISTENT_ATTRIBUTE:
- delattr(namespace, symbol)
- else:
- setattr(namespace, symbol, original_attribute)
- self._stubs = []
+ def unstub_all(self):
+ self._stubs.reverse()
+ for namespace, symbol, original_attribute in self._stubs:
+ if original_attribute == self.NONEXISTENT_ATTRIBUTE:
+ delattr(namespace, symbol)
+ else:
+ setattr(namespace, symbol, original_attribute)
+ self._stubs = []
- def __method_playback(self, symbol, *args, **dargs):
- if self._debug:
- print 'Mock call:', _dump_function_call(symbol,
- args, dargs)
- if len(self.recording) != 0:
- func_call = self.recording[0]
- if func_call.symbol != symbol:
- msg = ("Unexpected call: %s. Expected %s"
- % (_dump_function_call(symbol, args, dargs),
- func_call))
- self.errors.append(msg)
- return None
+ def __method_playback(self, symbol, *args, **dargs):
+ if self._debug:
+ print 'Mock call:', _dump_function_call(symbol,
+ args, dargs)
+ if len(self.recording) != 0:
+ func_call = self.recording[0]
+ if func_call.symbol != symbol:
+ msg = ("Unexpected call: %s. Expected %s"
+ % (_dump_function_call(symbol, args, dargs),
+ func_call))
+ self.errors.append(msg)
+ return None
- if not func_call.match(*args, **dargs):
- msg = ("%s called. Expected %s"
- % (_dump_function_call(symbol, args, dargs),
- func_call))
- self.errors.append(msg)
- return None
+ if not func_call.match(*args, **dargs):
+ msg = ("%s called. Expected %s"
+ % (_dump_function_call(symbol, args, dargs),
+ func_call))
+ self.errors.append(msg)
+ return None
- # this is the expected call so pop it and return
- self.recording.popleft()
- if func_call.error:
- raise func_call.error
- else:
- return func_call.return_val
- else:
- msg = ("unexpected call: %s"
- % (_dump_function_call(symbol, args, dargs)))
- self.errors.append(msg)
- return None
+ # this is the expected call so pop it and return
+ self.recording.popleft()
+ if func_call.error:
+ raise func_call.error
+ else:
+ return func_call.return_val
+ else:
+ msg = ("unexpected call: %s"
+ % (_dump_function_call(symbol, args, dargs)))
+ self.errors.append(msg)
+ return None
- def __record_call(self, mapping):
- self.recording.append(mapping)
+ def __record_call(self, mapping):
+ self.recording.append(mapping)
- def check_playback(self):
- """
- Report any errors that were encounterd during calls
- to __method_playback().
- """
- if len(self.errors) > 0:
- for error in self.errors:
- print error
- raise CheckPlaybackError
- elif len(self.recording) != 0:
- for func_call in self.recording:
- print "%s not called" % (func_call)
- raise CheckPlaybackError
+ def check_playback(self):
+ """
+ Report any errors that were encounterd during calls
+ to __method_playback().
+ """
+ if len(self.errors) > 0:
+ for error in self.errors:
+ print error
+ raise CheckPlaybackError
+ elif len(self.recording) != 0:
+ for func_call in self.recording:
+ print "%s not called" % (func_call)
+ raise CheckPlaybackError
- def mock_exit(self):
- def mock_exit_handler(self):
- raise ExitException
+ def mock_exit(self):
+ def mock_exit_handler(self):
+ raise ExitException
- self.saved_exit = sys.exit
- sys.exit = mock_exit_handler
+ self.saved_exit = sys.exit
+ sys.exit = mock_exit_handler
- def unmock_exit(self):
- sys.exit = self.saved_exit
- self.saved_exit = None
+ def unmock_exit(self):
+ sys.exit = self.saved_exit
+ self.saved_exit = None
- def mock_stdout_stderr(self):
- """Mocks and saves the stdout & stderr output"""
- self.mock_streams_stdout = StringIO.StringIO('')
- self.mock_streams_stderr = StringIO.StringIO('')
+ def mock_stdout_stderr(self):
+ """Mocks and saves the stdout & stderr output"""
+ self.mock_streams_stdout = StringIO.StringIO('')
+ self.mock_streams_stderr = StringIO.StringIO('')
- sys.stdout = self.mock_streams_stdout
- sys.stderr = self.mock_streams_stderr
+ sys.stdout = self.mock_streams_stdout
+ sys.stderr = self.mock_streams_stderr
- def unmock_stdout_stderr(self):
- """Restores the stdout & stderr, and returns both
- output strings"""
- sys.stdout = sys.__stdout__
- sys.stderr = sys.__stderr__
- values = (self.mock_streams_stdout.getvalue(),
- self.mock_streams_stderr.getvalue())
+ def unmock_stdout_stderr(self):
+ """Restores the stdout & stderr, and returns both
+ output strings"""
+ sys.stdout = sys.__stdout__
+ sys.stderr = sys.__stderr__
+ values = (self.mock_streams_stdout.getvalue(),
+ self.mock_streams_stderr.getvalue())
- self.mock_streams_stdout.close()
- self.mock_streams_stderr.close()
- return values
+ self.mock_streams_stdout.close()
+ self.mock_streams_stderr.close()
+ return values
- def mock_io_exit(self):
- self.mock_exit()
- self.mock_stdout_stderr()
+ def mock_io_exit(self):
+ self.mock_exit()
+ self.mock_stdout_stderr()
- def unmock_io_exit(self):
- self.unmock_exit()
- return self.unmock_stdout_stderr()
+ def unmock_io_exit(self):
+ self.unmock_exit()
+ return self.unmock_stdout_stderr()
def _arg_to_str(arg):
- if isinstance(arg, argument_comparator):
- return str(arg)
- return repr(arg)
+ if isinstance(arg, argument_comparator):
+ return str(arg)
+ return repr(arg)
def _dump_function_call(symbol, args, dargs):
- arg_vec = []
- for arg in args:
- arg_vec.append(_arg_to_str(arg))
- for key, val in dargs.iteritems():
- arg_vec.append("%s=%s" % (key, _arg_to_stv(val)))
- return "%s(%s)" % (symbol, ', '.join(arg_vec))
+ arg_vec = []
+ for arg in args:
+ arg_vec.append(_arg_to_str(arg))
+ for key, val in dargs.iteritems():
+ arg_vec.append("%s=%s" % (key, _arg_to_stv(val)))
+ return "%s(%s)" % (symbol, ', '.join(arg_vec))
diff --git a/client/common_lib/test_utils/mock_demo.py b/client/common_lib/test_utils/mock_demo.py
index fec116b..f01fe60 100644
--- a/client/common_lib/test_utils/mock_demo.py
+++ b/client/common_lib/test_utils/mock_demo.py
@@ -5,140 +5,140 @@
import mock, mock_demo_MUT
class MyError(Exception):
- pass
+ pass
class A(object):
- var = 8
+ var = 8
- def __init__(self):
- self.x = 0
+ def __init__(self):
+ self.x = 0
- def method1(self):
- self.x += 1
- return self.x
+ def method1(self):
+ self.x += 1
+ return self.x
- def method2(self, y):
- return y * self.x
+ def method2(self, y):
+ return y * self.x
class B(A):
- def method3(self, z):
- return self.x + z
+ def method3(self, z):
+ return self.x + z
- def method4(self, z, w):
- return self.x * z + w
+ def method4(self, z, w):
+ return self.x * z + w
class C(B):
- def method5(self):
- self.method1()
- t = self.method2(4)
- u = self.method3(t)
- return u
+ def method5(self):
+ self.method1()
+ t = self.method2(4)
+ u = self.method3(t)
+ return u
class D(C):
- def method6(self, error):
- if error:
- raise MyError("woops")
- else:
- return 10
+ def method6(self, error):
+ if error:
+ raise MyError("woops")
+ else:
+ return 10
class E(D):
- def __init__(self, val):
- self.val = val
+ def __init__(self, val):
+ self.val = val
# say we want to test that do_stuff is doing what we think it is doing
def do_stuff(a, b, func):
- print b.method1()
- print b.method3(10)
- print func("how many")
- print a.method2(5)
- print b.method1()
- print b.method4(1, 4)
- print b.method2(3)
- print b.method2("hello")
+ print b.method1()
+ print b.method3(10)
+ print func("how many")
+ print a.method2(5)
+ print b.method1()
+ print b.method4(1, 4)
+ print b.method2(3)
+ print b.method2("hello")
def do_more_stuff(d):
- print d.method6(False)
- try:
- d.method6(True)
- except:
- print "caught error"
+ print d.method6(False)
+ try:
+ d.method6(True)
+ except:
+ print "caught error"
def main():
- god = mock.mock_god()
+ god = mock.mock_god()
- m1 = god.create_mock_class(A, "A")
- print m1.var
- m2 = god.create_mock_class(B, "B")
- f = god.create_mock_function("func")
+ m1 = god.create_mock_class(A, "A")
+ print m1.var
+ m2 = god.create_mock_class(B, "B")
+ f = god.create_mock_function("func")
- print dir(m1)
- print dir(m2)
+ print dir(m1)
+ print dir(m2)
- # sets up the "recording"
- m2.method1.expect_call().and_return(1)
- m2.method3.expect_call(10).and_return(10)
- f.expect_call("how many").and_return(42)
- m1.method2.expect_call(5).and_return(0)
- m2.method1.expect_call().and_return(2)
- m2.method4.expect_call(1, 4).and_return(6)
- m2.method2.expect_call(3).and_return(6)
- m2.method2.expect_call(mock.is_string_comparator()).and_return("foo")
+ # sets up the "recording"
+ m2.method1.expect_call().and_return(1)
+ m2.method3.expect_call(10).and_return(10)
+ f.expect_call("how many").and_return(42)
+ m1.method2.expect_call(5).and_return(0)
+ m2.method1.expect_call().and_return(2)
+ m2.method4.expect_call(1, 4).and_return(6)
+ m2.method2.expect_call(3).and_return(6)
+ m2.method2.expect_call(mock.is_string_comparator()).and_return("foo")
- # check the recording order
- for func_call in god.recording:
- print func_call
+ # check the recording order
+ for func_call in god.recording:
+ print func_call
- # once we start making calls into the methods we are in
- # playback mode
- do_stuff(m1, m2, f)
+ # once we start making calls into the methods we are in
+ # playback mode
+ do_stuff(m1, m2, f)
- # we can now check that playback succeeded
- god.check_playback()
+ # we can now check that playback succeeded
+ god.check_playback()
- # now test the ability to mock out all methods of an object
- # except those under test
- c = C()
- god.mock_up(c, "c")
+ # now test the ability to mock out all methods of an object
+ # except those under test
+ c = C()
+ god.mock_up(c, "c")
- # setup recording
- c.method1.expect_call()
- c.method2.expect_call(4).and_return(4)
- c.method3.expect_call(4).and_return(5)
+ # setup recording
+ c.method1.expect_call()
+ c.method2.expect_call(4).and_return(4)
+ c.method3.expect_call(4).and_return(5)
- # perform the test
- answer = c.method5.run_original_function()
+ # perform the test
+ answer = c.method5.run_original_function()
- # check playback
- print "answer = %s" % (answer)
- god.check_playback()
+ # check playback
+ print "answer = %s" % (answer)
+ god.check_playback()
- # check exception returns too
- m3 = god.create_mock_class(D, "D")
- m3.method6.expect_call(False).and_return(10)
- m3.method6.expect_call(True).and_raises(MyError("woops"))
+ # check exception returns too
+ m3 = god.create_mock_class(D, "D")
+ m3.method6.expect_call(False).and_return(10)
+ m3.method6.expect_call(True).and_raises(MyError("woops"))
- do_more_stuff(m3)
- god.check_playback()
+ do_more_stuff(m3)
+ god.check_playback()
- # now check we can mock out a whole class (rather than just an instance)
- mockE = god.create_mock_class_obj(E, "E")
- oldE = mock_demo_MUT.E
- mock_demo_MUT.E = mockE
+ # now check we can mock out a whole class (rather than just an instance)
+ mockE = god.create_mock_class_obj(E, "E")
+ oldE = mock_demo_MUT.E
+ mock_demo_MUT.E = mockE
- m4 = mockE.expect_new(val=7)
- m4.method1.expect_call().and_return(1)
+ m4 = mockE.expect_new(val=7)
+ m4.method1.expect_call().and_return(1)
- mock_demo_MUT.do_create_stuff()
- god.check_playback()
+ mock_demo_MUT.do_create_stuff()
+ god.check_playback()
- mock_demo_MUT.E = oldE
+ mock_demo_MUT.E = oldE
if __name__ == "__main__":
- main()
+ main()
diff --git a/client/common_lib/test_utils/mock_demo_MUT.py b/client/common_lib/test_utils/mock_demo_MUT.py
index c44e5a2..b2fde77 100644
--- a/client/common_lib/test_utils/mock_demo_MUT.py
+++ b/client/common_lib/test_utils/mock_demo_MUT.py
@@ -1,5 +1,5 @@
from mock_demo import E
def do_create_stuff():
- obj = E(val=7)
- print obj.method1()
+ obj = E(val=7)
+ print obj.method1()
diff --git a/client/common_lib/utils.py b/client/common_lib/utils.py
index 92e53f7..f19ecc5 100644
--- a/client/common_lib/utils.py
+++ b/client/common_lib/utils.py
@@ -8,360 +8,360 @@
def read_one_line(filename):
- return open(filename, 'r').readline().strip()
+ return open(filename, 'r').readline().strip()
def write_one_line(filename, str):
- open(filename, 'w').write(str.rstrip() + "\n")
+ open(filename, 'w').write(str.rstrip() + "\n")
def read_keyval(path):
- """
- Read a key-value pair format file into a dictionary, and return it.
- Takes either a filename or directory name as input. If it's a
- directory name, we assume you want the file to be called keyval.
- """
- if os.path.isdir(path):
- path = os.path.join(path, 'keyval')
- keyval = {}
- for line in open(path):
- line = re.sub('#.*', '', line.rstrip())
- if not re.search(r'^[-\w]+=', line):
- raise ValueError('Invalid format line: %s' % line)
- key, value = line.split('=', 1)
- if re.search('^\d+$', value):
- value = int(value)
- elif re.search('^(\d+\.)?\d+$', value):
- value = float(value)
- keyval[key] = value
- return keyval
+ """
+ Read a key-value pair format file into a dictionary, and return it.
+ Takes either a filename or directory name as input. If it's a
+ directory name, we assume you want the file to be called keyval.
+ """
+ if os.path.isdir(path):
+ path = os.path.join(path, 'keyval')
+ keyval = {}
+ for line in open(path):
+ line = re.sub('#.*', '', line.rstrip())
+ if not re.search(r'^[-\w]+=', line):
+ raise ValueError('Invalid format line: %s' % line)
+ key, value = line.split('=', 1)
+ if re.search('^\d+$', value):
+ value = int(value)
+ elif re.search('^(\d+\.)?\d+$', value):
+ value = float(value)
+ keyval[key] = value
+ return keyval
def write_keyval(path, dictionary, type_tag=None):
- """
- Write a key-value pair format file out to a file. This uses append
- mode to open the file, so existing text will not be overwritten or
- reparsed.
+ """
+ Write a key-value pair format file out to a file. This uses append
+ mode to open the file, so existing text will not be overwritten or
+ reparsed.
- If type_tag is None, then the key must be composed of alphanumeric
- characters (or dashes+underscores). However, if type-tag is not
- null then the keys must also have "{type_tag}" as a suffix. At
- the moment the only valid values of type_tag are "attr" and "perf".
- """
- if os.path.isdir(path):
- path = os.path.join(path, 'keyval')
- keyval = open(path, 'a')
+ If type_tag is None, then the key must be composed of alphanumeric
+ characters (or dashes+underscores). However, if type-tag is not
+ null then the keys must also have "{type_tag}" as a suffix. At
+ the moment the only valid values of type_tag are "attr" and "perf".
+ """
+ if os.path.isdir(path):
+ path = os.path.join(path, 'keyval')
+ keyval = open(path, 'a')
- if type_tag is None:
- key_regex = re.compile(r'^[-\w]+$')
- else:
- if type_tag not in ('attr', 'perf'):
- raise ValueError('Invalid type tag: %s' % type_tag)
- escaped_tag = re.escape(type_tag)
- key_regex = re.compile(r'^[-\w]+\{%s\}$' % escaped_tag)
- try:
- for key, value in dictionary.iteritems():
- if not key_regex.search(key):
- raise ValueError('Invalid key: %s' % key)
- keyval.write('%s=%s\n' % (key, value))
- finally:
- keyval.close()
+ if type_tag is None:
+ key_regex = re.compile(r'^[-\w]+$')
+ else:
+ if type_tag not in ('attr', 'perf'):
+ raise ValueError('Invalid type tag: %s' % type_tag)
+ escaped_tag = re.escape(type_tag)
+ key_regex = re.compile(r'^[-\w]+\{%s\}$' % escaped_tag)
+ try:
+ for key, value in dictionary.iteritems():
+ if not key_regex.search(key):
+ raise ValueError('Invalid key: %s' % key)
+ keyval.write('%s=%s\n' % (key, value))
+ finally:
+ keyval.close()
def is_url(path):
- """Return true if path looks like a URL"""
- # for now, just handle http and ftp
- url_parts = urlparse.urlparse(path)
- return (url_parts[0] in ('http', 'ftp'))
+ """Return true if path looks like a URL"""
+ # for now, just handle http and ftp
+ url_parts = urlparse.urlparse(path)
+ return (url_parts[0] in ('http', 'ftp'))
def urlopen(url, data=None, proxies=None, timeout=300):
- """Wrapper to urllib.urlopen with timeout addition."""
+ """Wrapper to urllib.urlopen with timeout addition."""
- # Save old timeout
- old_timeout = socket.getdefaulttimeout()
- socket.setdefaulttimeout(timeout)
- try:
- return urllib.urlopen(url, data=data, proxies=proxies)
- finally:
- socket.setdefaulttimeout(old_timeout)
+ # Save old timeout
+ old_timeout = socket.getdefaulttimeout()
+ socket.setdefaulttimeout(timeout)
+ try:
+ return urllib.urlopen(url, data=data, proxies=proxies)
+ finally:
+ socket.setdefaulttimeout(old_timeout)
def urlretrieve(url, filename=None, reporthook=None, data=None, timeout=300):
- """Wrapper to urllib.urlretrieve with timeout addition."""
- old_timeout = socket.getdefaulttimeout()
- socket.setdefaulttimeout(timeout)
- try:
- return urllib.urlretrieve(url, filename=filename,
- reporthook=reporthook, data=data)
- finally:
- socket.setdefaulttimeout(old_timeout)
-
+ """Wrapper to urllib.urlretrieve with timeout addition."""
+ old_timeout = socket.getdefaulttimeout()
+ socket.setdefaulttimeout(timeout)
+ try:
+ return urllib.urlretrieve(url, filename=filename,
+ reporthook=reporthook, data=data)
+ finally:
+ socket.setdefaulttimeout(old_timeout)
+
def get_file(src, dest, permissions=None):
- """Get a file from src, which can be local or a remote URL"""
- if (src == dest):
- return
- if (is_url(src)):
- print 'PWD: ' + os.getcwd()
- print 'Fetching \n\t', src, '\n\t->', dest
- try:
- urllib.urlretrieve(src, dest)
- except IOError, e:
- raise error.AutotestError('Unable to retrieve %s (to %s)'
- % (src, dest), e)
- else:
- shutil.copyfile(src, dest)
- if permissions:
- os.chmod(dest, permissions)
- return dest
+ """Get a file from src, which can be local or a remote URL"""
+ if (src == dest):
+ return
+ if (is_url(src)):
+ print 'PWD: ' + os.getcwd()
+ print 'Fetching \n\t', src, '\n\t->', dest
+ try:
+ urllib.urlretrieve(src, dest)
+ except IOError, e:
+ raise error.AutotestError('Unable to retrieve %s (to %s)'
+ % (src, dest), e)
+ else:
+ shutil.copyfile(src, dest)
+ if permissions:
+ os.chmod(dest, permissions)
+ return dest
def unmap_url(srcdir, src, destdir='.'):
- """
- Receives either a path to a local file or a URL.
- returns either the path to the local file, or the fetched URL
+ """
+ Receives either a path to a local file or a URL.
+ returns either the path to the local file, or the fetched URL
- unmap_url('/usr/src', 'foo.tar', '/tmp')
- = '/usr/src/foo.tar'
- unmap_url('/usr/src', 'http://site/file', '/tmp')
- = '/tmp/file'
- (after retrieving it)
- """
- if is_url(src):
- url_parts = urlparse.urlparse(src)
- filename = os.path.basename(url_parts[2])
- dest = os.path.join(destdir, filename)
- return get_file(src, dest)
- else:
- return os.path.join(srcdir, src)
+ unmap_url('/usr/src', 'foo.tar', '/tmp')
+ = '/usr/src/foo.tar'
+ unmap_url('/usr/src', 'http://site/file', '/tmp')
+ = '/tmp/file'
+ (after retrieving it)
+ """
+ if is_url(src):
+ url_parts = urlparse.urlparse(src)
+ filename = os.path.basename(url_parts[2])
+ dest = os.path.join(destdir, filename)
+ return get_file(src, dest)
+ else:
+ return os.path.join(srcdir, src)
def update_version(srcdir, preserve_srcdir, new_version, install,
- *args, **dargs):
- """
- Make sure srcdir is version new_version
+ *args, **dargs):
+ """
+ Make sure srcdir is version new_version
- If not, delete it and install() the new version.
+ If not, delete it and install() the new version.
- In the preserve_srcdir case, we just check it's up to date,
- and if not, we rerun install, without removing srcdir
- """
- versionfile = os.path.join(srcdir, '.version')
- install_needed = True
+ In the preserve_srcdir case, we just check it's up to date,
+ and if not, we rerun install, without removing srcdir
+ """
+ versionfile = os.path.join(srcdir, '.version')
+ install_needed = True
- if os.path.exists(versionfile):
- old_version = pickle.load(open(versionfile))
- if old_version == new_version:
- install_needed = False
+ if os.path.exists(versionfile):
+ old_version = pickle.load(open(versionfile))
+ if old_version == new_version:
+ install_needed = False
- if install_needed:
- if not preserve_srcdir and os.path.exists(srcdir):
- shutil.rmtree(srcdir)
- install(*args, **dargs)
- if os.path.exists(srcdir):
- pickle.dump(new_version, open(versionfile, 'w'))
+ if install_needed:
+ if not preserve_srcdir and os.path.exists(srcdir):
+ shutil.rmtree(srcdir)
+ install(*args, **dargs)
+ if os.path.exists(srcdir):
+ pickle.dump(new_version, open(versionfile, 'w'))
def run(command, timeout=None, ignore_status=False,
- stdout_tee=None, stderr_tee=None):
- """
- Run a command on the host.
+ stdout_tee=None, stderr_tee=None):
+ """
+ Run a command on the host.
- Args:
- command: the command line string
- timeout: time limit in seconds before attempting to
- kill the running process. The run() function
- will take a few seconds longer than 'timeout'
- to complete if it has to kill the process.
- ignore_status: do not raise an exception, no matter what
- the exit code of the command is.
- stdout_tee: optional file-like object to which stdout data
- will be written as it is generated (data will still
- be stored in result.stdout)
- stderr_tee: likewise for stderr
+ Args:
+ command: the command line string
+ timeout: time limit in seconds before attempting to
+ kill the running process. The run() function
+ will take a few seconds longer than 'timeout'
+ to complete if it has to kill the process.
+ ignore_status: do not raise an exception, no matter what
+ the exit code of the command is.
+ stdout_tee: optional file-like object to which stdout data
+ will be written as it is generated (data will still
+ be stored in result.stdout)
+ stderr_tee: likewise for stderr
- Returns:
- a CmdResult object
+ Returns:
+ a CmdResult object
- Raises:
- CmdError: the exit code of the command
- execution was not 0
- """
- return join_bg_job(run_bg(command), command, timeout, ignore_status,
- stdout_tee, stderr_tee)
+ Raises:
+ CmdError: the exit code of the command
+ execution was not 0
+ """
+ return join_bg_job(run_bg(command), command, timeout, ignore_status,
+ stdout_tee, stderr_tee)
def run_bg(command):
- """Run the command in a subprocess and return the subprocess."""
- result = CmdResult(command)
- sp = subprocess.Popen(command, stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- shell=True, executable="/bin/bash")
- return sp, result
+ """Run the command in a subprocess and return the subprocess."""
+ result = CmdResult(command)
+ sp = subprocess.Popen(command, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ shell=True, executable="/bin/bash")
+ return sp, result
def join_bg_job(bg_job, command, timeout=None, ignore_status=False,
- stdout_tee=None, stderr_tee=None):
- """Join the subprocess with the current thread. See run description."""
- sp, result = bg_job
- stdout_file = StringIO.StringIO()
- stderr_file = StringIO.StringIO()
- (ret, timeouterr) = (0, False)
+ stdout_tee=None, stderr_tee=None):
+ """Join the subprocess with the current thread. See run description."""
+ sp, result = bg_job
+ stdout_file = StringIO.StringIO()
+ stderr_file = StringIO.StringIO()
+ (ret, timeouterr) = (0, False)
- try:
- # We are holding ends to stdin, stdout pipes
- # hence we need to be sure to close those fds no mater what
- start_time = time.time()
- (ret, timeouterr) = _wait_for_command(sp, start_time,
- timeout, stdout_file, stderr_file,
- stdout_tee, stderr_tee)
- result.exit_status = ret
- result.duration = time.time() - start_time
- # don't use os.read now, so we get all the rest of the output
- _process_output(sp.stdout, stdout_file, stdout_tee,
- use_os_read=False)
- _process_output(sp.stderr, stderr_file, stderr_tee,
- use_os_read=False)
- finally:
- # close our ends of the pipes to the sp no matter what
- sp.stdout.close()
- sp.stderr.close()
+ try:
+ # We are holding ends to stdin, stdout pipes
+ # hence we need to be sure to close those fds no mater what
+ start_time = time.time()
+ (ret, timeouterr) = _wait_for_command(sp, start_time,
+ timeout, stdout_file, stderr_file,
+ stdout_tee, stderr_tee)
+ result.exit_status = ret
+ result.duration = time.time() - start_time
+ # don't use os.read now, so we get all the rest of the output
+ _process_output(sp.stdout, stdout_file, stdout_tee,
+ use_os_read=False)
+ _process_output(sp.stderr, stderr_file, stderr_tee,
+ use_os_read=False)
+ finally:
+ # close our ends of the pipes to the sp no matter what
+ sp.stdout.close()
+ sp.stderr.close()
- result.stdout = stdout_file.getvalue()
- result.stderr = stderr_file.getvalue()
+ result.stdout = stdout_file.getvalue()
+ result.stderr = stderr_file.getvalue()
- if result.exit_status != 0:
- if timeouterr:
- raise error.CmdError(command, result, "Command did not "
- "complete within %d seconds" % timeout)
- elif not ignore_status:
- raise error.CmdError(command, result,
- "Command returned non-zero exit status")
+ if result.exit_status != 0:
+ if timeouterr:
+ raise error.CmdError(command, result, "Command did not "
+ "complete within %d seconds" % timeout)
+ elif not ignore_status:
+ raise error.CmdError(command, result,
+ "Command returned non-zero exit status")
- return result
+ return result
# this returns a tuple with the return code and a flag to specify if the error
# is due to the process not terminating within timeout
def _wait_for_command(subproc, start_time, timeout, stdout_file, stderr_file,
- stdout_tee, stderr_tee):
- if timeout:
- stop_time = start_time + timeout
- time_left = stop_time - time.time()
- else:
- time_left = None # so that select never times out
- while not timeout or time_left > 0:
- # select will return when stdout is ready (including when it is
- # EOF, that is the process has terminated).
- ready, _, _ = select.select([subproc.stdout, subproc.stderr],
- [], [], time_left)
- # os.read() has to be used instead of
- # subproc.stdout.read() which will otherwise block
- if subproc.stdout in ready:
- _process_output(subproc.stdout, stdout_file,
- stdout_tee)
- if subproc.stderr in ready:
- _process_output(subproc.stderr, stderr_file,
- stderr_tee)
+ stdout_tee, stderr_tee):
+ if timeout:
+ stop_time = start_time + timeout
+ time_left = stop_time - time.time()
+ else:
+ time_left = None # so that select never times out
+ while not timeout or time_left > 0:
+ # select will return when stdout is ready (including when it is
+ # EOF, that is the process has terminated).
+ ready, _, _ = select.select([subproc.stdout, subproc.stderr],
+ [], [], time_left)
+ # os.read() has to be used instead of
+ # subproc.stdout.read() which will otherwise block
+ if subproc.stdout in ready:
+ _process_output(subproc.stdout, stdout_file,
+ stdout_tee)
+ if subproc.stderr in ready:
+ _process_output(subproc.stderr, stderr_file,
+ stderr_tee)
- exit_status_indication = subproc.poll()
+ exit_status_indication = subproc.poll()
- if exit_status_indication is not None:
- return (exit_status_indication, False)
+ if exit_status_indication is not None:
+ return (exit_status_indication, False)
- if timeout:
- time_left = stop_time - time.time()
+ if timeout:
+ time_left = stop_time - time.time()
- # the process has not terminated within timeout,
- # kill it via an escalating series of signals.
- if exit_status_indication is None:
- exit_status_indication = nuke_subprocess(subproc)
+ # the process has not terminated within timeout,
+ # kill it via an escalating series of signals.
+ if exit_status_indication is None:
+ exit_status_indication = nuke_subprocess(subproc)
- return (exit_status_indication, True)
+ return (exit_status_indication, True)
def _process_output(pipe, fbuffer, teefile=None, use_os_read=True):
- if use_os_read:
- data = os.read(pipe.fileno(), 1024)
- else:
- data = pipe.read()
- fbuffer.write(data)
- if teefile:
- teefile.write(data)
- teefile.flush()
+ if use_os_read:
+ data = os.read(pipe.fileno(), 1024)
+ else:
+ data = pipe.read()
+ fbuffer.write(data)
+ if teefile:
+ teefile.write(data)
+ teefile.flush()
def nuke_subprocess(subproc):
- # the process has not terminated within timeout,
- # kill it via an escalating series of signals.
- signal_queue = [signal.SIGTERM, signal.SIGKILL]
- for sig in signal_queue:
- try:
- os.kill(subproc.pid, sig)
- # The process may have died before we could kill it.
- except OSError:
- pass
+ # the process has not terminated within timeout,
+ # kill it via an escalating series of signals.
+ signal_queue = [signal.SIGTERM, signal.SIGKILL]
+ for sig in signal_queue:
+ try:
+ os.kill(subproc.pid, sig)
+ # The process may have died before we could kill it.
+ except OSError:
+ pass
- for i in range(5):
- rc = subproc.poll()
- if rc != None:
- return rc
- time.sleep(1)
+ for i in range(5):
+ rc = subproc.poll()
+ if rc != None:
+ return rc
+ time.sleep(1)
def nuke_pid(pid):
- # the process has not terminated within timeout,
- # kill it via an escalating series of signals.
- signal_queue = [signal.SIGTERM, signal.SIGKILL]
- for sig in signal_queue:
- try:
- os.kill(pid, sig)
+ # the process has not terminated within timeout,
+ # kill it via an escalating series of signals.
+ signal_queue = [signal.SIGTERM, signal.SIGKILL]
+ for sig in signal_queue:
+ try:
+ os.kill(pid, sig)
- # The process may have died before we could kill it.
- except OSError:
- pass
+ # The process may have died before we could kill it.
+ except OSError:
+ pass
- try:
- for i in range(5):
- status = os.waitpid(pid, os.WNOHANG)[0]
- if status == pid:
- return
- time.sleep(1)
+ try:
+ for i in range(5):
+ status = os.waitpid(pid, os.WNOHANG)[0]
+ if status == pid:
+ return
+ time.sleep(1)
- if status != pid:
- raise error.AutoservRunError('Could not kill %d'
- % pid, None)
+ if status != pid:
+ raise error.AutoservRunError('Could not kill %d'
+ % pid, None)
- # the process died before we join it.
- except OSError:
- pass
+ # the process died before we join it.
+ except OSError:
+ pass
def _process_output(pipe, fbuffer, teefile=None, use_os_read=True):
- if use_os_read:
- data = os.read(pipe.fileno(), 1024)
- else:
- data = pipe.read()
- fbuffer.write(data)
- if teefile:
- teefile.write(data)
- teefile.flush()
+ if use_os_read:
+ data = os.read(pipe.fileno(), 1024)
+ else:
+ data = pipe.read()
+ fbuffer.write(data)
+ if teefile:
+ teefile.write(data)
+ teefile.flush()
def system(command, timeout=None, ignore_status=False):
- return run(command, timeout, ignore_status,
- stdout_tee=sys.stdout, stderr_tee=sys.stderr).exit_status
+ return run(command, timeout, ignore_status,
+ stdout_tee=sys.stdout, stderr_tee=sys.stderr).exit_status
def system_output(command, timeout=None, ignore_status=False,
- retain_output=False):
- if retain_output:
- out = run(command, timeout, ignore_status,
- stdout_tee=sys.stdout, stderr_tee=sys.stderr).stdout
- else:
- out = run(command, timeout, ignore_status).stdout
- if out[-1:] == '\n': out = out[:-1]
- return out
+ retain_output=False):
+ if retain_output:
+ out = run(command, timeout, ignore_status,
+ stdout_tee=sys.stdout, stderr_tee=sys.stderr).stdout
+ else:
+ out = run(command, timeout, ignore_status).stdout
+ if out[-1:] == '\n': out = out[:-1]
+ return out
"""
This function is used when there is a need to run more than one
@@ -400,134 +400,134 @@
"""
def get_sync_control_file(control, host_name, host_num,
- instance, num_jobs, port_base=63100):
- sc_bar_port = port_base
- c_bar_port = port_base
- if host_num < 0:
- print "Please provide a non negative number for the host"
- return None
- s_bar_port = port_base + 1 + host_num # The set of s_bar_ports are
- # the same for a given machine
+ instance, num_jobs, port_base=63100):
+ sc_bar_port = port_base
+ c_bar_port = port_base
+ if host_num < 0:
+ print "Please provide a non negative number for the host"
+ return None
+ s_bar_port = port_base + 1 + host_num # The set of s_bar_ports are
+ # the same for a given machine
- sc_bar_timeout = 180
- s_bar_timeout = c_bar_timeout = 120
+ sc_bar_timeout = 180
+ s_bar_timeout = c_bar_timeout = 120
- # The barrier code snippet is prepended into the conrol file
- # dynamically before at.run() is called finally.
- control_new = []
+ # The barrier code snippet is prepended into the conrol file
+ # dynamically before at.run() is called finally.
+ control_new = []
- # jobid is the unique name used to identify the processes
- # trying to reach the barriers
- jobid = "%s#%d" % (host_name, instance)
+ # jobid is the unique name used to identify the processes
+ # trying to reach the barriers
+ jobid = "%s#%d" % (host_name, instance)
- rendv = []
- # rendvstr is a temp holder for the rendezvous list of the processes
- for n in range(num_jobs):
- rendv.append("'%s#%d'" % (host_name, n))
- rendvstr = ",".join(rendv)
+ rendv = []
+ # rendvstr is a temp holder for the rendezvous list of the processes
+ for n in range(num_jobs):
+ rendv.append("'%s#%d'" % (host_name, n))
+ rendvstr = ",".join(rendv)
- if instance == 0:
- # Do the setup and wait at the server barrier
- # Clean up the tmp and the control dirs for the first instance
- control_new.append('if os.path.exists(job.tmpdir):')
- control_new.append("\t system('umount -f %s > /dev/null"
- "2> /dev/null' % job.tmpdir,"
- "ignore_status=True)")
- control_new.append("\t system('rm -rf ' + job.tmpdir)")
- control_new.append(
- 'b0 = job.barrier("%s", "sc_bar", %d, port=%d)'
- % (jobid, sc_bar_timeout, sc_bar_port))
- control_new.append(
- 'b0.rendevous_servers("PARALLEL_MASTER", "%s")'
- % jobid)
+ if instance == 0:
+ # Do the setup and wait at the server barrier
+ # Clean up the tmp and the control dirs for the first instance
+ control_new.append('if os.path.exists(job.tmpdir):')
+ control_new.append("\t system('umount -f %s > /dev/null"
+ "2> /dev/null' % job.tmpdir,"
+ "ignore_status=True)")
+ control_new.append("\t system('rm -rf ' + job.tmpdir)")
+ control_new.append(
+ 'b0 = job.barrier("%s", "sc_bar", %d, port=%d)'
+ % (jobid, sc_bar_timeout, sc_bar_port))
+ control_new.append(
+ 'b0.rendevous_servers("PARALLEL_MASTER", "%s")'
+ % jobid)
- elif instance == 1:
- # Wait at the server barrier to wait for instance=0
- # process to complete setup
- b0 = barrier.barrier("PARALLEL_MASTER", "sc_bar", sc_bar_timeout,
- port=sc_bar_port)
- b0.rendevous_servers("PARALLEL_MASTER", jobid)
+ elif instance == 1:
+ # Wait at the server barrier to wait for instance=0
+ # process to complete setup
+ b0 = barrier.barrier("PARALLEL_MASTER", "sc_bar", sc_bar_timeout,
+ port=sc_bar_port)
+ b0.rendevous_servers("PARALLEL_MASTER", jobid)
- if(num_jobs > 2):
- b1 = barrier.barrier(jobid, "s_bar", s_bar_timeout,
- port=s_bar_port)
- b1.rendevous(rendvstr)
+ if(num_jobs > 2):
+ b1 = barrier.barrier(jobid, "s_bar", s_bar_timeout,
+ port=s_bar_port)
+ b1.rendevous(rendvstr)
- else:
- # For the rest of the clients
- b2 = barrier.barrier(jobid, "s_bar", s_bar_timeout, port=s_bar_port)
- b2.rendevous(rendvstr)
+ else:
+ # For the rest of the clients
+ b2 = barrier.barrier(jobid, "s_bar", s_bar_timeout, port=s_bar_port)
+ b2.rendevous(rendvstr)
- # Client side barrier for all the tests to start at the same time
- control_new.append('b1 = job.barrier("%s", "c_bar", %d, port=%d)'
- % (jobid, c_bar_timeout, c_bar_port))
- control_new.append("b1.rendevous(%s)" % rendvstr)
+ # Client side barrier for all the tests to start at the same time
+ control_new.append('b1 = job.barrier("%s", "c_bar", %d, port=%d)'
+ % (jobid, c_bar_timeout, c_bar_port))
+ control_new.append("b1.rendevous(%s)" % rendvstr)
- # Stick in the rest of the control file
- control_new.append(control)
+ # Stick in the rest of the control file
+ control_new.append(control)
- return "\n".join(control_new)
+ return "\n".join(control_new)
class CmdResult(object):
- """
- Command execution result.
+ """
+ Command execution result.
- command: String containing the command line itself
- exit_status: Integer exit code of the process
- stdout: String containing stdout of the process
- stderr: String containing stderr of the process
- duration: Elapsed wall clock time running the process
- """
+ command: String containing the command line itself
+ exit_status: Integer exit code of the process
+ stdout: String containing stdout of the process
+ stderr: String containing stderr of the process
+ duration: Elapsed wall clock time running the process
+ """
- def __init__(self, command=None, stdout="", stderr="",
- exit_status=None, duration=0):
- self.command = command
- self.exit_status = exit_status
- self.stdout = stdout
- self.stderr = stderr
- self.duration = duration
+ def __init__(self, command=None, stdout="", stderr="",
+ exit_status=None, duration=0):
+ self.command = command
+ self.exit_status = exit_status
+ self.stdout = stdout
+ self.stderr = stderr
+ self.duration = duration
- def __repr__(self):
- wrapper = textwrap.TextWrapper(width = 78,
- initial_indent="\n ",
- subsequent_indent=" ")
-
- stdout = self.stdout.rstrip()
- if stdout:
- stdout = "\nstdout:\n%s" % stdout
-
- stderr = self.stderr.rstrip()
- if stderr:
- stderr = "\nstderr:\n%s" % stderr
-
- return ("* Command: %s\n"
- "Exit status: %s\n"
- "Duration: %s\n"
- "%s"
- "%s"
- % (wrapper.fill(self.command), self.exit_status,
- self.duration, stdout, stderr))
+ def __repr__(self):
+ wrapper = textwrap.TextWrapper(width = 78,
+ initial_indent="\n ",
+ subsequent_indent=" ")
+
+ stdout = self.stdout.rstrip()
+ if stdout:
+ stdout = "\nstdout:\n%s" % stdout
+
+ stderr = self.stderr.rstrip()
+ if stderr:
+ stderr = "\nstderr:\n%s" % stderr
+
+ return ("* Command: %s\n"
+ "Exit status: %s\n"
+ "Duration: %s\n"
+ "%s"
+ "%s"
+ % (wrapper.fill(self.command), self.exit_status,
+ self.duration, stdout, stderr))
class run_randomly:
- def __init__(self, run_sequentially=False):
- # Run sequentially is for debugging control files
- self.test_list = []
- self.run_sequentially = run_sequentially
+ def __init__(self, run_sequentially=False):
+ # Run sequentially is for debugging control files
+ self.test_list = []
+ self.run_sequentially = run_sequentially
- def add(self, *args, **dargs):
- test = (args, dargs)
- self.test_list.append(test)
+ def add(self, *args, **dargs):
+ test = (args, dargs)
+ self.test_list.append(test)
- def run(self, fn):
- while self.test_list:
- test_index = random.randint(0, len(self.test_list)-1)
- if self.run_sequentially:
- test_index = 0
- (args, dargs) = self.test_list.pop(test_index)
- fn(*args, **dargs)
+ def run(self, fn):
+ while self.test_list:
+ test_index = random.randint(0, len(self.test_list)-1)
+ if self.run_sequentially:
+ test_index = 0
+ (args, dargs) = self.test_list.pop(test_index)
+ fn(*args, **dargs)
diff --git a/client/deps/boottool/boottool.py b/client/deps/boottool/boottool.py
index e801357..43e5cb2 100755
--- a/client/deps/boottool/boottool.py
+++ b/client/deps/boottool/boottool.py
@@ -7,21 +7,21 @@
from autotest_lib.client.bin import autotest_utils
# To use this, you have to set PERL5LIB to:
-# autodir+'deps/boottool/lib/perl'
+# autodir+'deps/boottool/lib/perl'
# or on Ubuntu we also need
-# autodir+'deps/boottool/share/perl'
+# autodir+'deps/boottool/share/perl'
# because it uses nonstandard locations
version = 1
-def setup(tarball, topdir):
- srcdir = os.path.join(topdir, 'src')
- autotest_utils.extract_tarball_to_dir(tarball, srcdir)
- os.chdir(srcdir)
- utils.system ('perl Makefile.PL PREFIX=' + topdir)
- utils.system ('make')
- utils.system ('make install')
- os.chdir(topdir)
+def setup(tarball, topdir):
+ srcdir = os.path.join(topdir, 'src')
+ autotest_utils.extract_tarball_to_dir(tarball, srcdir)
+ os.chdir(srcdir)
+ utils.system ('perl Makefile.PL PREFIX=' + topdir)
+ utils.system ('make')
+ utils.system ('make install')
+ os.chdir(topdir)
pwd = os.getcwd()
diff --git a/client/deps/libaio/libaio.py b/client/deps/libaio/libaio.py
index cfb037d..e673678 100755
--- a/client/deps/libaio/libaio.py
+++ b/client/deps/libaio/libaio.py
@@ -9,12 +9,12 @@
version = 1
def setup(tarball, topdir):
- srcdir = os.path.join(topdir, 'src')
- autotest_utils.extract_tarball_to_dir(tarball, srcdir)
- os.chdir(srcdir)
- utils.system ('make')
- utils.system ('make prefix=%s install' % topdir)
- os.chdir(topdir)
+ srcdir = os.path.join(topdir, 'src')
+ autotest_utils.extract_tarball_to_dir(tarball, srcdir)
+ os.chdir(srcdir)
+ utils.system ('make')
+ utils.system ('make prefix=%s install' % topdir)
+ os.chdir(topdir)
# old source was
diff --git a/client/deps/libnet/libnet.py b/client/deps/libnet/libnet.py
index eac66b2..90ab023 100755
--- a/client/deps/libnet/libnet.py
+++ b/client/deps/libnet/libnet.py
@@ -8,19 +8,19 @@
version = 1
-def setup(tarball, topdir):
- srcdir = os.path.join(topdir, 'src')
- if not os.path.exists(tarball):
- utils.get_file('http://www.packetfactory.net/libnet/dist/libnet.tar.gz',
- tarball)
- autotest_utils.extract_tarball_to_dir(tarball, 'src')
- os.chdir(srcdir)
- utils.system ('./configure --prefix=%s/libnet' % topdir)
- utils.system('make')
- utils.system('make install')
+def setup(tarball, topdir):
+ srcdir = os.path.join(topdir, 'src')
+ if not os.path.exists(tarball):
+ utils.get_file('http://www.packetfactory.net/libnet/dist/libnet.tar.gz',
+ tarball)
+ autotest_utils.extract_tarball_to_dir(tarball, 'src')
+ os.chdir(srcdir)
+ utils.system ('./configure --prefix=%s/libnet' % topdir)
+ utils.system('make')
+ utils.system('make install')
- os.chdir(topdir)
-
+ os.chdir(topdir)
+
pwd = os.getcwd()
tarball = os.path.join(pwd, 'libnet.tar.gz')
utils.update_version(pwd+'/src', False, version, setup, tarball, pwd)
diff --git a/client/deps/mysql/mysql.py b/client/deps/mysql/mysql.py
index 0fafe27..9a9eeb6 100755
--- a/client/deps/mysql/mysql.py
+++ b/client/deps/mysql/mysql.py
@@ -8,32 +8,32 @@
version = 3
-def setup(tarball, topdir):
- srcdir = os.path.join(topdir, 'src')
- if not os.path.exists(tarball):
- utils.get_file('http://mirror.x10.com/mirror/mysql/Downloads/MySQL-5.0/mysql-5.0.45.tar.gz', tarball)
- autotest_utils.extract_tarball_to_dir(tarball, 'src')
- os.chdir(srcdir)
- utils.system ('./configure --prefix=%s/mysql --enable-thread-safe-client' \
- % topdir)
- utils.system('make -j %d' % count_cpus())
- utils.system('make install')
+def setup(tarball, topdir):
+ srcdir = os.path.join(topdir, 'src')
+ if not os.path.exists(tarball):
+ utils.get_file('http://mirror.x10.com/mirror/mysql/Downloads/MySQL-5.0/mysql-5.0.45.tar.gz', tarball)
+ autotest_utils.extract_tarball_to_dir(tarball, 'src')
+ os.chdir(srcdir)
+ utils.system ('./configure --prefix=%s/mysql --enable-thread-safe-client' \
+ % topdir)
+ utils.system('make -j %d' % count_cpus())
+ utils.system('make install')
- #
- # MySQL doesn't create this directory on it's own.
- # This is where database logs and files are created.
- #
- try:
- os.mkdir(topdir + '/mysql/var')
- except:
- pass
- #
- # Initialize the database.
- #
- utils.system('%s/mysql/bin/mysql_install_db' % topdir)
-
- os.chdir(topdir)
-
+ #
+ # MySQL doesn't create this directory on it's own.
+ # This is where database logs and files are created.
+ #
+ try:
+ os.mkdir(topdir + '/mysql/var')
+ except:
+ pass
+ #
+ # Initialize the database.
+ #
+ utils.system('%s/mysql/bin/mysql_install_db' % topdir)
+
+ os.chdir(topdir)
+
pwd = os.getcwd()
tarball = os.path.join(pwd, 'mysql-5.0.45.tar.gz')
utils.update_version(pwd+'/src', False, version, setup, tarball, pwd)
diff --git a/client/deps/pgpool/pgpool.py b/client/deps/pgpool/pgpool.py
index 3034b6a..24afc26 100755
--- a/client/deps/pgpool/pgpool.py
+++ b/client/deps/pgpool/pgpool.py
@@ -8,23 +8,23 @@
version = 1
-def setup(tarball, topdir):
- # FIXME - Waiting to be able to specify dependency.
- #self.job.setup_dep(['pgsql'])
- srcdir = os.path.join(topdir, 'src')
- if not os.path.exists(tarball):
- utils.get_file('http://pgfoundry.org/frs/download.php/1083/pgpool-II-1.0.1.tar.gz', tarball)
- autotest_utils.extract_tarball_to_dir(tarball, 'src')
- os.chdir(srcdir)
- # FIXEME - Waiting to be able to use self.autodir instead of
- # os.environ['AUTODIR']
- utils.system('./configure --prefix=%s/pgpool --with-pgsql=%s/deps/pgsql/pgsql' \
- % (topdir, os.environ['AUTODIR']))
- utils.system('make -j %d' % count_cpus())
- utils.system('make install')
+def setup(tarball, topdir):
+ # FIXME - Waiting to be able to specify dependency.
+ #self.job.setup_dep(['pgsql'])
+ srcdir = os.path.join(topdir, 'src')
+ if not os.path.exists(tarball):
+ utils.get_file('http://pgfoundry.org/frs/download.php/1083/pgpool-II-1.0.1.tar.gz', tarball)
+ autotest_utils.extract_tarball_to_dir(tarball, 'src')
+ os.chdir(srcdir)
+ # FIXEME - Waiting to be able to use self.autodir instead of
+ # os.environ['AUTODIR']
+ utils.system('./configure --prefix=%s/pgpool --with-pgsql=%s/deps/pgsql/pgsql' \
+ % (topdir, os.environ['AUTODIR']))
+ utils.system('make -j %d' % count_cpus())
+ utils.system('make install')
- os.chdir(topdir)
-
+ os.chdir(topdir)
+
pwd = os.getcwd()
tarball = os.path.join(pwd, 'pgpool-II-1.0.1.tar.gz')
utils.update_version(pwd+'/src', False, version, setup, tarball, pwd)
diff --git a/client/deps/pgsql/pgsql.py b/client/deps/pgsql/pgsql.py
index 1de96b8..cd53653 100755
--- a/client/deps/pgsql/pgsql.py
+++ b/client/deps/pgsql/pgsql.py
@@ -8,18 +8,18 @@
version = 4
-def setup(tarball, topdir):
- srcdir = os.path.join(topdir, 'src')
- if not os.path.exists(tarball):
- utils.get_file('ftp://ftp.postgresql.org/pub/source/v8.3.1/postgresql-8.3.1.tar.bz2', tarball)
- autotest_utils.extract_tarball_to_dir(tarball, 'src')
- os.chdir(srcdir)
- utils.system ('./configure --without-readline --without-zlib --enable-debug --prefix=%s/pgsql' % topdir)
- utils.system('make -j %d' % count_cpus())
- utils.system('make install')
-
- os.chdir(topdir)
-
+def setup(tarball, topdir):
+ srcdir = os.path.join(topdir, 'src')
+ if not os.path.exists(tarball):
+ utils.get_file('ftp://ftp.postgresql.org/pub/source/v8.3.1/postgresql-8.3.1.tar.bz2', tarball)
+ autotest_utils.extract_tarball_to_dir(tarball, 'src')
+ os.chdir(srcdir)
+ utils.system ('./configure --without-readline --without-zlib --enable-debug --prefix=%s/pgsql' % topdir)
+ utils.system('make -j %d' % count_cpus())
+ utils.system('make install')
+
+ os.chdir(topdir)
+
pwd = os.getcwd()
tarball = os.path.join(pwd, 'postgresql-8.3.1.tar.bz2')
utils.update_version(pwd+'/src', False, version, setup, tarball, pwd)
diff --git a/client/profilers/catprofile/catprofile.py b/client/profilers/catprofile/catprofile.py
index 30167fc..5ccc835 100755
--- a/client/profilers/catprofile/catprofile.py
+++ b/client/profilers/catprofile/catprofile.py
@@ -3,41 +3,40 @@
import profiler,time,os
class catprofile(profiler.profiler):
- version = 1
+ version = 1
- # filenames: list of filenames to cat
- def initialize(self, filenames = ['/proc/meminfo', '/proc/slabinfo'],
- outfile = 'monitor', interval = 1):
- self.filenames = filenames
- self.outfile = outfile
- self.interval = interval
+ # filenames: list of filenames to cat
+ def initialize(self, filenames = ['/proc/meminfo', '/proc/slabinfo'],
+ outfile = 'monitor', interval = 1):
+ self.filenames = filenames
+ self.outfile = outfile
+ self.interval = interval
- def start(self, test):
- self.child_pid = os.fork()
- if self.child_pid: # parent
- return None
- else: # child
- while 1:
- lines = []
- for filename in self.filenames:
- input = open(filename, 'r')
- lines += '\n----- %s -----\n' % filename
- lines += input.readlines()
- input.close
- outfile = test.profdir + '/' + self.outfile
- output = open(outfile, 'a')
- output.write(time.asctime() + '\n')
- output.writelines(lines)
- output.write('\n=========================\n')
- output.close()
- time.sleep(self.interval)
+ def start(self, test):
+ self.child_pid = os.fork()
+ if self.child_pid: # parent
+ return None
+ else: # child
+ while 1:
+ lines = []
+ for filename in self.filenames:
+ input = open(filename, 'r')
+ lines += '\n----- %s -----\n' % filename
+ lines += input.readlines()
+ input.close
+ outfile = test.profdir + '/' + self.outfile
+ output = open(outfile, 'a')
+ output.write(time.asctime() + '\n')
+ output.writelines(lines)
+ output.write('\n=========================\n')
+ output.close()
+ time.sleep(self.interval)
- def stop(self, test):
- os.kill(self.child_pid, 15)
+ def stop(self, test):
+ os.kill(self.child_pid, 15)
- def report(self, test):
- return None
-
+ def report(self, test):
+ return None
diff --git a/client/profilers/iostat/iostat.py b/client/profilers/iostat/iostat.py
index 11fb24e..a258c1f 100755
--- a/client/profilers/iostat/iostat.py
+++ b/client/profilers/iostat/iostat.py
@@ -3,24 +3,23 @@
import profiler,time,os,subprocess
class iostat(profiler.profiler):
- version = 1
+ version = 1
- def initialize(self, interval = 1):
- self.interval = interval
+ def initialize(self, interval = 1):
+ self.interval = interval
- def start(self, test):
- cmd = "/usr/bin/iostat %d" % self.interval
- logfile = open(os.path.join(test.profdir, "iostat"), 'w')
- p = subprocess.Popen(cmd, shell=True, stdout=logfile, \
- stderr=subprocess.STDOUT)
- self.pid = p.pid
+ def start(self, test):
+ cmd = "/usr/bin/iostat %d" % self.interval
+ logfile = open(os.path.join(test.profdir, "iostat"), 'w')
+ p = subprocess.Popen(cmd, shell=True, stdout=logfile, \
+ stderr=subprocess.STDOUT)
+ self.pid = p.pid
- def stop(self, test):
- os.kill(self.pid, 15)
+ def stop(self, test):
+ os.kill(self.pid, 15)
- def report(self, test):
- return None
-
+ def report(self, test):
+ return None
diff --git a/client/profilers/lockmeter/lockmeter.py b/client/profilers/lockmeter/lockmeter.py
index bea3cc6..8c3d014 100755
--- a/client/profilers/lockmeter/lockmeter.py
+++ b/client/profilers/lockmeter/lockmeter.py
@@ -1,47 +1,47 @@
# NOTE: if you get compile errors from config.h, referring you to a FAQ,
-# you might need to do 'cat < /dev/null > /usr/include/linux/config.h'.
+# you might need to do 'cat < /dev/null > /usr/include/linux/config.h'.
# But read the FAQ first.
import profiler
from autotest_lib.client.common_lib import utils
from autotest_lib.client.bin import autotest_utils
class lockmeter(profiler.profiler):
- version = 1
+ version = 1
# ftp://oss.sgi.com/projects/lockmeter/download/lockstat-1.4.11.tar.gz
# patched with lockstat.diff
# ftp://oss.sgi.com/projects/lockmeter/download/v2.6/patch.2.6.14-lockmeter-1.gz
# is the kernel patch
- def setup(self, tarball = 'lockstat-1.4.11.tar.bz2'):
- self.tarball = utils.unmap_url(self.bindir, tarball, self.tmpdir)
- autotest_utils.extract_tarball_to_dir(self.tarball, self.srcdir)
- os.chdir(self.srcdir)
+ def setup(self, tarball = 'lockstat-1.4.11.tar.bz2'):
+ self.tarball = utils.unmap_url(self.bindir, tarball, self.tmpdir)
+ autotest_utils.extract_tarball_to_dir(self.tarball, self.srcdir)
+ os.chdir(self.srcdir)
- utils.system('make')
- self.cmd = self.srcdir + '/lockstat'
+ utils.system('make')
+ self.cmd = self.srcdir + '/lockstat'
- def initialize(self):
- try:
- assert os.path.exists('/proc/lockmeter')
- except:
- print 'Lockmeter is not compiled into your kernel'
- print 'Please fix and try again'
- raise AssertionError
+ def initialize(self):
+ try:
+ assert os.path.exists('/proc/lockmeter')
+ except:
+ print 'Lockmeter is not compiled into your kernel'
+ print 'Please fix and try again'
+ raise AssertionError
- def start(self, test):
- utils.system(self.cmd + ' off')
- utils.system(self.cmd + ' reset')
- utils.system(self.cmd + ' on')
+ def start(self, test):
+ utils.system(self.cmd + ' off')
+ utils.system(self.cmd + ' reset')
+ utils.system(self.cmd + ' on')
- def stop(self, test):
- utils.system(self.cmd + ' off')
+ def stop(self, test):
+ utils.system(self.cmd + ' off')
- def report(self, test):
- args = ' -m ' + autotest_utils.get_systemmap()
- self.output = self.profdir + '/results/lockstat'
- utils.system(self.cmd + args + ' print > ' + self.output)
+ def report(self, test):
+ args = ' -m ' + autotest_utils.get_systemmap()
+ self.output = self.profdir + '/results/lockstat'
+ utils.system(self.cmd + args + ' print > ' + self.output)
diff --git a/client/profilers/mpstat/mpstat.py b/client/profilers/mpstat/mpstat.py
index 261bd25..5870c9c 100644
--- a/client/profilers/mpstat/mpstat.py
+++ b/client/profilers/mpstat/mpstat.py
@@ -2,24 +2,24 @@
import profiler,time,os,subprocess
class mpstat(profiler.profiler):
- version = 1
+ version = 1
- def initialize(self, interval = 1):
- self.interval = interval
+ def initialize(self, interval = 1):
+ self.interval = interval
- def start(self, test):
- cmd = "mpstat -P ALL %d" % self.interval
- logfile = open(os.path.join(test.profdir, "mpstat"), 'w')
- p = subprocess.Popen(cmd, shell=True, stdout=logfile,
- stderr=subprocess.STDOUT)
- self.pid = p.pid
+ def start(self, test):
+ cmd = "mpstat -P ALL %d" % self.interval
+ logfile = open(os.path.join(test.profdir, "mpstat"), 'w')
+ p = subprocess.Popen(cmd, shell=True, stdout=logfile,
+ stderr=subprocess.STDOUT)
+ self.pid = p.pid
- def stop(self, test):
- os.kill(self.pid, 15)
+ def stop(self, test):
+ os.kill(self.pid, 15)
- def report(self, test):
- return None
+ def report(self, test):
+ return None
diff --git a/client/profilers/oprofile/oprofile.py b/client/profilers/oprofile/oprofile.py
index a36989a..8f55736 100755
--- a/client/profilers/oprofile/oprofile.py
+++ b/client/profilers/oprofile/oprofile.py
@@ -1,112 +1,112 @@
-# Will need some libaries to compile. Do 'apt-get build-dep oprofile'
+# Will need some libaries to compile. Do 'apt-get build-dep oprofile'
import profiler, shutil
from autotest_lib.client.common_lib import utils
from autotest_lib.client.bin import autotest_utils
class oprofile(profiler.profiler):
- version = 5
+ version = 5
# Notes on whether to use the local copy or the builtin from source:
# local = None
# Try to use source copy if it works, else use local
# local = False
-# Force use of the source copy
+# Force use of the source copy
# local = True
-# Force use of the local copy
+# Force use of the local copy
# http://prdownloads.sourceforge.net/oprofile/oprofile-0.9.3.tar.gz
- def setup(self, tarball = 'oprofile-0.9.3.tar.bz2', local = None,
- *args, **dargs):
- if local == True:
- return
+ def setup(self, tarball = 'oprofile-0.9.3.tar.bz2', local = None,
+ *args, **dargs):
+ if local == True:
+ return
- try:
- self.tarball = utils.unmap_url(self.bindir, tarball,
- self.tmpdir)
- autotest_utils.extract_tarball_to_dir(self.tarball, self.srcdir)
- os.chdir(self.srcdir)
+ try:
+ self.tarball = utils.unmap_url(self.bindir, tarball,
+ self.tmpdir)
+ autotest_utils.extract_tarball_to_dir(self.tarball, self.srcdir)
+ os.chdir(self.srcdir)
- patch = os.path.join(self.bindir,"oprofile-69455.patch")
- utils.system('patch -p1 < %s' % patch)
- utils.system('./configure --with-kernel-support --prefix=' + \
- self.srcdir)
- utils.system('make')
- utils.system('make install')
- except:
- # Build from source failed.
- # But maybe can still use the local copy
- local_opcontrol = os.path.exists('/usr/bin/opcontrol')
- local_opreport = os.path.exists('/usr/bin/opreport')
- if local == False or not local_opcontrol or not local_opreport:
- raise
+ patch = os.path.join(self.bindir,"oprofile-69455.patch")
+ utils.system('patch -p1 < %s' % patch)
+ utils.system('./configure --with-kernel-support --prefix=' + \
+ self.srcdir)
+ utils.system('make')
+ utils.system('make install')
+ except:
+ # Build from source failed.
+ # But maybe can still use the local copy
+ local_opcontrol = os.path.exists('/usr/bin/opcontrol')
+ local_opreport = os.path.exists('/usr/bin/opreport')
+ if local == False or not local_opcontrol or not local_opreport:
+ raise
- def initialize(self, vmlinux = None, events = [], others = None,
- local = None):
- if not vmlinux:
- self.vmlinux = get_vmlinux()
- else:
- self.vmlinux = vmlinux
- if not len(events):
- self.events = ['default']
- else:
- self.events = events
- self.others = others
+ def initialize(self, vmlinux = None, events = [], others = None,
+ local = None):
+ if not vmlinux:
+ self.vmlinux = get_vmlinux()
+ else:
+ self.vmlinux = vmlinux
+ if not len(events):
+ self.events = ['default']
+ else:
+ self.events = events
+ self.others = others
- # If there is existing setup file, oprofile may fail to start with default parameters.
- if os.path.isfile('/root/.oprofile/daemonrc'):
- os.rename('/root/.oprofile/daemonrc', '/root/.oprofile/daemonrc.org')
+ # If there is existing setup file, oprofile may fail to start with default parameters.
+ if os.path.isfile('/root/.oprofile/daemonrc'):
+ os.rename('/root/.oprofile/daemonrc', '/root/.oprofile/daemonrc.org')
- setup = ' --setup'
- if not self.vmlinux:
- setup += ' --no-vmlinux'
- else:
- setup += ' --vmlinux=%s' % self.vmlinux
- for e in self.events:
- setup += ' --event=%s' % e
- if self.others:
- setup += ' ' + self.others
+ setup = ' --setup'
+ if not self.vmlinux:
+ setup += ' --no-vmlinux'
+ else:
+ setup += ' --vmlinux=%s' % self.vmlinux
+ for e in self.events:
+ setup += ' --event=%s' % e
+ if self.others:
+ setup += ' ' + self.others
- src_opreport = os.path.join(self.srcdir, '/bin/opreport')
- src_opcontrol = os.path.join(self.srcdir, '/bin/opcontrol')
- if local == False or (local == None and
- os.path.exists(src_opreport) and
- os.path.exists(src_opcontrol)):
- print "Using source-built copy of oprofile"
- self.opreport = src_opreport
- self.opcontrol = src_opcontrol
- else:
- print "Using machine local copy of oprofile"
- self.opreport = '/usr/bin/opreport'
- self.opcontrol = '/usr/bin/opcontrol'
+ src_opreport = os.path.join(self.srcdir, '/bin/opreport')
+ src_opcontrol = os.path.join(self.srcdir, '/bin/opcontrol')
+ if local == False or (local == None and
+ os.path.exists(src_opreport) and
+ os.path.exists(src_opcontrol)):
+ print "Using source-built copy of oprofile"
+ self.opreport = src_opreport
+ self.opcontrol = src_opcontrol
+ else:
+ print "Using machine local copy of oprofile"
+ self.opreport = '/usr/bin/opreport'
+ self.opcontrol = '/usr/bin/opcontrol'
- utils.system(self.opcontrol + setup)
+ utils.system(self.opcontrol + setup)
- def start(self, test):
- utils.system(self.opcontrol + ' --shutdown')
- utils.system(self.opcontrol + ' --reset')
- utils.system(self.opcontrol + ' --start')
+ def start(self, test):
+ utils.system(self.opcontrol + ' --shutdown')
+ utils.system(self.opcontrol + ' --reset')
+ utils.system(self.opcontrol + ' --start')
- def stop(self, test):
- utils.system(self.opcontrol + ' --stop')
- utils.system(self.opcontrol + ' --dump')
+ def stop(self, test):
+ utils.system(self.opcontrol + ' --stop')
+ utils.system(self.opcontrol + ' --dump')
- def report(self, test):
- # Output kernel per-symbol profile report
- reportfile = test.profdir + '/oprofile.kernel'
- if self.vmlinux:
- report = self.opreport + ' -l ' + self.vmlinux
- if os.path.exists(get_modules_dir()):
- report += ' -p ' + get_modules_dir()
- utils.system(report + ' > ' + reportfile)
- else:
- utils.system("echo 'no vmlinux found.' > %s" %reportfile)
+ def report(self, test):
+ # Output kernel per-symbol profile report
+ reportfile = test.profdir + '/oprofile.kernel'
+ if self.vmlinux:
+ report = self.opreport + ' -l ' + self.vmlinux
+ if os.path.exists(get_modules_dir()):
+ report += ' -p ' + get_modules_dir()
+ utils.system(report + ' > ' + reportfile)
+ else:
+ utils.system("echo 'no vmlinux found.' > %s" %reportfile)
- # output profile summary report
- reportfile = test.profdir + '/oprofile.user'
- utils.system(self.opreport + ' --long-filenames ' + ' > ' + reportfile)
+ # output profile summary report
+ reportfile = test.profdir + '/oprofile.user'
+ utils.system(self.opreport + ' --long-filenames ' + ' > ' + reportfile)
- utils.system(self.opcontrol + ' --shutdown')
+ utils.system(self.opcontrol + ' --shutdown')
diff --git a/client/profilers/readprofile/readprofile.py b/client/profilers/readprofile/readprofile.py
index 95b1c60..8ac1c37 100755
--- a/client/profilers/readprofile/readprofile.py
+++ b/client/profilers/readprofile/readprofile.py
@@ -3,45 +3,45 @@
from autotest_lib.client.bin import autotest_utils
class readprofile(profiler.profiler):
- version = 1
+ version = 1
# http://www.kernel.org/pub/linux/utils/util-linux/util-linux-2.12r.tar.bz2
- def setup(self, tarball = 'util-linux-2.12r.tar.bz2'):
- self.tarball = utils.unmap_url(self.bindir, tarball, self.tmpdir)
- autotest_utils.extract_tarball_to_dir(self.tarball, self.srcdir)
- os.chdir(self.srcdir)
+ def setup(self, tarball = 'util-linux-2.12r.tar.bz2'):
+ self.tarball = utils.unmap_url(self.bindir, tarball, self.tmpdir)
+ autotest_utils.extract_tarball_to_dir(self.tarball, self.srcdir)
+ os.chdir(self.srcdir)
- utils.system('./configure')
- os.chdir('sys-utils')
- utils.system('make readprofile')
+ utils.system('./configure')
+ os.chdir('sys-utils')
+ utils.system('make readprofile')
- def initialize(self):
- try:
- utils.system('grep -iq " profile=" /proc/cmdline')
- except CmdError:
- raise AutotestError('readprofile not enabled')
+ def initialize(self):
+ try:
+ utils.system('grep -iq " profile=" /proc/cmdline')
+ except CmdError:
+ raise AutotestError('readprofile not enabled')
- self.cmd = self.srcdir + '/sys-utils/readprofile'
+ self.cmd = self.srcdir + '/sys-utils/readprofile'
- def start(self, test):
- utils.system(self.cmd + ' -r')
+ def start(self, test):
+ utils.system(self.cmd + ' -r')
- def stop(self, test):
- # There's no real way to stop readprofile, so we stash the
- # raw data at this point instead. BAD EXAMPLE TO COPY! ;-)
- self.rawprofile = test.profdir + '/profile.raw'
- print "STOP"
- shutil.copyfile('/proc/profile', self.rawprofile)
+ def stop(self, test):
+ # There's no real way to stop readprofile, so we stash the
+ # raw data at this point instead. BAD EXAMPLE TO COPY! ;-)
+ self.rawprofile = test.profdir + '/profile.raw'
+ print "STOP"
+ shutil.copyfile('/proc/profile', self.rawprofile)
- def report(self, test):
- args = ' -n'
- args += ' -m ' + get_systemmap()
- args += ' -p ' + self.rawprofile
- cmd = self.cmd + ' ' + args
- txtprofile = test.profdir + '/profile.text'
- utils.system(cmd + ' | sort -nr > ' + txtprofile)
- utils.system('bzip2 ' + self.rawprofile)
+ def report(self, test):
+ args = ' -n'
+ args += ' -m ' + get_systemmap()
+ args += ' -p ' + self.rawprofile
+ cmd = self.cmd + ' ' + args
+ txtprofile = test.profdir + '/profile.text'
+ utils.system(cmd + ' | sort -nr > ' + txtprofile)
+ utils.system('bzip2 ' + self.rawprofile)
diff --git a/client/profilers/vmstat/vmstat.py b/client/profilers/vmstat/vmstat.py
index 632cc34..b381649 100755
--- a/client/profilers/vmstat/vmstat.py
+++ b/client/profilers/vmstat/vmstat.py
@@ -3,24 +3,23 @@
import profiler,time,os,subprocess
class vmstat(profiler.profiler):
- version = 1
+ version = 1
- def initialize(self, interval = 1):
- self.interval = interval
+ def initialize(self, interval = 1):
+ self.interval = interval
- def start(self, test):
- cmd = "/usr/bin/vmstat %d" % self.interval
- logfile = open(os.path.join(test.profdir, "vmstat"), 'w')
- p = subprocess.Popen(cmd, shell=True, stdout=logfile, \
- stderr=subprocess.STDOUT)
- self.pid = p.pid
+ def start(self, test):
+ cmd = "/usr/bin/vmstat %d" % self.interval
+ logfile = open(os.path.join(test.profdir, "vmstat"), 'w')
+ p = subprocess.Popen(cmd, shell=True, stdout=logfile, \
+ stderr=subprocess.STDOUT)
+ self.pid = p.pid
- def stop(self, test):
- os.kill(self.pid, 15)
+ def stop(self, test):
+ os.kill(self.pid, 15)
- def report(self, test):
- return None
-
+ def report(self, test):
+ return None
diff --git a/client/setup_modules.py b/client/setup_modules.py
index 4e2185e..f2de48c 100644
--- a/client/setup_modules.py
+++ b/client/setup_modules.py
@@ -4,95 +4,95 @@
def _create_module(name):
- """Create a single top-level module"""
- module = new.module(name)
- sys.modules[name] = module
- return module
+ """Create a single top-level module"""
+ module = new.module(name)
+ sys.modules[name] = module
+ return module
def _create_module_and_parents(name):
- """Create a module, and all the necessary parents"""
- parts = name.split(".")
- # first create the top-level module
- parent = _create_module(parts[0])
- created_parts = [parts[0]]
- parts.pop(0)
- # now, create any remaining child modules
- while parts:
- child_name = parts.pop(0)
- module = new.module(child_name)
- setattr(parent, child_name, module)
- created_parts.append(child_name)
- sys.modules[".".join(created_parts)] = module
- parent = module
+ """Create a module, and all the necessary parents"""
+ parts = name.split(".")
+ # first create the top-level module
+ parent = _create_module(parts[0])
+ created_parts = [parts[0]]
+ parts.pop(0)
+ # now, create any remaining child modules
+ while parts:
+ child_name = parts.pop(0)
+ module = new.module(child_name)
+ setattr(parent, child_name, module)
+ created_parts.append(child_name)
+ sys.modules[".".join(created_parts)] = module
+ parent = module
def _import_children_into_module(parent_module_name, path):
- """Import all the packages on a path into a parent module"""
- # find all the packages at 'path'
- names = []
- for filename in os.listdir(path):
- full_name = os.path.join(path, filename)
- if not os.path.isdir(full_name):
- continue # skip files
- if "__init__.py" in os.listdir(full_name):
- names.append(filename)
- # import all the packages and insert them into 'parent_module'
- sys.path.insert(0, path)
- for name in names:
- module = __import__(name)
- # add the package to the parent
- parent_module = sys.modules[parent_module_name]
- setattr(parent_module, name, module)
- full_name = parent_module_name + "." + name
- sys.modules[full_name] = module
- # restore the system path
- sys.path.pop(0)
+ """Import all the packages on a path into a parent module"""
+ # find all the packages at 'path'
+ names = []
+ for filename in os.listdir(path):
+ full_name = os.path.join(path, filename)
+ if not os.path.isdir(full_name):
+ continue # skip files
+ if "__init__.py" in os.listdir(full_name):
+ names.append(filename)
+ # import all the packages and insert them into 'parent_module'
+ sys.path.insert(0, path)
+ for name in names:
+ module = __import__(name)
+ # add the package to the parent
+ parent_module = sys.modules[parent_module_name]
+ setattr(parent_module, name, module)
+ full_name = parent_module_name + "." + name
+ sys.modules[full_name] = module
+ # restore the system path
+ sys.path.pop(0)
def _setup_common_library(root_module_name):
- """
- Setup aliases for all the common libraries, e.g.
- common -> autotest_lib.client.common_lib
- common.error -> autotest_lib.client.common_lib.error
- """
- # convert the root_module_name into a client module name
- parts = root_module_name.split(".")
- if parts[-1] == "client":
- client_name = root_module_name
- else:
- client_name = root_module_name + ".client"
- # import the top-level common library
- common_lib = __import__(client_name, globals(), locals(),
- ["common_lib"]).common_lib
- sys.modules["common"] = common_lib
- # patch up all the root_module_name.*.common libs
- for module_name in sys.modules.iterkeys():
- if (module_name.startswith(root_module_name + ".") and
- module_name.endswith(".common")):
- sys.modules[module_name] = common_lib
- # import the specific common libraries
- for library in common_lib.__all__:
- module = __import__(client_name + ".common_lib", globals(),
- locals(), [library])
- module = getattr(module, library)
- setattr(common_lib, library, module)
- sys.modules["common.%s" % library] = module
+ """
+ Setup aliases for all the common libraries, e.g.
+ common -> autotest_lib.client.common_lib
+ common.error -> autotest_lib.client.common_lib.error
+ """
+ # convert the root_module_name into a client module name
+ parts = root_module_name.split(".")
+ if parts[-1] == "client":
+ client_name = root_module_name
+ else:
+ client_name = root_module_name + ".client"
+ # import the top-level common library
+ common_lib = __import__(client_name, globals(), locals(),
+ ["common_lib"]).common_lib
+ sys.modules["common"] = common_lib
+ # patch up all the root_module_name.*.common libs
+ for module_name in sys.modules.iterkeys():
+ if (module_name.startswith(root_module_name + ".") and
+ module_name.endswith(".common")):
+ sys.modules[module_name] = common_lib
+ # import the specific common libraries
+ for library in common_lib.__all__:
+ module = __import__(client_name + ".common_lib", globals(),
+ locals(), [library])
+ module = getattr(module, library)
+ setattr(common_lib, library, module)
+ sys.modules["common.%s" % library] = module
def setup(base_path, root_module_name=""):
- """
- Perform all the necessary setup so that all the packages at
- 'base_path' can be imported via "import root_module_name.package".
- If root_module_name is empty, then all the packages at base_path
- are inserted as top-level packages.
+ """
+ Perform all the necessary setup so that all the packages at
+ 'base_path' can be imported via "import root_module_name.package".
+ If root_module_name is empty, then all the packages at base_path
+ are inserted as top-level packages.
- Also, setup all the common.* aliases for modules in the common
- library.
- """
- _create_module_and_parents(root_module_name)
- _import_children_into_module(root_module_name, base_path)
- _setup_common_library(root_module_name)
+ Also, setup all the common.* aliases for modules in the common
+ library.
+ """
+ _create_module_and_parents(root_module_name)
+ _import_children_into_module(root_module_name, base_path)
+ _setup_common_library(root_module_name)
# This must run on Python versions less than 2.4.
diff --git a/client/tests/aborttest/aborttest.py b/client/tests/aborttest/aborttest.py
index 44ceb17..6e8dadd 100755
--- a/client/tests/aborttest/aborttest.py
+++ b/client/tests/aborttest/aborttest.py
@@ -2,7 +2,7 @@
from autotest_lib.client.bin import test
class aborttest(test.test):
- version = 1
+ version = 1
- def execute(self):
- raise error.JobError('Arrrrrrrrggggh. You are DOOOMED')
+ def execute(self):
+ raise error.JobError('Arrrrrrrrggggh. You are DOOOMED')
diff --git a/client/tests/aio_dio_bugs/aio_dio_bugs.py b/client/tests/aio_dio_bugs/aio_dio_bugs.py
index a4563d8..05132db 100644
--- a/client/tests/aio_dio_bugs/aio_dio_bugs.py
+++ b/client/tests/aio_dio_bugs/aio_dio_bugs.py
@@ -5,36 +5,36 @@
# tests is a simple array of "cmd" "arguments"
tests = [["aio-dio-invalidate-failure", "poo"],
- ["aio-dio-subblock-eof-read", "eoftest"],
- ["aio-free-ring-with-bogus-nr-pages", ""],
- ["aio-io-setup-with-nonwritable-context-pointer", ""],
- ["aio-dio-extend-stat", "file"],
- ]
+ ["aio-dio-subblock-eof-read", "eoftest"],
+ ["aio-free-ring-with-bogus-nr-pages", ""],
+ ["aio-io-setup-with-nonwritable-context-pointer", ""],
+ ["aio-dio-extend-stat", "file"],
+ ]
name = 0
arglist = 1
class aio_dio_bugs(test.test):
- version = 5
- preserve_srcdir = True
+ version = 5
+ preserve_srcdir = True
- def initialize(self):
- self.job.setup_dep(['libaio'])
- ldflags = '-L ' + self.autodir + '/deps/libaio/lib'
- cflags = '-I ' + self.autodir + '/deps/libaio/include'
- self.gcc_flags = ldflags + ' ' + cflags
+ def initialize(self):
+ self.job.setup_dep(['libaio'])
+ ldflags = '-L ' + self.autodir + '/deps/libaio/lib'
+ cflags = '-I ' + self.autodir + '/deps/libaio/include'
+ self.gcc_flags = ldflags + ' ' + cflags
- def setup(self):
- os.chdir(self.srcdir)
- utils.system('make ' + '"CFLAGS=' + self.gcc_flags + '"')
+ def setup(self):
+ os.chdir(self.srcdir)
+ utils.system('make ' + '"CFLAGS=' + self.gcc_flags + '"')
- def execute(self, args = ''):
- os.chdir(self.tmpdir)
- libs = self.autodir + '/deps/libaio/lib/'
- ld_path = autotest_utils.prepend_path(libs,
- autotest_utils.environ('LD_LIBRARY_PATH'))
- var_ld_path = 'LD_LIBRARY_PATH=' + ld_path
- for test in tests:
- cmd = self.srcdir + '/' + test[name] + ' ' \
- + args + ' ' + test[arglist]
- utils.system(var_ld_path + ' ' + cmd)
+ def execute(self, args = ''):
+ os.chdir(self.tmpdir)
+ libs = self.autodir + '/deps/libaio/lib/'
+ ld_path = autotest_utils.prepend_path(libs,
+ autotest_utils.environ('LD_LIBRARY_PATH'))
+ var_ld_path = 'LD_LIBRARY_PATH=' + ld_path
+ for test in tests:
+ cmd = self.srcdir + '/' + test[name] + ' ' \
+ + args + ' ' + test[arglist]
+ utils.system(var_ld_path + ' ' + cmd)
diff --git a/client/tests/aiostress/aiostress.py b/client/tests/aiostress/aiostress.py
index 14878f8..6d39edc 100755
--- a/client/tests/aiostress/aiostress.py
+++ b/client/tests/aiostress/aiostress.py
@@ -1,7 +1,7 @@
# This requires aio headers to build.
# Should work automagically out of deps now.
-# NOTE - this should also have the ability to mount a filesystem,
+# NOTE - this should also have the ability to mount a filesystem,
# run the tests, unmount it, then fsck the filesystem
import os
from autotest_lib.client.bin import test, autotest_utils
@@ -9,68 +9,68 @@
class aiostress(test.test):
- version = 2
+ version = 2
- def initialize(self):
- self.job.setup_dep(['libaio'])
- ldflags = '-L ' + self.autodir + '/deps/libaio/lib'
- cflags = '-I ' + self.autodir + '/deps/libaio/include'
- self.gcc_flags = ldflags + ' ' + cflags
+ def initialize(self):
+ self.job.setup_dep(['libaio'])
+ ldflags = '-L ' + self.autodir + '/deps/libaio/lib'
+ cflags = '-I ' + self.autodir + '/deps/libaio/include'
+ self.gcc_flags = ldflags + ' ' + cflags
- # ftp://ftp.suse.com/pub/people/mason/utils/aio-stress.c
- def setup(self, tarball = None):
- print self.srcdir, self.bindir, self.tmpdir
- os.mkdir(self.srcdir)
- os.chdir(self.srcdir)
- utils.system('cp ' + self.bindir+'/aio-stress.c .')
- os.chdir(self.srcdir)
- self.gcc_flags += ' -Wall -lpthread -laio'
- cmd = 'gcc ' + self.gcc_flags + ' aio-stress.c -o aio-stress'
- utils.system(cmd)
+ # ftp://ftp.suse.com/pub/people/mason/utils/aio-stress.c
+ def setup(self, tarball = None):
+ print self.srcdir, self.bindir, self.tmpdir
+ os.mkdir(self.srcdir)
+ os.chdir(self.srcdir)
+ utils.system('cp ' + self.bindir+'/aio-stress.c .')
+ os.chdir(self.srcdir)
+ self.gcc_flags += ' -Wall -lpthread -laio'
+ cmd = 'gcc ' + self.gcc_flags + ' aio-stress.c -o aio-stress'
+ utils.system(cmd)
- def execute(self, args = ''):
- os.chdir(self.tmpdir)
- libs = self.autodir+'/deps/libaio/lib/'
- ld_path = autotest_utils.prepend_path(libs,
- autotest_utils.environ('LD_LIBRARY_PATH'))
- var_ld_path = 'LD_LIBRARY_PATH=' + ld_path
- cmd = self.srcdir + '/aio-stress ' + args + ' poo'
- profilers = self.job.profilers
+ def execute(self, args = ''):
+ os.chdir(self.tmpdir)
+ libs = self.autodir+'/deps/libaio/lib/'
+ ld_path = autotest_utils.prepend_path(libs,
+ autotest_utils.environ('LD_LIBRARY_PATH'))
+ var_ld_path = 'LD_LIBRARY_PATH=' + ld_path
+ cmd = self.srcdir + '/aio-stress ' + args + ' poo'
+ profilers = self.job.profilers
- if not profilers.only():
- utils.system(var_ld_path + ' ' + cmd)
- report = open(self.debugdir + '/stderr')
- keyval = open(self.resultsdir + '/keyval', 'w')
- _format_results(report, keyval)
+ if not profilers.only():
+ utils.system(var_ld_path + ' ' + cmd)
+ report = open(self.debugdir + '/stderr')
+ keyval = open(self.resultsdir + '/keyval', 'w')
+ _format_results(report, keyval)
- # Do a profiling run if necessary
- if profilers.present():
- profilers.start(self)
- utils.system(var_ld_path + ' ' + cmd)
- profilers.stop(self)
- profilers.report(self)
- if profilers.only():
- report = open(self.debugdir + '/stderr')
- keyval = open(self.resultsdir + '/keyval', 'w')
- _format_results(report, keyval)
+ # Do a profiling run if necessary
+ if profilers.present():
+ profilers.start(self)
+ utils.system(var_ld_path + ' ' + cmd)
+ profilers.stop(self)
+ profilers.report(self)
+ if profilers.only():
+ report = open(self.debugdir + '/stderr')
+ keyval = open(self.resultsdir + '/keyval', 'w')
+ _format_results(report, keyval)
def _format_results(report, keyval):
- for line in report:
- if 'threads' in line:
- if 'files' in line:
- if 'contexts' in line:
- break
+ for line in report:
+ if 'threads' in line:
+ if 'files' in line:
+ if 'contexts' in line:
+ break
- for line in report:
- line = line.split(')')[0]
- key, value = line.split('(')
- key = key.strip().replace(' ', '_')
- value = value.split()[0]
- print >> keyval, '%s=%s' % (key, value)
+ for line in report:
+ line = line.split(')')[0]
+ key, value = line.split('(')
+ key = key.strip().replace(' ', '_')
+ value = value.split()[0]
+ print >> keyval, '%s=%s' % (key, value)
"""
diff --git a/client/tests/barriertest/barriertest.py b/client/tests/barriertest/barriertest.py
index cf890ad..26bcab7 100644
--- a/client/tests/barriertest/barriertest.py
+++ b/client/tests/barriertest/barriertest.py
@@ -2,30 +2,30 @@
from autotest_utils.client.bin import test
class barriertest(test.test):
- version = 1
+ version = 1
- def execute(self, timeout_sync, timeout_start, timeout_stop,
- hostid, masterid, all_ids):
- profilers = self.job.profilers
+ def execute(self, timeout_sync, timeout_start, timeout_stop,
+ hostid, masterid, all_ids):
+ profilers = self.job.profilers
- b0 = self.job.barrier(hostid, "sync_profilers",
- timeout_start, port=63100)
- b0.rendevous_servers(masterid, hostid)
+ b0 = self.job.barrier(hostid, "sync_profilers",
+ timeout_start, port=63100)
+ b0.rendevous_servers(masterid, hostid)
- b1 = self.job.barrier(hostid, "start_profilers",
- timeout_start, port=63100)
- b1.rendevous_servers(masterid, hostid)
+ b1 = self.job.barrier(hostid, "start_profilers",
+ timeout_start, port=63100)
+ b1.rendevous_servers(masterid, hostid)
- b2 = self.job.barrier(hostid, "local_sync_profilers",
- timeout_sync)
- b2.rendevous(*all_ids)
+ b2 = self.job.barrier(hostid, "local_sync_profilers",
+ timeout_sync)
+ b2.rendevous(*all_ids)
- profilers.start(self)
+ profilers.start(self)
- b3 = self.job.barrier(hostid, "stop_profilers",
- timeout_stop, port=63100)
- b3.rendevous_servers(masterid, hostid)
+ b3 = self.job.barrier(hostid, "stop_profilers",
+ timeout_stop, port=63100)
+ b3.rendevous_servers(masterid, hostid)
- profilers.stop(self)
- profilers.report(self)
+ profilers.stop(self)
+ profilers.report(self)
diff --git a/client/tests/bash_shared_mapping/bash_shared_mapping.py b/client/tests/bash_shared_mapping/bash_shared_mapping.py
index e3fb69d..df993c1 100755
--- a/client/tests/bash_shared_mapping/bash_shared_mapping.py
+++ b/client/tests/bash_shared_mapping/bash_shared_mapping.py
@@ -3,39 +3,39 @@
from autotest_lib.client.common_lib import utils
class bash_shared_mapping(test.test):
- version = 3
+ version = 3
- # http://www.zip.com.au/~akpm/linux/patches/stuff/ext3-tools.tar.gz
- def setup(self, tarball = 'ext3-tools.tar.gz'):
- self.tarball = utils.unmap_url(self.bindir, tarball,
- self.tmpdir)
- autotest_utils.extract_tarball_to_dir(self.tarball, self.srcdir)
+ # http://www.zip.com.au/~akpm/linux/patches/stuff/ext3-tools.tar.gz
+ def setup(self, tarball = 'ext3-tools.tar.gz'):
+ self.tarball = utils.unmap_url(self.bindir, tarball,
+ self.tmpdir)
+ autotest_utils.extract_tarball_to_dir(self.tarball, self.srcdir)
- os.chdir(self.srcdir)
- utils.system('make bash-shared-mapping usemem')
+ os.chdir(self.srcdir)
+ utils.system('make bash-shared-mapping usemem')
- def execute(self, testdir = None, iterations = 10000):
- if not testdir:
- testdir = self.tmpdir
- os.chdir(testdir)
- file = os.path.join(testdir, 'foo')
- # Want to use 3/4 of all memory for each of
- # bash-shared-mapping and usemem
- kilobytes = (3 * autotest_utils.memtotal()) / 4
+ def execute(self, testdir = None, iterations = 10000):
+ if not testdir:
+ testdir = self.tmpdir
+ os.chdir(testdir)
+ file = os.path.join(testdir, 'foo')
+ # Want to use 3/4 of all memory for each of
+ # bash-shared-mapping and usemem
+ kilobytes = (3 * autotest_utils.memtotal()) / 4
- # Want two usemem -m megabytes in parallel in background.
- pid = [None, None]
- usemem = os.path.join(self.srcdir, 'usemem')
- args = ('usemem', '-N', '-m', '%d' % (kilobytes / 1024))
- # print_to_tty ('2 x ' + ' '.join(args))
- for i in (0,1):
- pid[i] = os.spawnv(os.P_NOWAIT, usemem, args)
+ # Want two usemem -m megabytes in parallel in background.
+ pid = [None, None]
+ usemem = os.path.join(self.srcdir, 'usemem')
+ args = ('usemem', '-N', '-m', '%d' % (kilobytes / 1024))
+ # print_to_tty ('2 x ' + ' '.join(args))
+ for i in (0,1):
+ pid[i] = os.spawnv(os.P_NOWAIT, usemem, args)
- cmd = "%s/bash-shared-mapping %s %d -t %d -n %d" % \
- (self.srcdir, file, kilobytes,
- count_cpus(), iterations)
- os.system(cmd)
+ cmd = "%s/bash-shared-mapping %s %d -t %d -n %d" % \
+ (self.srcdir, file, kilobytes,
+ count_cpus(), iterations)
+ os.system(cmd)
- for i in (0,1):
- os.kill(pid[i], signal.SIGKILL)
+ for i in (0,1):
+ os.kill(pid[i], signal.SIGKILL)
diff --git a/client/tests/bonnie/bonnie.py b/client/tests/bonnie/bonnie.py
index e025221..e6def25 100755
--- a/client/tests/bonnie/bonnie.py
+++ b/client/tests/bonnie/bonnie.py
@@ -4,73 +4,73 @@
def convert_size(values):
- values = values.split(':')
- size = values[0]
- if len(values) > 1:
- chunk = values[1]
+ values = values.split(':')
+ size = values[0]
+ if len(values) > 1:
+ chunk = values[1]
+ else:
+ chunk = 0
+ if size.endswith('G') or size.endswith('g'):
+ size = int(size[:-1]) * 2**30
+ else:
+ if size.endswith('M') or size.endswith('m'):
+ size = int(size[:-1])
+ size = int(size) * 2**20
+ if chunk:
+ if chunk.endswith('K') or chunk.endswith('k'):
+ chunk = int(chunk[:-1]) * 2**10
else:
- chunk = 0
- if size.endswith('G') or size.endswith('g'):
- size = int(size[:-1]) * 2**30
- else:
- if size.endswith('M') or size.endswith('m'):
- size = int(size[:-1])
- size = int(size) * 2**20
- if chunk:
- if chunk.endswith('K') or chunk.endswith('k'):
- chunk = int(chunk[:-1]) * 2**10
- else:
- chunk = int(chunk)
- return [size, chunk]
+ chunk = int(chunk)
+ return [size, chunk]
class bonnie(test.test):
- version = 1
+ version = 1
- # http://www.coker.com.au/bonnie++/bonnie++-1.03a.tgz
- def setup(self, tarball = 'bonnie++-1.03a.tgz'):
- tarball = utils.unmap_url(self.bindir, tarball,
- self.tmpdir)
- autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
- os.chdir(self.srcdir)
+ # http://www.coker.com.au/bonnie++/bonnie++-1.03a.tgz
+ def setup(self, tarball = 'bonnie++-1.03a.tgz'):
+ tarball = utils.unmap_url(self.bindir, tarball,
+ self.tmpdir)
+ autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
+ os.chdir(self.srcdir)
- os_dep.command('g++')
- utils.system('./configure')
- utils.system('make')
+ os_dep.command('g++')
+ utils.system('./configure')
+ utils.system('make')
- def execute(self, testdir = None, iterations = 1, extra_args = '', user = 'root'):
- if not testdir:
- testdir = self.tmpdir
+ def execute(self, testdir = None, iterations = 1, extra_args = '', user = 'root'):
+ if not testdir:
+ testdir = self.tmpdir
- args = '-d ' + testdir + ' -u ' + user + ' ' + extra_args
- cmd = self.srcdir + '/bonnie++ ' + args
- results = []
- profilers = self.job.profilers
- if not profilers.only():
- for i in range(iterations):
- results.append(utils.system_output(cmd,
- retain_output=True))
+ args = '-d ' + testdir + ' -u ' + user + ' ' + extra_args
+ cmd = self.srcdir + '/bonnie++ ' + args
+ results = []
+ profilers = self.job.profilers
+ if not profilers.only():
+ for i in range(iterations):
+ results.append(utils.system_output(cmd,
+ retain_output=True))
- # Do a profiling run if necessary
- if profilers.present():
- profilers.start(self)
- results.append(utils.system_output(cmd,
- retain_output=True))
- profilers.stop(self)
- profilers.report(self)
+ # Do a profiling run if necessary
+ if profilers.present():
+ profilers.start(self)
+ results.append(utils.system_output(cmd,
+ retain_output=True))
+ profilers.stop(self)
+ profilers.report(self)
- self.__format_results("\n".join(results))
+ self.__format_results("\n".join(results))
- def __format_results(self, results):
- strip_plus = lambda s: re.sub(r"^\++$", "0", s)
- out = open(self.resultsdir + '/keyval', 'w')
- for line in results.split('\n'):
- if len([c for c in line if c == ',']) != 26:
- continue
- fields = tuple(line.split(','))
- fields = [strip_plus(f) for f in fields]
- fields = tuple(convert_size(fields[1]) + fields[2:])
- print >> out, """size=%s
+ def __format_results(self, results):
+ strip_plus = lambda s: re.sub(r"^\++$", "0", s)
+ out = open(self.resultsdir + '/keyval', 'w')
+ for line in results.split('\n'):
+ if len([c for c in line if c == ',']) != 26:
+ continue
+ fields = tuple(line.split(','))
+ fields = [strip_plus(f) for f in fields]
+ fields = tuple(convert_size(fields[1]) + fields[2:])
+ print >> out, """size=%s
chnk=%s
seqout_perchr_ksec=%s
seqout_perchr_pctcp=%s
@@ -98,4 +98,4 @@
randcreate_delete_ksec=%s
randcreate_delete_pctcp=%s
""" % fields
- out.close()
+ out.close()
diff --git a/client/tests/btreplay/btreplay.py b/client/tests/btreplay/btreplay.py
index ba6e5cb..5bfc289 100644
--- a/client/tests/btreplay/btreplay.py
+++ b/client/tests/btreplay/btreplay.py
@@ -4,131 +4,130 @@
class btreplay(test.test):
- version = 1
+ version = 1
- # http://brick.kernel.dk/snaps/blktrace-git-latest.tar.gz
- def setup(self, tarball = 'blktrace-git-latest.tar.gz'):
- tarball = utils.unmap_url(self.bindir, tarball,
- self.tmpdir)
- autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
+ # http://brick.kernel.dk/snaps/blktrace-git-latest.tar.gz
+ def setup(self, tarball = 'blktrace-git-latest.tar.gz'):
+ tarball = utils.unmap_url(self.bindir, tarball,
+ self.tmpdir)
+ autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
- self.job.setup_dep(['libaio'])
- libs = '-L' + self.autodir + '/deps/libaio/lib -laio'
- cflags = '-I ' + self.autodir + '/deps/libaio/include'
- var_libs = 'LIBS="' + libs + '"'
- var_cflags = 'CFLAGS="' + cflags + '"'
- self.make_flags = var_libs + ' ' + var_cflags
+ self.job.setup_dep(['libaio'])
+ libs = '-L' + self.autodir + '/deps/libaio/lib -laio'
+ cflags = '-I ' + self.autodir + '/deps/libaio/include'
+ var_libs = 'LIBS="' + libs + '"'
+ var_cflags = 'CFLAGS="' + cflags + '"'
+ self.make_flags = var_libs + ' ' + var_cflags
- os.chdir(self.srcdir)
- utils.system('patch -p1 < ../Makefile.patch')
- utils.system(self.make_flags + ' make')
+ os.chdir(self.srcdir)
+ utils.system('patch -p1 < ../Makefile.patch')
+ utils.system(self.make_flags + ' make')
- def initialize(self):
- self.ldlib = 'LD_LIBRARY_PATH=%s/deps/libaio/lib'%(self.autodir)
-
-
- def _run_btreplay(self, dev, devices, tmpdir, extra_args):
- alldevs="-d /dev/"+dev
- alldnames = dev
- for d in devices.split():
- alldevs += " -d /dev/"+d
- alldnames += " "+d
-
- # convert the trace (assumed to be in this test's base
- # directory) into btreplay's required format
- utils.system("./btreplay/btrecord -d .. -D "+tmpdir+" "+dev)
-
- # time a replay that omits "thinktime" between requests
- # (by use of the -N flag)
- utils.system(self.ldlib+" /usr/bin/time ./btreplay/btreplay -d "+\
- tmpdir+" -N -W "+dev+" "+extra_args+" 2>&1")
-
- # trace a replay that reproduces inter-request delays, and
- # analyse the trace with btt to determine the average request
- # completion latency
- utils.system("./blktrace -D "+tmpdir+" "+alldevs+" >/dev/null &")
- utils.system(self.ldlib+" ./btreplay/btreplay -d "+tmpdir+" -W "+\
- dev+" "+extra_args)
- utils.system("killall -INT blktrace")
-
- # wait until blktrace is really done
- slept = 0.0
- while utils.system("ps -C blktrace > /dev/null",
- ignore_status=True) == 0:
- time.sleep(0.1)
- slept += 0.1
- if slept > 30.0:
- utils.system("killall -9 blktrace")
- raise error.TestError("blktrace failed to exit after 30 seconds")
- utils.system("./blkparse -q -D "+tmpdir+" -d "+tmpdir+\
- "/trace.bin -O "+alldnames+" >/dev/null")
- utils.system("./btt/btt -i "+tmpdir+"/trace.bin")
-
- def execute(self, iterations = 1, dev="", devices="",
- extra_args = '', tmpdir = None):
- # @dev: The device against which the trace will be replayed.
- # e.g. "sdb" or "md_d1"
- # @devices: A space-separated list of the underlying devices
- # which make up dev, e.g. "sdb sdc". You only need to set
- # devices if dev is an MD, LVM, or similar device;
- # otherwise leave it as an empty string.
-
- if not tmpdir:
- tmpdir = self.tmpdir
-
- os.chdir(self.srcdir)
-
- profilers = self.job.profilers
- if not profilers.only():
- for i in range(iterations):
- self._run_btreplay(dev, devices, tmpdir, extra_args)
-
- # Do a profiling run if necessary
- if profilers.present():
- profilers.start(self)
- self._run_btreplay(dev, devices, tmpdir, extra_args)
- profilers.stop(self)
- profilers.report(self)
-
- self.job.stdout.filehandle.flush()
- self.__format_results(open(self.debugdir + '/stdout').read())
+ def initialize(self):
+ self.ldlib = 'LD_LIBRARY_PATH=%s/deps/libaio/lib'%(self.autodir)
- def __format_results(self, results):
- out = open(self.resultsdir + '/keyval', 'w')
- lines = results.split('\n')
+ def _run_btreplay(self, dev, devices, tmpdir, extra_args):
+ alldevs="-d /dev/"+dev
+ alldnames = dev
+ for d in devices.split():
+ alldevs += " -d /dev/"+d
+ alldnames += " "+d
- for n in range(len(lines)):
- if lines[n].strip() == "==================== All Devices ====================":
- words = lines[n-2].split()
- s = words[1].strip('sytem').split(':')
- e = words[2].strip('elapsd').split(':')
- break
+ # convert the trace (assumed to be in this test's base
+ # directory) into btreplay's required format
+ utils.system("./btreplay/btrecord -d .. -D "+tmpdir+" "+dev)
- systime = 0.0
- for n in range(len(s)):
- i = (len(s)-1) - n
- systime += float(s[i])*(60**n)
- elapsed = 0.0
- for n in range(len(e)):
- i = (len(e)-1) - n
- elapsed += float(e[i])*(60**n)
+ # time a replay that omits "thinktime" between requests
+ # (by use of the -N flag)
+ utils.system(self.ldlib+" /usr/bin/time ./btreplay/btreplay -d "+\
+ tmpdir+" -N -W "+dev+" "+extra_args+" 2>&1")
- q2c = 0.0
- for line in lines:
- words = line.split()
- if len(words) < 3:
- continue
- if words[0] == 'Q2C':
- q2c = float(words[2])
- break
+ # trace a replay that reproduces inter-request delays, and
+ # analyse the trace with btt to determine the average request
+ # completion latency
+ utils.system("./blktrace -D "+tmpdir+" "+alldevs+" >/dev/null &")
+ utils.system(self.ldlib+" ./btreplay/btreplay -d "+tmpdir+" -W "+\
+ dev+" "+extra_args)
+ utils.system("killall -INT blktrace")
-
- print >> out, """\
+ # wait until blktrace is really done
+ slept = 0.0
+ while utils.system("ps -C blktrace > /dev/null",
+ ignore_status=True) == 0:
+ time.sleep(0.1)
+ slept += 0.1
+ if slept > 30.0:
+ utils.system("killall -9 blktrace")
+ raise error.TestError("blktrace failed to exit after 30 seconds")
+ utils.system("./blkparse -q -D "+tmpdir+" -d "+tmpdir+\
+ "/trace.bin -O "+alldnames+" >/dev/null")
+ utils.system("./btt/btt -i "+tmpdir+"/trace.bin")
+
+ def execute(self, iterations = 1, dev="", devices="",
+ extra_args = '', tmpdir = None):
+ # @dev: The device against which the trace will be replayed.
+ # e.g. "sdb" or "md_d1"
+ # @devices: A space-separated list of the underlying devices
+ # which make up dev, e.g. "sdb sdc". You only need to set
+ # devices if dev is an MD, LVM, or similar device;
+ # otherwise leave it as an empty string.
+
+ if not tmpdir:
+ tmpdir = self.tmpdir
+
+ os.chdir(self.srcdir)
+
+ profilers = self.job.profilers
+ if not profilers.only():
+ for i in range(iterations):
+ self._run_btreplay(dev, devices, tmpdir, extra_args)
+
+ # Do a profiling run if necessary
+ if profilers.present():
+ profilers.start(self)
+ self._run_btreplay(dev, devices, tmpdir, extra_args)
+ profilers.stop(self)
+ profilers.report(self)
+
+ self.job.stdout.filehandle.flush()
+ self.__format_results(open(self.debugdir + '/stdout').read())
+
+
+ def __format_results(self, results):
+ out = open(self.resultsdir + '/keyval', 'w')
+ lines = results.split('\n')
+
+ for n in range(len(lines)):
+ if lines[n].strip() == "==================== All Devices ====================":
+ words = lines[n-2].split()
+ s = words[1].strip('sytem').split(':')
+ e = words[2].strip('elapsd').split(':')
+ break
+
+ systime = 0.0
+ for n in range(len(s)):
+ i = (len(s)-1) - n
+ systime += float(s[i])*(60**n)
+ elapsed = 0.0
+ for n in range(len(e)):
+ i = (len(e)-1) - n
+ elapsed += float(e[i])*(60**n)
+
+ q2c = 0.0
+ for line in lines:
+ words = line.split()
+ if len(words) < 3:
+ continue
+ if words[0] == 'Q2C':
+ q2c = float(words[2])
+ break
+
+
+ print >> out, """\
time=%f
systime=%f
avg_q2c_latency=%f
""" % (elapsed, systime, q2c)
- out.close()
-
+ out.close()
diff --git a/client/tests/container_functional/container_functional.py b/client/tests/container_functional/container_functional.py
index 62bc3e8..eec0532 100644
--- a/client/tests/container_functional/container_functional.py
+++ b/client/tests/container_functional/container_functional.py
@@ -7,66 +7,66 @@
class container_functional(test.test):
- version = 1
+ version = 1
- def execute(self, mbytes=None, cpus=None, root='', name=None):
- """Check that the container was setup.
- The arguments must be the same than
- job.new_container()"""
- if not name:
- raise error.TestError("Must have a container name")
+ def execute(self, mbytes=None, cpus=None, root='', name=None):
+ """Check that the container was setup.
+ The arguments must be the same than
+ job.new_container()"""
+ if not name:
+ raise error.TestError("Must have a container name")
- # Do container exists?
- for container in ['sys', name]:
- try:
- utils.system('ls %s > /dev/null' % \
- os.path.join('/dev/cpuset',
- container))
- except error.CmdError:
- raise error.TestError("Container %s not created." % \
- container)
+ # Do container exists?
+ for container in ['sys', name]:
+ try:
+ utils.system('ls %s > /dev/null' % \
+ os.path.join('/dev/cpuset',
+ container))
+ except error.CmdError:
+ raise error.TestError("Container %s not created." % \
+ container)
- # Did we get the CPUs?
- if cpus:
- actual_cpus = utils.system_output('cat %s' % \
- os.path.join('/dev/cpuset',
- name,
- 'cpus'))
- if cpus != cpuset.rangelist_to_list(actual_cpus):
- raise error.TestError(("CPUs = %s, "
- "expecting: %s") %
- (actual_cpus, cpus))
+ # Did we get the CPUs?
+ if cpus:
+ actual_cpus = utils.system_output('cat %s' % \
+ os.path.join('/dev/cpuset',
+ name,
+ 'cpus'))
+ if cpus != cpuset.rangelist_to_list(actual_cpus):
+ raise error.TestError(("CPUs = %s, "
+ "expecting: %s") %
+ (actual_cpus, cpus))
- # Are we in this container?
- actual_pid = utils.system_output('cat %s' % \
- os.path.join('/dev/cpuset',
- name,
- 'tasks'))
+ # Are we in this container?
+ actual_pid = utils.system_output('cat %s' % \
+ os.path.join('/dev/cpuset',
+ name,
+ 'tasks'))
- if str(os.getpid()) not in actual_pid:
- raise error.TestError("My pid %s is not in "
- "container task list: %s" % \
- (str(os.getpid()), actual_pid))
+ if str(os.getpid()) not in actual_pid:
+ raise error.TestError("My pid %s is not in "
+ "container task list: %s" % \
+ (str(os.getpid()), actual_pid))
- # Our memory nodes != sys memory nodes
- actual_mems = utils.system_output('cat %s' % \
- os.path.join('/dev/cpuset',
- name,
- 'mems'))
- sys_mems = utils.system_output('cat %s' % \
- os.path.join('/dev/cpuset',
- 'sys',
- 'mems'))
+ # Our memory nodes != sys memory nodes
+ actual_mems = utils.system_output('cat %s' % \
+ os.path.join('/dev/cpuset',
+ name,
+ 'mems'))
+ sys_mems = utils.system_output('cat %s' % \
+ os.path.join('/dev/cpuset',
+ 'sys',
+ 'mems'))
- actual_nodes = set(cpuset.rangelist_to_list(actual_mems))
- sys_nodes = set(cpuset.rangelist_to_list(sys_mems))
+ actual_nodes = set(cpuset.rangelist_to_list(actual_mems))
+ sys_nodes = set(cpuset.rangelist_to_list(sys_mems))
- if actual_nodes.intersection(sys_nodes):
- raise error.TestError("Sys nodes = %s\n"
- "My nodes = %s" % \
- (sys_nodes, actual_nodes))
+ if actual_nodes.intersection(sys_nodes):
+ raise error.TestError("Sys nodes = %s\n"
+ "My nodes = %s" % \
+ (sys_nodes, actual_nodes))
- # Should only have one node for 100MB
- if len(actual_nodes) != 1:
- raise error.TestError(("Got more than 1 node: %s" %
- actual_nodes))
+ # Should only have one node for 100MB
+ if len(actual_nodes) != 1:
+ raise error.TestError(("Got more than 1 node: %s" %
+ actual_nodes))
diff --git a/client/tests/cpu_hotplug/cpu_hotplug.py b/client/tests/cpu_hotplug/cpu_hotplug.py
index 9d016cc..352145b 100644
--- a/client/tests/cpu_hotplug/cpu_hotplug.py
+++ b/client/tests/cpu_hotplug/cpu_hotplug.py
@@ -3,44 +3,44 @@
from autotest_lib.client.common_lib import utils
class cpu_hotplug(test.test):
- version = 2
+ version = 2
- # http://developer.osdl.org/dev/hotplug/tests/lhcs_regression-1.6.tgz
- def setup(self, tarball = 'lhcs_regression-1.6.tgz'):
- tarball = utils.unmap_url(self.bindir, tarball,
- self.tmpdir)
- autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
-
- def execute(self):
- # Check if the kernel supports cpu hotplug
- if autotest_utils.running_config():
- autotest_utils.check_for_kernel_feature('HOTPLUG_CPU')
-
- # Check cpu nums, if equals 1, quit.
- if autotest_utils.count_cpus() == 1:
- print 'Just only single cpu online, quiting...'
- sys.exit()
-
- # Have a simple and quick check first, FIX me please.
- utils.system('dmesg -c > /dev/null')
- for cpu in autotest_utils.cpu_online_map():
- if os.path.isfile('/sys/devices/system/cpu/cpu%s/online' % cpu):
- utils.system('echo 0 > /sys/devices/system/cpu/cpu%s/online' % cpu, 1)
- utils.system('dmesg -c')
- time.sleep(3)
- utils.system('echo 1 > /sys/devices/system/cpu/cpu%s/online' % cpu, 1)
- utils.system('dmesg -c')
- time.sleep(3)
-
- # Begin this cpu hotplug test big guru.
- os.chdir(self.srcdir)
- profilers = self.job.profilers
- if not profilers.only():
- utils.system('./runtests.sh')
+ # http://developer.osdl.org/dev/hotplug/tests/lhcs_regression-1.6.tgz
+ def setup(self, tarball = 'lhcs_regression-1.6.tgz'):
+ tarball = utils.unmap_url(self.bindir, tarball,
+ self.tmpdir)
+ autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
- # Do a profiling run if necessary
- if profilers.present():
- profilers.start(self)
- utils.system('./runtests.sh')
- profilers.stop(self)
- profilers.report(self)
+ def execute(self):
+ # Check if the kernel supports cpu hotplug
+ if autotest_utils.running_config():
+ autotest_utils.check_for_kernel_feature('HOTPLUG_CPU')
+
+ # Check cpu nums, if equals 1, quit.
+ if autotest_utils.count_cpus() == 1:
+ print 'Just only single cpu online, quiting...'
+ sys.exit()
+
+ # Have a simple and quick check first, FIX me please.
+ utils.system('dmesg -c > /dev/null')
+ for cpu in autotest_utils.cpu_online_map():
+ if os.path.isfile('/sys/devices/system/cpu/cpu%s/online' % cpu):
+ utils.system('echo 0 > /sys/devices/system/cpu/cpu%s/online' % cpu, 1)
+ utils.system('dmesg -c')
+ time.sleep(3)
+ utils.system('echo 1 > /sys/devices/system/cpu/cpu%s/online' % cpu, 1)
+ utils.system('dmesg -c')
+ time.sleep(3)
+
+ # Begin this cpu hotplug test big guru.
+ os.chdir(self.srcdir)
+ profilers = self.job.profilers
+ if not profilers.only():
+ utils.system('./runtests.sh')
+
+ # Do a profiling run if necessary
+ if profilers.present():
+ profilers.start(self)
+ utils.system('./runtests.sh')
+ profilers.stop(self)
+ profilers.report(self)
diff --git a/client/tests/cyclictest/cyclictest.py b/client/tests/cyclictest/cyclictest.py
index 5f0f98e..5c9189d 100755
--- a/client/tests/cyclictest/cyclictest.py
+++ b/client/tests/cyclictest/cyclictest.py
@@ -4,14 +4,14 @@
class cyclictest(test.test):
- version = 2
- preserve_srcdir = True
+ version = 2
+ preserve_srcdir = True
- # git://git.kernel.org/pub/scm/linux/kernel/git/tglx/rt-tests.git
+ # git://git.kernel.org/pub/scm/linux/kernel/git/tglx/rt-tests.git
- def setup(self):
- os.chdir(self.srcdir)
- utils.system('make')
+ def setup(self):
+ os.chdir(self.srcdir)
+ utils.system('make')
- def execute(self, args = '-t 10 -l 100000'):
- utils.system(self.srcdir + '/cyclictest ' + args)
+ def execute(self, args = '-t 10 -l 100000'):
+ utils.system(self.srcdir + '/cyclictest ' + args)
diff --git a/client/tests/dacapo/dacapo.py b/client/tests/dacapo/dacapo.py
index 38bb711..e7f10c2 100644
--- a/client/tests/dacapo/dacapo.py
+++ b/client/tests/dacapo/dacapo.py
@@ -1,10 +1,10 @@
# Dacapo test suite wrapper
#
-# This benchmark suite is intended as a tool for Java benchmarking by the
+# This benchmark suite is intended as a tool for Java benchmarking by the
# programming language, memory management and computer architecture communities.
-# It consists of a set of open source, real world applications with non-trivial
-# memory loads. The suite is the culmination of over five years work at eight
-# institutions, as part of the DaCapo research project, which was funded by a
+# It consists of a set of open source, real world applications with non-trivial
+# memory loads. The suite is the culmination of over five years work at eight
+# institutions, as part of the DaCapo research project, which was funded by a
# National Science Foundation ITR Grant, CCR-0085792.
#
import os
@@ -14,101 +14,100 @@
class dacapo(test.test):
- version = 1
+ version = 1
- def set_java_environment(self, jvm, java_root):
- '''\
- Setup java environment variables (path and classpath in order to
- execute a specific jvm specified by the java_root variable.
- java_root - Base of the java vm installation
- '''
- # Sun has changed the directory layout for java 6
- # (now there's no jre directory). Let's work around this...
- if jvm == 'sun16':
- self.java_home = java_root
- else:
- self.java_home = os.path.join(java_root, 'jre')
- self.java_bin = os.path.join(self.java_home, 'bin')
- self.java_lib = os.path.join(self.java_home, 'lib')
- os.environ['JAVA_ROOT'] = java_root
- os.environ['JAVA_HOME'] = self.java_home
- os.environ['JRE_HOME'] = self.java_home
- os.environ['CLASSPATH'] = self.java_lib
- os.environ['JAVA_BINDIR'] = self.java_bin
- os.environ['PATH'] = self.java_bin + ':' + os.environ['PATH']
+ def set_java_environment(self, jvm, java_root):
+ '''\
+ Setup java environment variables (path and classpath in order to
+ execute a specific jvm specified by the java_root variable.
+ java_root - Base of the java vm installation
+ '''
+ # Sun has changed the directory layout for java 6
+ # (now there's no jre directory). Let's work around this...
+ if jvm == 'sun16':
+ self.java_home = java_root
+ else:
+ self.java_home = os.path.join(java_root, 'jre')
+ self.java_bin = os.path.join(self.java_home, 'bin')
+ self.java_lib = os.path.join(self.java_home, 'lib')
+ os.environ['JAVA_ROOT'] = java_root
+ os.environ['JAVA_HOME'] = self.java_home
+ os.environ['JRE_HOME'] = self.java_home
+ os.environ['CLASSPATH'] = self.java_lib
+ os.environ['JAVA_BINDIR'] = self.java_bin
+ os.environ['PATH'] = self.java_bin + ':' + os.environ['PATH']
- def execute(self, test = 'antlr', config = './dacapo.cfg', jvm = 'ibm14-ppc64'):
- # Load the test configuration. If needed, use autotest tmpdir to write
- # files.
- my_config = config_loader(config, self.tmpdir)
- # Directory where we will cache the dacapo jar file
- # and the jvm package files
- self.cachedir = os.path.join(self.bindir, 'cache')
- if not os.path.isdir(self.cachedir):
- os.makedirs(self.cachedir)
+ def execute(self, test = 'antlr', config = './dacapo.cfg', jvm = 'ibm14-ppc64'):
+ # Load the test configuration. If needed, use autotest tmpdir to write
+ # files.
+ my_config = config_loader(config, self.tmpdir)
+ # Directory where we will cache the dacapo jar file
+ # and the jvm package files
+ self.cachedir = os.path.join(self.bindir, 'cache')
+ if not os.path.isdir(self.cachedir):
+ os.makedirs(self.cachedir)
- # Get dacapo jar URL
- # (It's possible to override the default URL that points to the
- # sourceforge repository)
- if my_config.get('dacapo', 'override_default_url') == 'no':
- self.dacapo_url = my_config.get('dacapo', 'tarball_url')
- else:
- self.dacapo_url = my_config.get('dacapo', 'tarball_url_alt')
- if not self.dacapo_url:
- raise error.TestError('Could not read dacapo URL from conf file')
- # We can cache the dacapo package file if we take some
- # precautions (checking md5 sum of the downloaded file)
- self.dacapo_md5 = my_config.get('dacapo', 'package_md5')
- if not self.dacapo_md5:
- e_msg = 'Could not read dacapo package md5sum from conf file'
- raise error.TestError(e_msg)
- self.dacapo_pkg = \
- autotest_utils.unmap_url_cache(self.cachedir, self.dacapo_url,
- self.dacapo_md5)
+ # Get dacapo jar URL
+ # (It's possible to override the default URL that points to the
+ # sourceforge repository)
+ if my_config.get('dacapo', 'override_default_url') == 'no':
+ self.dacapo_url = my_config.get('dacapo', 'tarball_url')
+ else:
+ self.dacapo_url = my_config.get('dacapo', 'tarball_url_alt')
+ if not self.dacapo_url:
+ raise error.TestError('Could not read dacapo URL from conf file')
+ # We can cache the dacapo package file if we take some
+ # precautions (checking md5 sum of the downloaded file)
+ self.dacapo_md5 = my_config.get('dacapo', 'package_md5')
+ if not self.dacapo_md5:
+ e_msg = 'Could not read dacapo package md5sum from conf file'
+ raise error.TestError(e_msg)
+ self.dacapo_pkg = \
+ autotest_utils.unmap_url_cache(self.cachedir, self.dacapo_url,
+ self.dacapo_md5)
- # Get jvm package URL
- self.jvm_pkg_url = my_config.get(jvm, 'jvm_pkg_url')
- if not self.jvm_pkg_url:
- raise error.TestError('Could not read java vm URL from conf file')
- # Let's cache the jvm package as well
- self.jvm_pkg_md5 = my_config.get(jvm, 'package_md5')
- if not self.jvm_pkg_md5:
- raise error.TestError('Could not read java package_md5 from conf file')
- self.jvm_pkg = \
- autotest_utils.unmap_url_cache(self.cachedir, self.jvm_pkg_url,
- self.jvm_pkg_md5)
+ # Get jvm package URL
+ self.jvm_pkg_url = my_config.get(jvm, 'jvm_pkg_url')
+ if not self.jvm_pkg_url:
+ raise error.TestError('Could not read java vm URL from conf file')
+ # Let's cache the jvm package as well
+ self.jvm_pkg_md5 = my_config.get(jvm, 'package_md5')
+ if not self.jvm_pkg_md5:
+ raise error.TestError('Could not read java package_md5 from conf file')
+ self.jvm_pkg = \
+ autotest_utils.unmap_url_cache(self.cachedir, self.jvm_pkg_url,
+ self.jvm_pkg_md5)
- # Install the jvm pakage
- package.install(self.jvm_pkg)
+ # Install the jvm pakage
+ package.install(self.jvm_pkg)
- # Basic Java environment variables setup
- self.java_root = my_config.get(jvm, 'java_root')
- if not self.java_root:
- raise error.TestError('Could not read java root dir from conf file')
- self.set_java_environment(jvm, self.java_root)
+ # Basic Java environment variables setup
+ self.java_root = my_config.get(jvm, 'java_root')
+ if not self.java_root:
+ raise error.TestError('Could not read java root dir from conf file')
+ self.set_java_environment(jvm, self.java_root)
- # If use_global is set to 'yes', then we want to use the global
- # setting instead of per test settings
- if my_config.get('global', 'use_global') == 'yes':
- self.iterations = my_config.get('global', 'iterations')
- self.workload = my_config.get('global', 'workload')
- else:
- self.iterations = my_config.get(test, 'iterations')
- self.workload = my_config.get(test, 'workload')
+ # If use_global is set to 'yes', then we want to use the global
+ # setting instead of per test settings
+ if my_config.get('global', 'use_global') == 'yes':
+ self.iterations = my_config.get('global', 'iterations')
+ self.workload = my_config.get('global', 'workload')
+ else:
+ self.iterations = my_config.get(test, 'iterations')
+ self.workload = my_config.get(test, 'workload')
- self.verbose = '-v '
- self.workload = '-s %s ' % self.workload
- self.iterations = '-n %s ' % self.iterations
- self.scratch = '-scratch %s ' % os.path.join(self.resultsdir, test)
- # Compose the arguments string
- self.args = self.verbose + self.workload + self.scratch \
- + self.iterations + test
- # Execute the actual test
- try:
- utils.system('java -jar %s %s' % (self.dacapo_pkg, self.args))
- except:
- e_msg = \
- 'Test %s has failed, command line options "%s"' % (test, self.args)
- raise error.TestError(e_msg)
-
+ self.verbose = '-v '
+ self.workload = '-s %s ' % self.workload
+ self.iterations = '-n %s ' % self.iterations
+ self.scratch = '-scratch %s ' % os.path.join(self.resultsdir, test)
+ # Compose the arguments string
+ self.args = self.verbose + self.workload + self.scratch \
+ + self.iterations + test
+ # Execute the actual test
+ try:
+ utils.system('java -jar %s %s' % (self.dacapo_pkg, self.args))
+ except:
+ e_msg = \
+ 'Test %s has failed, command line options "%s"' % (test, self.args)
+ raise error.TestError(e_msg)
diff --git a/client/tests/dbench/dbench.py b/client/tests/dbench/dbench.py
index a75adc6..085b4b8 100755
--- a/client/tests/dbench/dbench.py
+++ b/client/tests/dbench/dbench.py
@@ -3,47 +3,47 @@
from autotest_lib.client.common_lib import utils
class dbench(test.test):
- version = 1
+ version = 1
- # http://samba.org/ftp/tridge/dbench/dbench-3.04.tar.gz
- def setup(self, tarball = 'dbench-3.04.tar.gz'):
- tarball = utils.unmap_url(self.bindir, tarball, self.tmpdir)
- autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
- os.chdir(self.srcdir)
+ # http://samba.org/ftp/tridge/dbench/dbench-3.04.tar.gz
+ def setup(self, tarball = 'dbench-3.04.tar.gz'):
+ tarball = utils.unmap_url(self.bindir, tarball, self.tmpdir)
+ autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
+ os.chdir(self.srcdir)
- utils.system('./configure')
- utils.system('make')
+ utils.system('./configure')
+ utils.system('make')
- def execute(self, iterations = 1, dir = None, nprocs = None, args = ''):
- if not nprocs:
- nprocs = self.job.cpu_count()
- profilers = self.job.profilers
- args = args + ' -c '+self.srcdir+'/client.txt'
- if dir:
- args += ' -D ' + dir
- args += ' %s' % nprocs
- cmd = self.srcdir + '/dbench ' + args
- results = []
- if not profilers.only():
- for i in range(iterations):
- results.append(utils.system_output(cmd,
- retain_output=True))
+ def execute(self, iterations = 1, dir = None, nprocs = None, args = ''):
+ if not nprocs:
+ nprocs = self.job.cpu_count()
+ profilers = self.job.profilers
+ args = args + ' -c '+self.srcdir+'/client.txt'
+ if dir:
+ args += ' -D ' + dir
+ args += ' %s' % nprocs
+ cmd = self.srcdir + '/dbench ' + args
+ results = []
+ if not profilers.only():
+ for i in range(iterations):
+ results.append(utils.system_output(cmd,
+ retain_output=True))
- # Do a profiling run if necessary
- if profilers.present():
- profilers.start(self)
- results.append(utils.system_output(cmd,
- retain_output=True))
- profilers.stop(self)
- profilers.report(self)
+ # Do a profiling run if necessary
+ if profilers.present():
+ profilers.start(self)
+ results.append(utils.system_output(cmd,
+ retain_output=True))
+ profilers.stop(self)
+ profilers.report(self)
- self.__format_results("\n".join(results))
+ self.__format_results("\n".join(results))
- def __format_results(self, results):
- out = open(self.resultsdir + '/keyval', 'w')
- pattern = re.compile(r"Throughput (.*?) MB/sec (.*?) procs")
- for result in pattern.findall(results):
- print >> out, "throughput=%s\nprocs=%s\n" % result
- out.close()
+ def __format_results(self, results):
+ out = open(self.resultsdir + '/keyval', 'w')
+ pattern = re.compile(r"Throughput (.*?) MB/sec (.*?) procs")
+ for result in pattern.findall(results):
+ print >> out, "throughput=%s\nprocs=%s\n" % result
+ out.close()
diff --git a/client/tests/dbt2/dbt2.py b/client/tests/dbt2/dbt2.py
index d35018b..5d6d6fc 100644
--- a/client/tests/dbt2/dbt2.py
+++ b/client/tests/dbt2/dbt2.py
@@ -3,72 +3,72 @@
from autotest_lib.client.common_lib import utils
-# Dbt-2 is a fair-use implementation of the TPC-C benchmark. The test is
+# Dbt-2 is a fair-use implementation of the TPC-C benchmark. The test is
# currently hardcoded to use PostgreSQL but the kit also supports MySQL.
class dbt2(test.test):
- version = 2
+ version = 2
- # http://osdn.dl.sourceforge.net/sourceforge/osdldbt/dbt2-0.39.tar.gz
- def setup(self, tarball = 'dbt2-0.39.tar.bz2'):
- tarball = utils.unmap_url(self.bindir, tarball,
- self.tmpdir)
- autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
- self.job.setup_dep(['pgsql', 'pgpool', 'mysql'])
+ # http://osdn.dl.sourceforge.net/sourceforge/osdldbt/dbt2-0.39.tar.gz
+ def setup(self, tarball = 'dbt2-0.39.tar.bz2'):
+ tarball = utils.unmap_url(self.bindir, tarball,
+ self.tmpdir)
+ autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
+ self.job.setup_dep(['pgsql', 'pgpool', 'mysql'])
- #
- # Extract one copy of the kit for MySQL.
- #
- utils.system('cp -pR ' + self.srcdir + ' ' + self.srcdir + '.mysql')
- os.chdir(self.srcdir + '.mysql')
- utils.system('./configure --with-mysql=%s/deps/mysql/mysql' \
- % self.autodir)
- utils.system('make')
+ #
+ # Extract one copy of the kit for MySQL.
+ #
+ utils.system('cp -pR ' + self.srcdir + ' ' + self.srcdir + '.mysql')
+ os.chdir(self.srcdir + '.mysql')
+ utils.system('./configure --with-mysql=%s/deps/mysql/mysql' \
+ % self.autodir)
+ utils.system('make')
- #
- # Extract one copy of the kit for PostgreSQL.
- #
- utils.system('cp -pR ' + self.srcdir + ' ' + self.srcdir + '.pgsql')
- os.chdir(self.srcdir + '.pgsql')
- utils.system('./configure --with-postgresql=%s/deps/pgsql/pgsql' \
- % self.autodir)
- utils.system('make')
+ #
+ # Extract one copy of the kit for PostgreSQL.
+ #
+ utils.system('cp -pR ' + self.srcdir + ' ' + self.srcdir + '.pgsql')
+ os.chdir(self.srcdir + '.pgsql')
+ utils.system('./configure --with-postgresql=%s/deps/pgsql/pgsql' \
+ % self.autodir)
+ utils.system('make')
- # Create symlinks to autotest's results directory from dbt-2's
- # preferred results directory to self.resultsdir
- utils.system('ln -s %s %s' % (self.resultsdir, \
- self.srcdir + '.mysql/scripts/output'))
- utils.system('ln -s %s %s' % (self.resultsdir, \
- self.srcdir + '.pgsql/scripts/output'))
+ # Create symlinks to autotest's results directory from dbt-2's
+ # preferred results directory to self.resultsdir
+ utils.system('ln -s %s %s' % (self.resultsdir, \
+ self.srcdir + '.mysql/scripts/output'))
+ utils.system('ln -s %s %s' % (self.resultsdir, \
+ self.srcdir + '.pgsql/scripts/output'))
- def execute(self, db_type, args = ''):
- logfile = self.resultsdir + '/dbt2.log'
+ def execute(self, db_type, args = ''):
+ logfile = self.resultsdir + '/dbt2.log'
- if (db_type == "mysql"):
- self.execute_mysql(args)
- elif (db_type == "pgpool"):
- self.execute_pgpool(args)
- elif (db_type == "pgsql"):
- self.execute_pgsql(args)
+ if (db_type == "mysql"):
+ self.execute_mysql(args)
+ elif (db_type == "pgpool"):
+ self.execute_pgpool(args)
+ elif (db_type == "pgsql"):
+ self.execute_pgsql(args)
- def execute_mysql(self, args = ''):
- args = args
- utils.system(self.srcdir + '.mysql/scripts/mysql/build_db.sh -g -w 1')
- utils.system(self.srcdir + '.mysql/scripts/run_workload.sh ' + args)
+ def execute_mysql(self, args = ''):
+ args = args
+ utils.system(self.srcdir + '.mysql/scripts/mysql/build_db.sh -g -w 1')
+ utils.system(self.srcdir + '.mysql/scripts/run_workload.sh ' + args)
- def execute_pgpool(self, args = ''):
- utils.system('%s/deps/pgpool/pgpool/bin/pgpool -f %s/../pgpool.conf' \
- % (self.autodir, self.srcdir))
- self.execute_pgsql(args)
- utils.system('%s/deps/pgpool/pgpool/bin/pgpool stop' % self.autodir)
+ def execute_pgpool(self, args = ''):
+ utils.system('%s/deps/pgpool/pgpool/bin/pgpool -f %s/../pgpool.conf' \
+ % (self.autodir, self.srcdir))
+ self.execute_pgsql(args)
+ utils.system('%s/deps/pgpool/pgpool/bin/pgpool stop' % self.autodir)
- def execute_pgsql(self, args = ''):
- utils.system(self.srcdir + '.pgsql/scripts/pgsql/build_db.sh -g -w 1')
- utils.system(self.srcdir + '.pgsql/scripts/run_workload.sh ' + args)
- #
- # Clean up by dropping the database after the test.
- #
- utils.system(self.srcdir + '.pgsql/scripts/pgsql/start_db.sh')
- utils.system(self.srcdir + '.pgsql/scripts/pgsql/drop_db.sh')
- utils.system(self.srcdir + '.pgsql/scripts/pgsql/stop_db.sh')
+ def execute_pgsql(self, args = ''):
+ utils.system(self.srcdir + '.pgsql/scripts/pgsql/build_db.sh -g -w 1')
+ utils.system(self.srcdir + '.pgsql/scripts/run_workload.sh ' + args)
+ #
+ # Clean up by dropping the database after the test.
+ #
+ utils.system(self.srcdir + '.pgsql/scripts/pgsql/start_db.sh')
+ utils.system(self.srcdir + '.pgsql/scripts/pgsql/drop_db.sh')
+ utils.system(self.srcdir + '.pgsql/scripts/pgsql/stop_db.sh')
diff --git a/client/tests/disktest/disktest.py b/client/tests/disktest/disktest.py
index e021cb8..2019c00 100755
--- a/client/tests/disktest/disktest.py
+++ b/client/tests/disktest/disktest.py
@@ -4,58 +4,57 @@
class disktest(test.test):
- version = 1
+ version = 1
- def setup(self):
- os.mkdir(self.srcdir)
- os.chdir(self.bindir)
- utils.system('cp disktest.c src/')
- os.chdir(self.srcdir)
- cflags = '-D_FILE_OFFSET_BITS=64 -D _GNU_SOURCE -static -Wall'
- utils.system('cc disktest.c ' + cflags + ' -o disktest')
+ def setup(self):
+ os.mkdir(self.srcdir)
+ os.chdir(self.bindir)
+ utils.system('cp disktest.c src/')
+ os.chdir(self.srcdir)
+ cflags = '-D_FILE_OFFSET_BITS=64 -D _GNU_SOURCE -static -Wall'
+ utils.system('cc disktest.c ' + cflags + ' -o disktest')
- def test_one_disk_chunk(self, disk, chunk):
- print "testing %d MB files on %s in %d MB memory" % \
- (self.chunk_mb, disk, self.memory_mb)
- cmd = "%s/disktest -m %d -f %s/testfile.%d -i -S" % \
- (self.srcdir, self.chunk_mb, disk, chunk)
- p = subprocess.Popen(cmd, shell=True)
- return(p.pid)
+ def test_one_disk_chunk(self, disk, chunk):
+ print "testing %d MB files on %s in %d MB memory" % \
+ (self.chunk_mb, disk, self.memory_mb)
+ cmd = "%s/disktest -m %d -f %s/testfile.%d -i -S" % \
+ (self.srcdir, self.chunk_mb, disk, chunk)
+ p = subprocess.Popen(cmd, shell=True)
+ return(p.pid)
- def execute(self, disks = None, gigabytes = None,
- chunk_mb = autotest_utils.memtotal() / 1024):
- os.chdir(self.srcdir)
+ def execute(self, disks = None, gigabytes = None,
+ chunk_mb = autotest_utils.memtotal() / 1024):
+ os.chdir(self.srcdir)
- if not disks:
- disks = [self.tmpdir]
- if not gigabytes:
- free = 100 # cap it at 100GB by default
- for disk in disks:
- free = min(autotest_utils.freespace(disk) / 1024**3, free)
- gigabytes = free
- print "resizing to %s GB" % gigabytes
- sys.stdout.flush()
+ if not disks:
+ disks = [self.tmpdir]
+ if not gigabytes:
+ free = 100 # cap it at 100GB by default
+ for disk in disks:
+ free = min(autotest_utils.freespace(disk) / 1024**3, free)
+ gigabytes = free
+ print "resizing to %s GB" % gigabytes
+ sys.stdout.flush()
- self.chunk_mb = chunk_mb
- self.memory_mb = autotest_utils.memtotal()/1024
- if self.memory_mb > chunk_mb:
- e_msg = "Too much RAM (%dMB) for this test to work" % self.memory_mb
- raise error.TestError(e_msg)
+ self.chunk_mb = chunk_mb
+ self.memory_mb = autotest_utils.memtotal()/1024
+ if self.memory_mb > chunk_mb:
+ e_msg = "Too much RAM (%dMB) for this test to work" % self.memory_mb
+ raise error.TestError(e_msg)
- chunks = (1024 * gigabytes) / chunk_mb
+ chunks = (1024 * gigabytes) / chunk_mb
- for i in range(chunks):
- pids = []
- for disk in disks:
- pid = self.test_one_disk_chunk(disk, i)
- pids.append(pid)
- errors = []
- for pid in pids:
- (junk, retval) = os.waitpid(pid, 0)
- if (retval != 0):
- errors.append(retval)
- if errors:
- raise error.TestError("Errors from children: %s" % errors)
-
+ for i in range(chunks):
+ pids = []
+ for disk in disks:
+ pid = self.test_one_disk_chunk(disk, i)
+ pids.append(pid)
+ errors = []
+ for pid in pids:
+ (junk, retval) = os.waitpid(pid, 0)
+ if (retval != 0):
+ errors.append(retval)
+ if errors:
+ raise error.TestError("Errors from children: %s" % errors)
diff --git a/client/tests/fio/fio.py b/client/tests/fio/fio.py
index e1d9734..aad6bf7 100644
--- a/client/tests/fio/fio.py
+++ b/client/tests/fio/fio.py
@@ -4,36 +4,36 @@
class fio(test.test):
- version = 2
+ version = 2
- # http://brick.kernel.dk/snaps/fio-1.16.5.tar.bz2
- def setup(self, tarball = 'fio-1.16.5.tar.bz2'):
- tarball = utils.unmap_url(self.bindir, tarball,
- self.tmpdir)
- autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
+ # http://brick.kernel.dk/snaps/fio-1.16.5.tar.bz2
+ def setup(self, tarball = 'fio-1.16.5.tar.bz2'):
+ tarball = utils.unmap_url(self.bindir, tarball,
+ self.tmpdir)
+ autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
- self.job.setup_dep(['libaio'])
- ldflags = '-L' + self.autodir + '/deps/libaio/lib'
- cflags = '-I' + self.autodir + '/deps/libaio/include'
- var_ldflags = 'LDFLAGS="' + ldflags + '"'
- var_cflags = 'CFLAGS="' + cflags + '"'
+ self.job.setup_dep(['libaio'])
+ ldflags = '-L' + self.autodir + '/deps/libaio/lib'
+ cflags = '-I' + self.autodir + '/deps/libaio/include'
+ var_ldflags = 'LDFLAGS="' + ldflags + '"'
+ var_cflags = 'CFLAGS="' + cflags + '"'
- os.chdir(self.srcdir)
- utils.system('patch -p1 < ../Makefile.patch')
- utils.system('%s %s make' % (var_ldflags, var_cflags))
+ os.chdir(self.srcdir)
+ utils.system('patch -p1 < ../Makefile.patch')
+ utils.system('%s %s make' % (var_ldflags, var_cflags))
- def execute(self, args = '', user = 'root'):
- os.chdir(self.srcdir)
- ##vars = 'TMPDIR=\"%s\" RESULTDIR=\"%s\"' % (self.tmpdir, self.resultsdir)
- vars = 'LD_LIBRARY_PATH="' + self.autodir + '/deps/libaio/lib"'
- ##args = '-m -o ' + self.resultsdir + '/fio-tio.log ' + self.srcdir + '/examples/tiobench-example';
- args = '--output ' + self.resultsdir + '/fio-mixed.log ' + self.bindir + '/fio-mixed.job';
- utils.system(vars + ' ./fio ' + args)
+ def execute(self, args = '', user = 'root'):
+ os.chdir(self.srcdir)
+ ##vars = 'TMPDIR=\"%s\" RESULTDIR=\"%s\"' % (self.tmpdir, self.resultsdir)
+ vars = 'LD_LIBRARY_PATH="' + self.autodir + '/deps/libaio/lib"'
+ ##args = '-m -o ' + self.resultsdir + '/fio-tio.log ' + self.srcdir + '/examples/tiobench-example';
+ args = '--output ' + self.resultsdir + '/fio-mixed.log ' + self.bindir + '/fio-mixed.job';
+ utils.system(vars + ' ./fio ' + args)
- # Do a profiling run if necessary
- profilers = self.job.profilers
- if profilers.present():
- profilers.start(self)
- utils.system(vars + ' ./fio ' + args)
- profilers.stop(self)
- profilers.report(self)
+ # Do a profiling run if necessary
+ profilers = self.job.profilers
+ if profilers.present():
+ profilers.start(self)
+ utils.system(vars + ' ./fio ' + args)
+ profilers.stop(self)
+ profilers.report(self)
diff --git a/client/tests/fs_mark/fs_mark.py b/client/tests/fs_mark/fs_mark.py
index f38f011..6b52e19 100644
--- a/client/tests/fs_mark/fs_mark.py
+++ b/client/tests/fs_mark/fs_mark.py
@@ -4,30 +4,30 @@
class fs_mark(test.test):
- version = 1
+ version = 1
- # http://developer.osdl.org/dev/doubt/fs_mark/archive/fs_mark-3.2.tgz
- def setup(self, tarball = 'fs_mark-3.2.tgz'):
- tarball = utils.unmap_url(self.bindir, tarball,
- self.tmpdir)
- autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
- os.chdir(self.srcdir)
+ # http://developer.osdl.org/dev/doubt/fs_mark/archive/fs_mark-3.2.tgz
+ def setup(self, tarball = 'fs_mark-3.2.tgz'):
+ tarball = utils.unmap_url(self.bindir, tarball,
+ self.tmpdir)
+ autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
+ os.chdir(self.srcdir)
- utils.system('make')
-
- def execute(self, dir, iterations = 2, args = None):
- os.chdir(self.srcdir)
- if not args:
- # Just provide a sample run parameters
- args = '-s 10240 -n 1000'
- profilers = self.job.profilers
- if not profilers.only():
- for i in range(iterations):
- utils.system('./fs_mark -d %s %s' %(dir, args))
+ utils.system('make')
- # Do a profiling run if necessary
- if profilers.present():
- profilers.start(self)
- utils.system('./fs_mark -d %s %s' %(dir, args))
- profilers.stop(self)
- profilers.report(self)
+ def execute(self, dir, iterations = 2, args = None):
+ os.chdir(self.srcdir)
+ if not args:
+ # Just provide a sample run parameters
+ args = '-s 10240 -n 1000'
+ profilers = self.job.profilers
+ if not profilers.only():
+ for i in range(iterations):
+ utils.system('./fs_mark -d %s %s' %(dir, args))
+
+ # Do a profiling run if necessary
+ if profilers.present():
+ profilers.start(self)
+ utils.system('./fs_mark -d %s %s' %(dir, args))
+ profilers.stop(self)
+ profilers.report(self)
diff --git a/client/tests/fsfuzzer/fsfuzzer.py b/client/tests/fsfuzzer/fsfuzzer.py
index 07d2c09..d8310e3 100755
--- a/client/tests/fsfuzzer/fsfuzzer.py
+++ b/client/tests/fsfuzzer/fsfuzzer.py
@@ -4,27 +4,27 @@
class fsfuzzer(test.test):
- version = 1
+ version = 1
- # http://people.redhat.com/sgrubb/files/fsfuzzer-0.6.tar.gz
- def setup(self, tarball = 'fsfuzzer-0.6.tar.gz'):
- tarball = utils.unmap_url(self.bindir, tarball,
- self.tmpdir)
- autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
- os.chdir(self.srcdir)
+ # http://people.redhat.com/sgrubb/files/fsfuzzer-0.6.tar.gz
+ def setup(self, tarball = 'fsfuzzer-0.6.tar.gz'):
+ tarball = utils.unmap_url(self.bindir, tarball,
+ self.tmpdir)
+ autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
+ os.chdir(self.srcdir)
- utils.system('make')
-
- def execute(self, iterations = 1, fstype = 'iso9660'):
- profilers = self.job.profilers
- args = fstype + ' 1'
- if not profilers.only():
- for i in range(iterations):
- utils.system(self.srcdir + '/run_test ' + args)
+ utils.system('make')
- # Do a profiling run if necessary
- if profilers.present():
- profilers.start(self)
- utils.system(self.srcdir + '/run_test ' + args)
- profilers.stop(self)
- profilers.report(self)
+ def execute(self, iterations = 1, fstype = 'iso9660'):
+ profilers = self.job.profilers
+ args = fstype + ' 1'
+ if not profilers.only():
+ for i in range(iterations):
+ utils.system(self.srcdir + '/run_test ' + args)
+
+ # Do a profiling run if necessary
+ if profilers.present():
+ profilers.start(self)
+ utils.system(self.srcdir + '/run_test ' + args)
+ profilers.stop(self)
+ profilers.report(self)
diff --git a/client/tests/fsstress/fsstress.py b/client/tests/fsstress/fsstress.py
index 9b2a2db..6a6e20f 100644
--- a/client/tests/fsstress/fsstress.py
+++ b/client/tests/fsstress/fsstress.py
@@ -5,33 +5,33 @@
class fsstress(test.test):
- version = 1
+ version = 1
- # http://www.zip.com.au/~akpm/linux/patches/stuff/ext3-tools.tar.gz
- def setup(self, tarball = 'ext3-tools.tar.gz'):
- self.tarball = utils.unmap_url(self.bindir, tarball,
- self.tmpdir)
- autotest_utils.extract_tarball_to_dir(self.tarball, self.srcdir)
+ # http://www.zip.com.au/~akpm/linux/patches/stuff/ext3-tools.tar.gz
+ def setup(self, tarball = 'ext3-tools.tar.gz'):
+ self.tarball = utils.unmap_url(self.bindir, tarball,
+ self.tmpdir)
+ autotest_utils.extract_tarball_to_dir(self.tarball, self.srcdir)
- os.chdir(self.srcdir)
- utils.system('patch -p1 < ../fsstress-ltp.patch')
- utils.system('make fsstress')
+ os.chdir(self.srcdir)
+ utils.system('patch -p1 < ../fsstress-ltp.patch')
+ utils.system('make fsstress')
- def execute(self, testdir = None, extra_args = '', nproc = '1000', nops = '1000'):
- if not testdir:
- testdir = self.tmpdir
+ def execute(self, testdir = None, extra_args = '', nproc = '1000', nops = '1000'):
+ if not testdir:
+ testdir = self.tmpdir
- args = '-d ' + testdir + ' -p ' + nproc + ' -n ' + nops + ' ' + extra_args
+ args = '-d ' + testdir + ' -p ' + nproc + ' -n ' + nops + ' ' + extra_args
- cmd = self.srcdir + '/fsstress ' + args
- profilers = self.job.profilers
- if not profilers.only():
- utils.system(cmd)
+ cmd = self.srcdir + '/fsstress ' + args
+ profilers = self.job.profilers
+ if not profilers.only():
+ utils.system(cmd)
- # Do a profiling run if necessary
- if profilers.present():
- profilers.start(self)
- utils.system(cmd)
- profilers.stop(self)
- profilers.report(self)
+ # Do a profiling run if necessary
+ if profilers.present():
+ profilers.start(self)
+ utils.system(cmd)
+ profilers.stop(self)
+ profilers.report(self)
diff --git a/client/tests/fsx/fsx.py b/client/tests/fsx/fsx.py
index 5e77cf1..08328be 100755
--- a/client/tests/fsx/fsx.py
+++ b/client/tests/fsx/fsx.py
@@ -1,7 +1,7 @@
# This requires aio headers to build.
# Should work automagically out of deps now.
-# NOTE - this should also have the ability to mount a filesystem,
+# NOTE - this should also have the ability to mount a filesystem,
# run the tests, unmount it, then fsck the filesystem
import os
from autotest_lib.client.bin import test, autotest_utils
@@ -9,43 +9,43 @@
class fsx(test.test):
- version = 3
+ version = 3
- # http://www.zip.com.au/~akpm/linux/patches/stuff/ext3-tools.tar.gz
- def setup(self, tarball = 'ext3-tools.tar.gz'):
- self.tarball = utils.unmap_url(self.bindir, tarball,
- self.tmpdir)
- autotest_utils.extract_tarball_to_dir(self.tarball, self.srcdir)
+ # http://www.zip.com.au/~akpm/linux/patches/stuff/ext3-tools.tar.gz
+ def setup(self, tarball = 'ext3-tools.tar.gz'):
+ self.tarball = utils.unmap_url(self.bindir, tarball,
+ self.tmpdir)
+ autotest_utils.extract_tarball_to_dir(self.tarball, self.srcdir)
- self.job.setup_dep(['libaio'])
- ldflags = '-L' + self.autodir + '/deps/libaio/lib'
- cflags = '-I' + self.autodir + '/deps/libaio/include'
- var_ldflags = 'LDFLAGS="' + ldflags + '"'
- var_cflags = 'CFLAGS="' + cflags + '"'
- self.make_flags = var_ldflags + ' ' + var_cflags
-
- os.chdir(self.srcdir)
- utils.system('patch -p1 < ../fsx-linux.diff')
- utils.system(self.make_flags + ' make fsx-linux')
+ self.job.setup_dep(['libaio'])
+ ldflags = '-L' + self.autodir + '/deps/libaio/lib'
+ cflags = '-I' + self.autodir + '/deps/libaio/include'
+ var_ldflags = 'LDFLAGS="' + ldflags + '"'
+ var_cflags = 'CFLAGS="' + cflags + '"'
+ self.make_flags = var_ldflags + ' ' + var_cflags
+
+ os.chdir(self.srcdir)
+ utils.system('patch -p1 < ../fsx-linux.diff')
+ utils.system(self.make_flags + ' make fsx-linux')
- def execute(self, testdir = None, repeat = '100000'):
- args = '-N ' + repeat
- if not testdir:
- testdir = self.tmpdir
- os.chdir(testdir)
- libs = self.autodir+'/deps/libaio/lib/'
- ld_path = autotest_utils.prepend_path(libs,
- autotest_utils.environ('LD_LIBRARY_PATH'))
- var_ld_path = 'LD_LIBRARY_PATH=' + ld_path
- cmd = self.srcdir + '/fsx-linux ' + args + ' poo'
- profilers = self.job.profilers
- if not profilers.only():
- utils.system(var_ld_path + ' ' + cmd)
+ def execute(self, testdir = None, repeat = '100000'):
+ args = '-N ' + repeat
+ if not testdir:
+ testdir = self.tmpdir
+ os.chdir(testdir)
+ libs = self.autodir+'/deps/libaio/lib/'
+ ld_path = autotest_utils.prepend_path(libs,
+ autotest_utils.environ('LD_LIBRARY_PATH'))
+ var_ld_path = 'LD_LIBRARY_PATH=' + ld_path
+ cmd = self.srcdir + '/fsx-linux ' + args + ' poo'
+ profilers = self.job.profilers
+ if not profilers.only():
+ utils.system(var_ld_path + ' ' + cmd)
- # Do a profiling run if necessary
- if profilers.present():
- profilers.start(self)
- utils.system(var_ld_path + ' ' + cmd)
- profilers.stop(self)
- profilers.report(self)
+ # Do a profiling run if necessary
+ if profilers.present():
+ profilers.start(self)
+ utils.system(var_ld_path + ' ' + cmd)
+ profilers.stop(self)
+ profilers.report(self)
diff --git a/client/tests/interbench/interbench.py b/client/tests/interbench/interbench.py
index 0bbdec4..aca7834 100644
--- a/client/tests/interbench/interbench.py
+++ b/client/tests/interbench/interbench.py
@@ -4,31 +4,31 @@
class interbench(test.test):
- version = 1
+ version = 1
- # http://www.kernel.org/pub/linux/kernel/people/ck/apps/interbench/interbench-0.30.tar.bz2
- def setup(self, tarball = 'interbench-0.30.tar.bz2'):
- tarball = utils.unmap_url(self.bindir, tarball,
- self.tmpdir)
- autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
- os.chdir(self.srcdir)
+ # http://www.kernel.org/pub/linux/kernel/people/ck/apps/interbench/interbench-0.30.tar.bz2
+ def setup(self, tarball = 'interbench-0.30.tar.bz2'):
+ tarball = utils.unmap_url(self.bindir, tarball,
+ self.tmpdir)
+ autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
+ os.chdir(self.srcdir)
- utils.system('make')
-
- def execute(self, iterations = 1, args = ''):
- os.chdir(self.tmpdir)
- args += " -c"
+ utils.system('make')
- profilers = self.job.profilers
- if not profilers.only():
- for i in range(iterations):
- utils.system("%s/interbench -m 'run #%s' %s" % \
- (self.srcdir, i, args))
+ def execute(self, iterations = 1, args = ''):
+ os.chdir(self.tmpdir)
+ args += " -c"
- # Do a profiling run if necessary
- if profilers.present():
- profilers.start(self)
- utils.system("%s/interbench -m 'profile run' %s" % \
- (self.srcdir, args))
- profilers.stop(self)
- profilers.report(self)
+ profilers = self.job.profilers
+ if not profilers.only():
+ for i in range(iterations):
+ utils.system("%s/interbench -m 'run #%s' %s" % \
+ (self.srcdir, i, args))
+
+ # Do a profiling run if necessary
+ if profilers.present():
+ profilers.start(self)
+ utils.system("%s/interbench -m 'profile run' %s" % \
+ (self.srcdir, args))
+ profilers.stop(self)
+ profilers.report(self)
diff --git a/client/tests/iozone/iozone.py b/client/tests/iozone/iozone.py
index 26c23d6..8347a32 100644
--- a/client/tests/iozone/iozone.py
+++ b/client/tests/iozone/iozone.py
@@ -5,66 +5,66 @@
class iozone(test.test):
- version = 1
+ version = 1
- # http://www.iozone.org/src/current/iozone3_283.tar
- def setup(self, tarball = 'iozone3_283.tar'):
- tarball = utils.unmap_url(self.bindir, tarball,
- self.tmpdir)
- autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
- os.chdir(os.path.join(self.srcdir, 'src/current'))
+ # http://www.iozone.org/src/current/iozone3_283.tar
+ def setup(self, tarball = 'iozone3_283.tar'):
+ tarball = utils.unmap_url(self.bindir, tarball,
+ self.tmpdir)
+ autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
+ os.chdir(os.path.join(self.srcdir, 'src/current'))
- arch = autotest_utils.get_current_kernel_arch()
- if (arch == 'ppc'):
- utils.system('make linux-powerpc')
- elif (arch == 'ppc64'):
- utils.system('make linux-powerpc64')
- elif (arch == 'x86_64'):
- utils.system('make linux-AMD64')
- else:
- utils.system('make linux')
+ arch = autotest_utils.get_current_kernel_arch()
+ if (arch == 'ppc'):
+ utils.system('make linux-powerpc')
+ elif (arch == 'ppc64'):
+ utils.system('make linux-powerpc64')
+ elif (arch == 'x86_64'):
+ utils.system('make linux-AMD64')
+ else:
+ utils.system('make linux')
- def execute(self, dir = None, iterations=1, args = None):
- self.keyval = open(os.path.join(self.resultsdir, 'keyval'),
- 'w')
- if not dir:
- dir = self.tmpdir
- os.chdir(dir)
- if not args:
- args = '-a'
- profilers = self.job.profilers
- if not profilers.only():
- for i in range(iterations):
- output = utils.system_output('%s/src/current/iozone %s' %
- (self.srcdir, args))
- self.__format_results(output)
+ def execute(self, dir = None, iterations=1, args = None):
+ self.keyval = open(os.path.join(self.resultsdir, 'keyval'),
+ 'w')
+ if not dir:
+ dir = self.tmpdir
+ os.chdir(dir)
+ if not args:
+ args = '-a'
+ profilers = self.job.profilers
+ if not profilers.only():
+ for i in range(iterations):
+ output = utils.system_output('%s/src/current/iozone %s' %
+ (self.srcdir, args))
+ self.__format_results(output)
- # Do a profiling run if necessary
- if profilers.present():
- profilers.start(self)
- output = utils.system_output('%s/src/current/iozone %s' %
- (self.srcdir, args))
- self.__format_results(output)
- profilers.stop(self)
- profilers.report(self)
+ # Do a profiling run if necessary
+ if profilers.present():
+ profilers.start(self)
+ output = utils.system_output('%s/src/current/iozone %s' %
+ (self.srcdir, args))
+ self.__format_results(output)
+ profilers.stop(self)
+ profilers.report(self)
- self.keyval.close()
+ self.keyval.close()
- def __format_results(self, results):
- labels = ('write', 'rewrite', 'read', 'reread', 'randread',
- 'randwrite', 'bkwdread', 'recordrewrite',
- 'strideread', 'fwrite', 'frewrite',
- 'fread', 'freread')
- for line in results.splitlines():
- fields = line.split()
- if len(fields) != 15:
- continue
- try:
- fields = tuple([int(i) for i in fields])
- except ValueError:
- continue
- for l, v in zip(labels, fields[2:]):
- print >> self.keyval, "%d-%d-%s=%d" % (fields[0], fields[1], l, v)
- print >> self.keyval
+ def __format_results(self, results):
+ labels = ('write', 'rewrite', 'read', 'reread', 'randread',
+ 'randwrite', 'bkwdread', 'recordrewrite',
+ 'strideread', 'fwrite', 'frewrite',
+ 'fread', 'freread')
+ for line in results.splitlines():
+ fields = line.split()
+ if len(fields) != 15:
+ continue
+ try:
+ fields = tuple([int(i) for i in fields])
+ except ValueError:
+ continue
+ for l, v in zip(labels, fields[2:]):
+ print >> self.keyval, "%d-%d-%s=%d" % (fields[0], fields[1], l, v)
+ print >> self.keyval
diff --git a/client/tests/isic/isic.py b/client/tests/isic/isic.py
index 9378830..df616be 100644
--- a/client/tests/isic/isic.py
+++ b/client/tests/isic/isic.py
@@ -4,23 +4,23 @@
class isic(test.test):
- version = 2
+ version = 2
- # http://www.packetfactory.net/Projects/ISIC/isic-0.06.tgz
- # + http://www.stardust.webpages.pl/files/crap/isic-gcc41-fix.patch
+ # http://www.packetfactory.net/Projects/ISIC/isic-0.06.tgz
+ # + http://www.stardust.webpages.pl/files/crap/isic-gcc41-fix.patch
- def initialize(self):
- self.job.setup_dep(['libnet'])
+ def initialize(self):
+ self.job.setup_dep(['libnet'])
- def setup(self, tarball = 'isic-0.06.tar.bz2'):
- tarball = utils.unmap_url(self.bindir, tarball,
- self.tmpdir)
- autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
- os.chdir(self.srcdir)
+ def setup(self, tarball = 'isic-0.06.tar.bz2'):
+ tarball = utils.unmap_url(self.bindir, tarball,
+ self.tmpdir)
+ autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
+ os.chdir(self.srcdir)
- utils.system('patch -p1 < ../build-fixes.patch')
- utils.system('PREFIX=' + self.autodir + '/deps/libnet/libnet/ ./configure')
- utils.system('make')
+ utils.system('patch -p1 < ../build-fixes.patch')
+ utils.system('PREFIX=' + self.autodir + '/deps/libnet/libnet/ ./configure')
+ utils.system('make')
- def execute(self, args = '-s rand -d 127.0.0.1 -p 10000000'):
- utils.system(self.srcdir + '/isic ' + args)
+ def execute(self, args = '-s rand -d 127.0.0.1 -p 10000000'):
+ utils.system(self.srcdir + '/isic ' + args)
diff --git a/client/tests/kernbench/kernbench.py b/client/tests/kernbench/kernbench.py
index 8abf6a3..de7d849 100755
--- a/client/tests/kernbench/kernbench.py
+++ b/client/tests/kernbench/kernbench.py
@@ -4,107 +4,107 @@
class kernbench(test.test):
- version = 2
+ version = 2
- def setup(self, build_dir = None):
- if not build_dir:
- build_dir = self.srcdir
- os.mkdir(build_dir)
+ def setup(self, build_dir = None):
+ if not build_dir:
+ build_dir = self.srcdir
+ os.mkdir(build_dir)
- def __init_tree(self, build_dir, version = None):
- #
- # If we have a local copy of the 2.6.14 tarball use that
- # else let the kernel object use the defined mirrors
- # to obtain it.
- #
- # http://kernel.org/pub/linux/kernel/v2.6/linux-2.6.14.tar.bz2
- #
- # On ia64, we default to 2.6.20, as it can't compile 2.6.14.
- if version:
- default_ver = version
- elif autotest_utils.get_current_kernel_arch() == 'ia64':
- default_ver = '2.6.20'
- else:
- default_ver = '2.6.14'
+ def __init_tree(self, build_dir, version = None):
+ #
+ # If we have a local copy of the 2.6.14 tarball use that
+ # else let the kernel object use the defined mirrors
+ # to obtain it.
+ #
+ # http://kernel.org/pub/linux/kernel/v2.6/linux-2.6.14.tar.bz2
+ #
+ # On ia64, we default to 2.6.20, as it can't compile 2.6.14.
+ if version:
+ default_ver = version
+ elif autotest_utils.get_current_kernel_arch() == 'ia64':
+ default_ver = '2.6.20'
+ else:
+ default_ver = '2.6.14'
- kversionfile = os.path.join(build_dir, ".kversion")
- install_needed = True
- if os.path.exists(kversionfile):
- old_version = pickle.load(open(kversionfile, 'r'))
- if (old_version == default_ver):
- install_needed = False
+ kversionfile = os.path.join(build_dir, ".kversion")
+ install_needed = True
+ if os.path.exists(kversionfile):
+ old_version = pickle.load(open(kversionfile, 'r'))
+ if (old_version == default_ver):
+ install_needed = False
- if not install_needed:
- return
+ if not install_needed:
+ return
- # Clear out the old version
- utils.system("echo rm -rf '" + build_dir + "/*'")
+ # Clear out the old version
+ utils.system("echo rm -rf '" + build_dir + "/*'")
- pickle.dump(default_ver, open(kversionfile, 'w'))
+ pickle.dump(default_ver, open(kversionfile, 'w'))
- tarball = None
- for dir in (self.bindir, '/usr/local/src'):
- tar = 'linux-%s.tar.bz2' % default_ver
- path = os.path.join(dir, tar)
- if os.path.exists(path):
- tarball = path
- break
- if not tarball:
- tarball = default_ver
+ tarball = None
+ for dir in (self.bindir, '/usr/local/src'):
+ tar = 'linux-%s.tar.bz2' % default_ver
+ path = os.path.join(dir, tar)
+ if os.path.exists(path):
+ tarball = path
+ break
+ if not tarball:
+ tarball = default_ver
- # Do the extraction of the kernel tree
- kernel = self.job.kernel(tarball, self.tmpdir, build_dir)
- kernel.config(defconfig=True, logged=False)
+ # Do the extraction of the kernel tree
+ kernel = self.job.kernel(tarball, self.tmpdir, build_dir)
+ kernel.config(defconfig=True, logged=False)
- def execute(self, iterations = 1, threads = None, dir = None, version = None):
- if not threads:
- threads = self.job.cpu_count()*2
- if dir:
- build_dir = dir
- else:
- build_dir = os.path.join(self.tmpdir, "src")
- if not os.path.exists(build_dir):
- os.makedirs(build_dir)
+ def execute(self, iterations = 1, threads = None, dir = None, version = None):
+ if not threads:
+ threads = self.job.cpu_count()*2
+ if dir:
+ build_dir = dir
+ else:
+ build_dir = os.path.join(self.tmpdir, "src")
+ if not os.path.exists(build_dir):
+ os.makedirs(build_dir)
- self.__init_tree(build_dir, version)
+ self.__init_tree(build_dir, version)
- kernel = self.job.kernel(build_dir, self.tmpdir, build_dir,
- leave = True)
- print "kernbench x %d: %d threads" % (iterations, threads)
+ kernel = self.job.kernel(build_dir, self.tmpdir, build_dir,
+ leave = True)
+ print "kernbench x %d: %d threads" % (iterations, threads)
- logfile = os.path.join(self.debugdir, 'build_log')
+ logfile = os.path.join(self.debugdir, 'build_log')
- print "Warmup run ..."
- kernel.build_timed(threads, output = logfile) # warmup run
+ print "Warmup run ..."
+ kernel.build_timed(threads, output = logfile) # warmup run
- profilers = self.job.profilers
- if not profilers.only():
- for i in range(iterations):
- print "Performance run, iteration %d ..." % i
- timefile = os.path.join(self.resultsdir,
- 'time.%d' % i)
- kernel.build_timed(threads, timefile)
+ profilers = self.job.profilers
+ if not profilers.only():
+ for i in range(iterations):
+ print "Performance run, iteration %d ..." % i
+ timefile = os.path.join(self.resultsdir,
+ 'time.%d' % i)
+ kernel.build_timed(threads, timefile)
- # Do a profiling run if necessary
- if profilers.present():
- profilers.start(self)
- print "Profiling run ..."
- timefile = os.path.join(self.resultsdir, 'time.profile')
- kernel.build_timed(threads, timefile)
- profilers.stop(self)
- profilers.report(self)
+ # Do a profiling run if necessary
+ if profilers.present():
+ profilers.start(self)
+ print "Profiling run ..."
+ timefile = os.path.join(self.resultsdir, 'time.profile')
+ kernel.build_timed(threads, timefile)
+ profilers.stop(self)
+ profilers.report(self)
- kernel.clean(logged=False) # Don't leave litter lying around
- os.chdir(self.resultsdir)
- utils.system("grep -h elapsed time.* > time")
+ kernel.clean(logged=False) # Don't leave litter lying around
+ os.chdir(self.resultsdir)
+ utils.system("grep -h elapsed time.* > time")
- self.__format_results(open('time').read())
+ self.__format_results(open('time').read())
- def __format_results(self, results):
- out = open('keyval', 'w')
- for result in autotest_utils.extract_all_time_results(results):
- print >> out, "user=%s\nsystem=%s\nelapsed=%s\n" % result
- out.close()
+ def __format_results(self, results):
+ out = open('keyval', 'w')
+ for result in autotest_utils.extract_all_time_results(results):
+ print >> out, "user=%s\nsystem=%s\nelapsed=%s\n" % result
+ out.close()
diff --git a/client/tests/kernelbuild/kernelbuild.py b/client/tests/kernelbuild/kernelbuild.py
index 95b639a..836a766 100755
--- a/client/tests/kernelbuild/kernelbuild.py
+++ b/client/tests/kernelbuild/kernelbuild.py
@@ -2,12 +2,12 @@
class kernelbuild(test.test):
- version = 1
+ version = 1
- def execute(self, base_tree, patches, config, config_list = None):
- kernel = self.job.kernel(base_tree, self.outputdir)
- if patches:
- kernel.patch(*patches)
- kernel.config(config, config_list)
+ def execute(self, base_tree, patches, config, config_list = None):
+ kernel = self.job.kernel(base_tree, self.outputdir)
+ if patches:
+ kernel.patch(*patches)
+ kernel.config(config, config_list)
- kernel.build()
+ kernel.build()
diff --git a/client/tests/kvmtest/control.with_modbuild b/client/tests/kvmtest/control.with_modbuild
index 09c3992..a527c68 100644
--- a/client/tests/kvmtest/control.with_modbuild
+++ b/client/tests/kvmtest/control.with_modbuild
@@ -1,6 +1,6 @@
# CHANGEME - point to kvm release tarball
-#
-#
+#
+#
# nightly kvm snapshot base URL
SNAPBASE = 'http://people.qumranet.com/avi/snapshots/'
@@ -19,44 +19,44 @@
def install_kvm_external_modules(tarball=None, base=SNAPBASE, daysold=DAYSOLD):
- dldir = os.environ['AUTODIR']+'/tmp'
- srcdir = os.environ['AUTODIR']+'/tmp/kvm'
- print "kvm dldir->%s"%(dldir)
- print "kvm srcdir->%s"%(srcdir)
-
- # ex: http://people.qumranet.com/avi/snapshots/kvm-snapshot-20071021.tar.gz
- if tarball == None:
- d = (date.today() - timedelta(days=daysold)).strftime('%Y%m%d')
- tarball = base+'kvm-snapshot-%s.tar.gz' %(d)
- sys.stderr.write("tarball url: %s\n" %(tarball))
+ dldir = os.environ['AUTODIR']+'/tmp'
+ srcdir = os.environ['AUTODIR']+'/tmp/kvm'
+ print "kvm dldir->%s"%(dldir)
+ print "kvm srcdir->%s"%(srcdir)
- tarball = unmap_url("/", tarball, dldir)
- extract_tarball_to_dir(tarball, srcdir)
- os.chdir(srcdir)
+ # ex: http://people.qumranet.com/avi/snapshots/kvm-snapshot-20071021.tar.gz
+ if tarball == None:
+ d = (date.today() - timedelta(days=daysold)).strftime('%Y%m%d')
+ tarball = base+'kvm-snapshot-%s.tar.gz' %(d)
+ sys.stderr.write("tarball url: %s\n" %(tarball))
- print "detecting cpu vendor..."
- vendor = "intel"
- if os.system("grep vmx /proc/cpuinfo 1>/dev/null") != 0:
- vendor = "amd"
- print "detected cpu vendor as '%s'" %(vendor)
+ tarball = unmap_url("/", tarball, dldir)
+ extract_tarball_to_dir(tarball, srcdir)
+ os.chdir(srcdir)
- print "building kvm..."
- system('./configure')
- system('make')
- system('make install')
- print "done building and installing kvm"
+ print "detecting cpu vendor..."
+ vendor = "intel"
+ if os.system("grep vmx /proc/cpuinfo 1>/dev/null") != 0:
+ vendor = "amd"
+ print "detected cpu vendor as '%s'" %(vendor)
- # remove existing in kernel kvm modules
- print "unloading loaded kvm modules (if present) ..."
- if system("grep kvm_%s /proc/modules 1>/dev/null" %(vendor), 1) == 0:
- system("rmmod -f kvm_%s" %(vendor))
- if system("grep kvm /proc/modules 1>/dev/null", 1) == 0:
- system("rmmod -f kvm")
+ print "building kvm..."
+ system('./configure')
+ system('make')
+ system('make install')
+ print "done building and installing kvm"
- # load new modules
- print "loading new kvm modules..."
- os.chdir(srcdir+'/kernel')
- system("insmod ./kvm.ko && sleep 1 && insmod ./kvm-%s.ko" %(vendor))
+ # remove existing in kernel kvm modules
+ print "unloading loaded kvm modules (if present) ..."
+ if system("grep kvm_%s /proc/modules 1>/dev/null" %(vendor), 1) == 0:
+ system("rmmod -f kvm_%s" %(vendor))
+ if system("grep kvm /proc/modules 1>/dev/null", 1) == 0:
+ system("rmmod -f kvm")
+
+ # load new modules
+ print "loading new kvm modules..."
+ os.chdir(srcdir+'/kernel')
+ system("insmod ./kvm.ko && sleep 1 && insmod ./kvm-%s.ko" %(vendor))
# build and install kvm external modules
diff --git a/client/tests/kvmtest/kvmtest.py b/client/tests/kvmtest/kvmtest.py
index 96806c0..f776033 100644
--- a/client/tests/kvmtest/kvmtest.py
+++ b/client/tests/kvmtest/kvmtest.py
@@ -4,158 +4,158 @@
class kvmtest(test.test):
- version = 1
+ version = 1
- def setup(self, tarball = 'kvm-test.tar.gz'):
- tarball = utils.unmap_url(self.bindir, tarball,
- self.tmpdir)
- autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
- os.chdir(self.srcdir)
- utils.system('python setup.py install')
+ def setup(self, tarball = 'kvm-test.tar.gz'):
+ tarball = utils.unmap_url(self.bindir, tarball,
+ self.tmpdir)
+ autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
+ os.chdir(self.srcdir)
+ utils.system('python setup.py install')
- def execute(self, testdir = '', args = ''):
- dirs = []
- results = []
- passed = 0
- failed = 0
+ def execute(self, testdir = '', args = ''):
+ dirs = []
+ results = []
+ passed = 0
+ failed = 0
- # spawn vncserver if needed
- if not os.environ.has_key('DISPLAY'):
- print("No DISPLAY set in environment,"
- "spawning vncserver...")
- display = self.__create_vncserver(os.environ['HOME']+"/.vnc")
- print("Setting DISPLAY=%s"%(display))
- os.environ['DISPLAY'] = display
+ # spawn vncserver if needed
+ if not os.environ.has_key('DISPLAY'):
+ print("No DISPLAY set in environment,"
+ "spawning vncserver...")
+ display = self.__create_vncserver(os.environ['HOME']+"/.vnc")
+ print("Setting DISPLAY=%s"%(display))
+ os.environ['DISPLAY'] = display
- # build a list of dirs with 'vm.log' files
- os.path.walk(testdir, self.__has_vmlog, dirs)
+ # build a list of dirs with 'vm.log' files
+ os.path.walk(testdir, self.__has_vmlog, dirs)
- for d in dirs:
- replaydir = os.path.join(self.resultsdir,
- os.path.basename(d))
- os.mkdir(replaydir)
- logfile = replaydir + "/%s.log" %(os.path.basename(d))
+ for d in dirs:
+ replaydir = os.path.join(self.resultsdir,
+ os.path.basename(d))
+ os.mkdir(replaydir)
+ logfile = replaydir + "/%s.log" %(os.path.basename(d))
- os.chdir(d)
- rv = utils.system("kvm-test-replay > %s" %(logfile), 1)
+ os.chdir(d)
+ rv = utils.system("kvm-test-replay > %s" %(logfile), 1)
- results.append((d, rv))
- if rv != 0:
- screenshot = self.__get_expected_file(logfile)
- expected = "expected-%03d.png" %(
- random.randint(0, 999))
- dest = os.path.join(replaydir,expected)
+ results.append((d, rv))
+ if rv != 0:
+ screenshot = self.__get_expected_file(logfile)
+ expected = "expected-%03d.png" %(
+ random.randint(0, 999))
+ dest = os.path.join(replaydir,expected)
- # make a copy of the screen shot
- utils.system("cp %s %s" %(screenshot, dest), 1)
-
- # move the failure
- utils.system("mv failure-*.png %s" %(replaydir), 1)
+ # make a copy of the screen shot
+ utils.system("cp %s %s" %(screenshot, dest), 1)
- # generate html output
- self.__format_results(results)
+ # move the failure
+ utils.system("mv failure-*.png %s" %(replaydir), 1)
- # produce pass/fail output
- for (x, y) in results:
- if y != 0:
- print("FAIL: '%s' with rv %s" %(x, y))
- failed = failed + 1
- else:
- print("pass: '%s' with rv %s" %(x, y))
- passed = passed + 1
+ # generate html output
+ self.__format_results(results)
- print("Summary: Passed %d Failed %d" %(passed, failed))
- # if we had any tests not passed, fail entire test
- if failed != 0:
- raise error.TestError('kvm-test-replay')
+ # produce pass/fail output
+ for (x, y) in results:
+ if y != 0:
+ print("FAIL: '%s' with rv %s" %(x, y))
+ failed = failed + 1
+ else:
+ print("pass: '%s' with rv %s" %(x, y))
+ passed = passed + 1
+
+ print("Summary: Passed %d Failed %d" %(passed, failed))
+ # if we had any tests not passed, fail entire test
+ if failed != 0:
+ raise error.TestError('kvm-test-replay')
- def __get_expected_file(self, logfile):
- # pull out screeshot name from logfile
- return filter(lambda x: "Expected" in x,
- open(logfile, 'r').readlines())\
- [0].split('{')[1].split('}')[0]
+ def __get_expected_file(self, logfile):
+ # pull out screeshot name from logfile
+ return filter(lambda x: "Expected" in x,
+ open(logfile, 'r').readlines())\
+ [0].split('{')[1].split('}')[0]
- def __create_vncserver(self, dirname):
- """
- this test may run without an X connection in kvm/qemu needs
- a DISPLAY to push the vga buffer. If a DISPLAY is not set
- in the environment, then attempt to spawn a vncserver, and
- change env DISPLAY so that kvmtest can run
- """
- for pidfile in locate("*:*.pid", dirname):
- pid = open(pidfile, 'r').readline().strip()
- # if the server is still active, just use it for display
- if os.path.exists('/proc/%s/status' % pid):
- vncdisplay = os.path.basename(pidfile)\
- .split(":")[1].split(".")[0]
- print("Found vncserver on port %s, using it"%(
- vncdisplay))
- return ':%s.0' %(vncdisplay)
-
- # none of the vncserver were still alive, spawn our own and
- # return the display whack existing server first, then spawn it
- vncdisplay = "1"
- print("Spawning vncserver on port %s"%(vncdisplay))
- utils.system('vncserver :%s' %(vncdisplay))
- return ':%s.0' %(vncdisplay)
+ def __create_vncserver(self, dirname):
+ """
+ this test may run without an X connection in kvm/qemu needs
+ a DISPLAY to push the vga buffer. If a DISPLAY is not set
+ in the environment, then attempt to spawn a vncserver, and
+ change env DISPLAY so that kvmtest can run
+ """
+ for pidfile in locate("*:*.pid", dirname):
+ pid = open(pidfile, 'r').readline().strip()
+ # if the server is still active, just use it for display
+ if os.path.exists('/proc/%s/status' % pid):
+ vncdisplay = os.path.basename(pidfile)\
+ .split(":")[1].split(".")[0]
+ print("Found vncserver on port %s, using it"%(
+ vncdisplay))
+ return ':%s.0' %(vncdisplay)
+
+ # none of the vncserver were still alive, spawn our own and
+ # return the display whack existing server first, then spawn it
+ vncdisplay = "1"
+ print("Spawning vncserver on port %s"%(vncdisplay))
+ utils.system('vncserver :%s' %(vncdisplay))
+ return ':%s.0' %(vncdisplay)
- def __has_vmlog(self, arg, dirname, names):
- if os.path.exists(os.path.join(dirname, 'vm.log')):
- arg.append(dirname)
+ def __has_vmlog(self, arg, dirname, names):
+ if os.path.exists(os.path.join(dirname, 'vm.log')):
+ arg.append(dirname)
- def __gen_fail_html(self, testdir):
- # generate a failure index.html to display the expected and failure
- # images
- fail_dir = os.path.join(self.resultsdir, os.path.basename(testdir))
- fail_index = os.path.join(fail_dir, "index.html")
+ def __gen_fail_html(self, testdir):
+ # generate a failure index.html to display the expected and failure
+ # images
+ fail_dir = os.path.join(self.resultsdir, os.path.basename(testdir))
+ fail_index = os.path.join(fail_dir, "index.html")
- # lambda helpers for pulling out image files
- is_png = lambda x: x.endswith('.png')
- failure_filter = lambda x: x.startswith('failure') and is_png(x)
- expected_filter = lambda x: x.startswith('expected') and is_png(x)
+ # lambda helpers for pulling out image files
+ is_png = lambda x: x.endswith('.png')
+ failure_filter = lambda x: x.startswith('failure') and is_png(x)
+ expected_filter = lambda x: x.startswith('expected') and is_png(x)
- failure_img = filter(failure_filter, os.listdir(fail_dir))[0]
- expected_img = filter(expected_filter, os.listdir(fail_dir))[0]
- if not failure_img or not expected_img:
- raise "Failed to find images"
+ failure_img = filter(failure_filter, os.listdir(fail_dir))[0]
+ expected_img = filter(expected_filter, os.listdir(fail_dir))[0]
+ if not failure_img or not expected_img:
+ raise "Failed to find images"
- fail_buff = "<html><table border=1><tr><th>Barrier Diff</th>\n" + \
- "<th>Expected Barrier</th><th>Failure</th></tr><tr><td></td>\n"
- for img in expected_img, failure_img:
- fail_buff = fail_buff + "<td><a href=\"%s\"><img width=320 " \
- "height=200 src=\"%s\"></a></td>\n" %(img, img)
+ fail_buff = "<html><table border=1><tr><th>Barrier Diff</th>\n" + \
+ "<th>Expected Barrier</th><th>Failure</th></tr><tr><td></td>\n"
+ for img in expected_img, failure_img:
+ fail_buff = fail_buff + "<td><a href=\"%s\"><img width=320 " \
+ "height=200 src=\"%s\"></a></td>\n" %(img, img)
- fail_buff = fail_buff + "</tr></table></html>\n"
+ fail_buff = fail_buff + "</tr></table></html>\n"
- fh = open(fail_index, "w+")
- fh.write(fail_buff)
- fh.close()
+ fh = open(fail_index, "w+")
+ fh.write(fail_buff)
+ fh.close()
- def __format_results(self, results):
- # generate kvmtest/index.html and an index.html for each fail
- test_index = os.path.join(self.outputdir, "index.html")
- test_buff = "<html><table border=1><tr><th>Test</th>\n"
+ def __format_results(self, results):
+ # generate kvmtest/index.html and an index.html for each fail
+ test_index = os.path.join(self.outputdir, "index.html")
+ test_buff = "<html><table border=1><tr><th>Test</th>\n"
- for (x,y) in results:
- test_buff = test_buff + "<th>%s</th>\n" %(os.path.basename(x))
+ for (x,y) in results:
+ test_buff = test_buff + "<th>%s</th>\n" %(os.path.basename(x))
- test_buff = test_buff + "</tr><tr><td></td>\n"
+ test_buff = test_buff + "</tr><tr><td></td>\n"
- for (x,y) in results:
- if y != 0:
- fail = "<td><a href=\"results/%s/\">FAIL</a></td>\n" %(os.path.basename(x))
- test_buff = test_buff + fail
- self.__gen_fail_html(x)
- else:
- test_buff = test_buff + "<td>GOOD</td>\n"
+ for (x,y) in results:
+ if y != 0:
+ fail = "<td><a href=\"results/%s/\">FAIL</a></td>\n" %(os.path.basename(x))
+ test_buff = test_buff + fail
+ self.__gen_fail_html(x)
+ else:
+ test_buff = test_buff + "<td>GOOD</td>\n"
- test_buff = test_buff + "</tr></table></html>"
+ test_buff = test_buff + "</tr></table></html>"
- fh = open(test_index, "w+")
- fh.write(test_buff)
- fh.close()
+ fh = open(test_index, "w+")
+ fh.write(test_buff)
+ fh.close()
diff --git a/client/tests/libhugetlbfs/libhugetlbfs.py b/client/tests/libhugetlbfs/libhugetlbfs.py
index 1f0aed1..aa018c5 100644
--- a/client/tests/libhugetlbfs/libhugetlbfs.py
+++ b/client/tests/libhugetlbfs/libhugetlbfs.py
@@ -3,58 +3,58 @@
from autotest_lib.client.common_lib import utils, error
class libhugetlbfs(test.test):
- version = 4
+ version = 4
- # http://libhugetlbfs.ozlabs.org/releases/libhugetlbfs-1.3-pre1.tar.gz
- def setup(self, tarball = 'libhugetlbfs-1.3-pre1.tar.gz'):
- tarball = utils.unmap_url(self.bindir, tarball,
- self.tmpdir)
- autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
- os.chdir(self.srcdir)
+ # http://libhugetlbfs.ozlabs.org/releases/libhugetlbfs-1.3-pre1.tar.gz
+ def setup(self, tarball = 'libhugetlbfs-1.3-pre1.tar.gz'):
+ tarball = utils.unmap_url(self.bindir, tarball,
+ self.tmpdir)
+ autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
+ os.chdir(self.srcdir)
- # make might fail if there are no proper headers for the 32 bit
- # version, in that case try only for the 64 bit version
- try:
- utils.system('make')
- except:
- utils.system('make OBJDIRS=obj64')
+ # make might fail if there are no proper headers for the 32 bit
+ # version, in that case try only for the 64 bit version
+ try:
+ utils.system('make')
+ except:
+ utils.system('make OBJDIRS=obj64')
- def execute(self, dir = None, pages_requested = 20):
- autotest_utils.check_kernel_ver("2.6.16")
+ def execute(self, dir = None, pages_requested = 20):
+ autotest_utils.check_kernel_ver("2.6.16")
- # Check huge page number
- pages_available = 0
- if os.path.exists('/proc/sys/vm/nr_hugepages'):
- utils.write_one_line('/proc/sys/vm/nr_hugepages',
- str(pages_requested))
- pages_available = int(open('/proc/sys/vm/nr_hugepages', 'r').readline())
- else:
- raise error.TestNAError('Kernel does not support hugepages')
+ # Check huge page number
+ pages_available = 0
+ if os.path.exists('/proc/sys/vm/nr_hugepages'):
+ utils.write_one_line('/proc/sys/vm/nr_hugepages',
+ str(pages_requested))
+ pages_available = int(open('/proc/sys/vm/nr_hugepages', 'r').readline())
+ else:
+ raise error.TestNAError('Kernel does not support hugepages')
- if pages_available < pages_requested:
- raise error.TestError('%d huge pages available, < %d pages requested' % (pages_available, pages_requested))
+ if pages_available < pages_requested:
+ raise error.TestError('%d huge pages available, < %d pages requested' % (pages_available, pages_requested))
- # Check if hugetlbfs has been mounted
- if not autotest_utils.file_contains_pattern('/proc/mounts', 'hugetlbfs'):
- if not dir:
- dir = os.path.join(self.tmpdir, 'hugetlbfs')
- os.makedirs(dir)
- utils.system('mount -t hugetlbfs none %s' % dir)
+ # Check if hugetlbfs has been mounted
+ if not autotest_utils.file_contains_pattern('/proc/mounts', 'hugetlbfs'):
+ if not dir:
+ dir = os.path.join(self.tmpdir, 'hugetlbfs')
+ os.makedirs(dir)
+ utils.system('mount -t hugetlbfs none %s' % dir)
- os.chdir(self.srcdir)
+ os.chdir(self.srcdir)
- profilers = self.job.profilers
- if profilers.present():
- profilers.start(self)
- os.chdir(self.srcdir)
- # make check might fail for 32 bit if the 32 bit compile earlier
- # had failed. See if it passes for 64 bit in that case.
- try:
- utils.system('make check')
- except:
- utils.system('make check OBJDIRS=obj64')
- if profilers.present():
- profilers.stop(self)
- profilers.report(self)
+ profilers = self.job.profilers
+ if profilers.present():
+ profilers.start(self)
+ os.chdir(self.srcdir)
+ # make check might fail for 32 bit if the 32 bit compile earlier
+ # had failed. See if it passes for 64 bit in that case.
+ try:
+ utils.system('make check')
+ except:
+ utils.system('make check OBJDIRS=obj64')
+ if profilers.present():
+ profilers.stop(self)
+ profilers.report(self)
- utils.system('umount %s' % dir)
+ utils.system('umount %s' % dir)
diff --git a/client/tests/linus_stress/linus_stress.py b/client/tests/linus_stress/linus_stress.py
index 3a81134..6160307 100755
--- a/client/tests/linus_stress/linus_stress.py
+++ b/client/tests/linus_stress/linus_stress.py
@@ -4,40 +4,40 @@
class linus_stress(test.test):
- version = 1
+ version = 1
- def setup(self):
- os.mkdir(self.srcdir)
- os.chdir(self.bindir)
- utils.system('cp linus_stress.c src/')
- os.chdir(self.srcdir)
- utils.system('cc linus_stress.c -D_POSIX_C_SOURCE=200112 -o linus_stress')
+ def setup(self):
+ os.mkdir(self.srcdir)
+ os.chdir(self.bindir)
+ utils.system('cp linus_stress.c src/')
+ os.chdir(self.srcdir)
+ utils.system('cc linus_stress.c -D_POSIX_C_SOURCE=200112 -o linus_stress')
- def run_the_test(self, iterations):
- utils.write_one_line('/proc/sys/vm/dirty_ratio', '4')
- utils.write_one_line('/proc/sys/vm/dirty_background_ratio', '2')
+ def run_the_test(self, iterations):
+ utils.write_one_line('/proc/sys/vm/dirty_ratio', '4')
+ utils.write_one_line('/proc/sys/vm/dirty_background_ratio', '2')
- cmd = os.path.join(self.srcdir, 'linus_stress')
- args = "%d" % (autotest_utils.memtotal() / 32)
+ cmd = os.path.join(self.srcdir, 'linus_stress')
+ args = "%d" % (autotest_utils.memtotal() / 32)
- profilers = self.job.profilers
- if profilers.present():
- profilers.start(self)
+ profilers = self.job.profilers
+ if profilers.present():
+ profilers.start(self)
- for i in range(iterations):
- utils.system(cmd + ' ' + args)
+ for i in range(iterations):
+ utils.system(cmd + ' ' + args)
- if profilers.present():
- profilers.stop(self)
- profilers.report(self)
+ if profilers.present():
+ profilers.stop(self)
+ profilers.report(self)
- def execute(self, iterations = 1):
- dirty_ratio = utils.read_one_line('/proc/sys/vm/dirty_ratio')
- dirty_background_ratio = utils.read_one_line('/proc/sys/vm/dirty_background_ratio')
- try:
- self.run_the_test(iterations)
- finally:
- utils.write_one_line('/proc/sys/vm/dirty_ratio', dirty_ratio)
- utils.write_one_line('/proc/sys/vm/dirty_background_ratio', dirty_background_ratio)
+ def execute(self, iterations = 1):
+ dirty_ratio = utils.read_one_line('/proc/sys/vm/dirty_ratio')
+ dirty_background_ratio = utils.read_one_line('/proc/sys/vm/dirty_background_ratio')
+ try:
+ self.run_the_test(iterations)
+ finally:
+ utils.write_one_line('/proc/sys/vm/dirty_ratio', dirty_ratio)
+ utils.write_one_line('/proc/sys/vm/dirty_background_ratio', dirty_background_ratio)
diff --git a/client/tests/lmbench/lmbench.py b/client/tests/lmbench/lmbench.py
index fc4807b..a4bef45 100755
--- a/client/tests/lmbench/lmbench.py
+++ b/client/tests/lmbench/lmbench.py
@@ -5,44 +5,44 @@
class lmbench(test.test):
- version = 2
+ version = 2
- def setup(self, tarball = 'lmbench3.tar.bz2'):
- tarball = utils.unmap_url(self.bindir, tarball,
- self.tmpdir)
- # http://www.bitmover.com/lm/lmbench/lmbench3.tar.gz
- # + lmbench3.diff
- # removes Makefile references to bitkeeper
- # default mail to no, fix job placement defaults (masouds)
- autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
- os.chdir(self.srcdir)
+ def setup(self, tarball = 'lmbench3.tar.bz2'):
+ tarball = utils.unmap_url(self.bindir, tarball,
+ self.tmpdir)
+ # http://www.bitmover.com/lm/lmbench/lmbench3.tar.gz
+ # + lmbench3.diff
+ # removes Makefile references to bitkeeper
+ # default mail to no, fix job placement defaults (masouds)
+ autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
+ os.chdir(self.srcdir)
- utils.system('make')
+ utils.system('make')
- def execute(self, iterations = 1, mem = '', fastmem = 'NO',
- slowfs = 'NO', disks = '', disks_desc = '',
- mhz = '', remote = '', enough = '5000', sync_max = '1',
- fsdir = None, file = None):
- if not fsdir:
- fsdir = self.tmpdir
- if not file:
- file = self.tmpdir+'XXX'
+ def execute(self, iterations = 1, mem = '', fastmem = 'NO',
+ slowfs = 'NO', disks = '', disks_desc = '',
+ mhz = '', remote = '', enough = '5000', sync_max = '1',
+ fsdir = None, file = None):
+ if not fsdir:
+ fsdir = self.tmpdir
+ if not file:
+ file = self.tmpdir+'XXX'
- os.chdir(self.srcdir)
- cmd = "yes '' | make rerun"
- profilers = self.job.profilers
- if not profilers.only():
- for i in range(iterations):
- utils.system(cmd)
+ os.chdir(self.srcdir)
+ cmd = "yes '' | make rerun"
+ profilers = self.job.profilers
+ if not profilers.only():
+ for i in range(iterations):
+ utils.system(cmd)
- # Do a profiling run if necessary
- if profilers.present():
- profilers.start(self)
- utils.system(cmd)
- profilers.stop(self)
- profilers.report(self)
- # Get the results:
- outputdir = self.srcdir + "/results"
- results = self.resultsdir + "/summary.txt"
- utils.system("make -C " + outputdir + " summary > " + results)
+ # Do a profiling run if necessary
+ if profilers.present():
+ profilers.start(self)
+ utils.system(cmd)
+ profilers.stop(self)
+ profilers.report(self)
+ # Get the results:
+ outputdir = self.srcdir + "/results"
+ results = self.resultsdir + "/summary.txt"
+ utils.system("make -C " + outputdir + " summary > " + results)
diff --git a/client/tests/lsb_dtk/lsb_dtk.py b/client/tests/lsb_dtk/lsb_dtk.py
index 0d1db50..f8d25df 100644
--- a/client/tests/lsb_dtk/lsb_dtk.py
+++ b/client/tests/lsb_dtk/lsb_dtk.py
@@ -12,154 +12,154 @@
'''
class lsb_dtk(test.test):
- version = 1
- def get_lsb_arch(self):
- self.arch = autotest_utils.get_current_kernel_arch()
- if self.arch in ['i386', 'i486', 'i586', 'i686', 'athlon']:
- return 'ia32'
- elif self.arch == 'ppc':
- return 'ppc32'
- elif self.arch in ['s390', 's390x', 'ia64', 'x86_64', 'ppc64']:
- return self.arch
- else:
- e_msg = 'Architecture %s not supported by LSB' % self.arch
- raise error.TestError(e_msg)
+ version = 1
+ def get_lsb_arch(self):
+ self.arch = autotest_utils.get_current_kernel_arch()
+ if self.arch in ['i386', 'i486', 'i586', 'i686', 'athlon']:
+ return 'ia32'
+ elif self.arch == 'ppc':
+ return 'ppc32'
+ elif self.arch in ['s390', 's390x', 'ia64', 'x86_64', 'ppc64']:
+ return self.arch
+ else:
+ e_msg = 'Architecture %s not supported by LSB' % self.arch
+ raise error.TestError(e_msg)
- def install_lsb_packages(self, srcdir, cachedir, my_config):
- # First, we download the LSB DTK manager package, worry about installing it later
- self.dtk_manager_arch = my_config.get('dtk-manager', 'arch-%s' % self.get_lsb_arch())
- self.dtk_manager_url = my_config.get('dtk-manager', 'tarball_url') % self.dtk_manager_arch
- if not self.dtk_manager_url:
- raise error.TestError('Could not get DTK manager URL from configuration file')
- self.dtk_md5 = my_config.get('dtk-manager', 'md5-%s' % self.get_lsb_arch())
- if self.dtk_md5:
- print 'Caching LSB DTK manager RPM'
- self.dtk_manager_pkg = autotest_utils.unmap_url_cache(cachedir, self.dtk_manager_url, self.dtk_md5)
- else:
- raise error.TestError('Could not find DTK manager package md5, cannot cache DTK manager tarball')
+ def install_lsb_packages(self, srcdir, cachedir, my_config):
+ # First, we download the LSB DTK manager package, worry about installing it later
+ self.dtk_manager_arch = my_config.get('dtk-manager', 'arch-%s' % self.get_lsb_arch())
+ self.dtk_manager_url = my_config.get('dtk-manager', 'tarball_url') % self.dtk_manager_arch
+ if not self.dtk_manager_url:
+ raise error.TestError('Could not get DTK manager URL from configuration file')
+ self.dtk_md5 = my_config.get('dtk-manager', 'md5-%s' % self.get_lsb_arch())
+ if self.dtk_md5:
+ print 'Caching LSB DTK manager RPM'
+ self.dtk_manager_pkg = autotest_utils.unmap_url_cache(cachedir, self.dtk_manager_url, self.dtk_md5)
+ else:
+ raise error.TestError('Could not find DTK manager package md5, cannot cache DTK manager tarball')
- # Get LSB tarball, cache it and uncompress under autotest srcdir
- if my_config.get('lsb', 'override_default_url') == 'no':
- self.lsb_url = my_config.get('lsb', 'tarball_url') % self.get_lsb_arch()
- else:
- self.lsb_url = my_config.get('lsb', 'tarball_url_alt') % self.get_lsb_arch()
- if not self.lsb_url:
- raise TestError('Could not get lsb URL from configuration file')
- self.md5_key = 'md5-%s' % self.get_lsb_arch()
- self.lsb_md5 = my_config.get('lsb', self.md5_key)
- if self.lsb_md5:
- print 'Caching LSB tarball'
- self.lsb_pkg = autotest_utils.unmap_url_cache(self.cachedir, self.lsb_url, self.lsb_md5)
- else:
- raise error.TestError('Could not find LSB package md5, cannot cache LSB tarball')
+ # Get LSB tarball, cache it and uncompress under autotest srcdir
+ if my_config.get('lsb', 'override_default_url') == 'no':
+ self.lsb_url = my_config.get('lsb', 'tarball_url') % self.get_lsb_arch()
+ else:
+ self.lsb_url = my_config.get('lsb', 'tarball_url_alt') % self.get_lsb_arch()
+ if not self.lsb_url:
+ raise TestError('Could not get lsb URL from configuration file')
+ self.md5_key = 'md5-%s' % self.get_lsb_arch()
+ self.lsb_md5 = my_config.get('lsb', self.md5_key)
+ if self.lsb_md5:
+ print 'Caching LSB tarball'
+ self.lsb_pkg = autotest_utils.unmap_url_cache(self.cachedir, self.lsb_url, self.lsb_md5)
+ else:
+ raise error.TestError('Could not find LSB package md5, cannot cache LSB tarball')
- autotest_utils.extract_tarball_to_dir(self.lsb_pkg, srcdir)
+ autotest_utils.extract_tarball_to_dir(self.lsb_pkg, srcdir)
- # Lets load a file that contains the list of RPMs
- os.chdir(srcdir)
- if not os.path.isfile('inst-config'):
- raise IOError('Could not find file with package info, inst-config')
- self.rpm_file_list = open('inst-config', 'r')
- self.pkg_pattern = re.compile('[A-Za-z0-9_.-]*[.][r][p][m]')
- self.lsb_pkg_list = []
- for self.line in self.rpm_file_list.readlines():
- try:
- # We will install lsb-dtk-manager separately, so we can remove
- # it from the list of packages
- if not 'lsb-dtk-manager' in self.line:
- self.line = re.findall(self.pkg_pattern, self.line)[0]
- self.lsb_pkg_list.append(self.line)
- except:
- # If we don't get a match, no problem
- pass
+ # Lets load a file that contains the list of RPMs
+ os.chdir(srcdir)
+ if not os.path.isfile('inst-config'):
+ raise IOError('Could not find file with package info, inst-config')
+ self.rpm_file_list = open('inst-config', 'r')
+ self.pkg_pattern = re.compile('[A-Za-z0-9_.-]*[.][r][p][m]')
+ self.lsb_pkg_list = []
+ for self.line in self.rpm_file_list.readlines():
+ try:
+ # We will install lsb-dtk-manager separately, so we can remove
+ # it from the list of packages
+ if not 'lsb-dtk-manager' in self.line:
+ self.line = re.findall(self.pkg_pattern, self.line)[0]
+ self.lsb_pkg_list.append(self.line)
+ except:
+ # If we don't get a match, no problem
+ pass
- # Lets figure out the host distro
- distro_pkg_support = package.os_support()
- if os.path.isfile('/etc/debian_version') and distro_pkg_support['dpkg']:
- print 'Debian based distro detected'
- if distro_pkg_support['conversion']:
- print 'Package conversion supported'
- self.distro_type = 'debian-based'
- else:
- e_msg = 'Package conversion not supported. Cannot handle LSB package installation'
- raise EnvironmentError(e_msg)
- elif distro_pkg_support['rpm']:
- print 'Red Hat based distro detected'
- self.distro_type = 'redhat-based'
- else:
- print 'OS does not seem to be red hat or debian based'
- e_msg = 'Cannot handle LSB package installation'
- raise EnvironmentError(e_msg)
+ # Lets figure out the host distro
+ distro_pkg_support = package.os_support()
+ if os.path.isfile('/etc/debian_version') and distro_pkg_support['dpkg']:
+ print 'Debian based distro detected'
+ if distro_pkg_support['conversion']:
+ print 'Package conversion supported'
+ self.distro_type = 'debian-based'
+ else:
+ e_msg = 'Package conversion not supported. Cannot handle LSB package installation'
+ raise EnvironmentError(e_msg)
+ elif distro_pkg_support['rpm']:
+ print 'Red Hat based distro detected'
+ self.distro_type = 'redhat-based'
+ else:
+ print 'OS does not seem to be red hat or debian based'
+ e_msg = 'Cannot handle LSB package installation'
+ raise EnvironmentError(e_msg)
- # According to the host distro detection, we can install the packages
- # using the list previously assembled
- if self.distro_type == 'redhat-based':
- print 'Installing LSB RPM packages'
- package.install(self.dtk_manager_pkg)
- for self.lsb_rpm in self.lsb_pkg_list:
- package.install(self.lsb_rpm, nodeps = True)
- elif self.distro_type == 'debian-based':
- print 'Remember that you must have the following lsb compliance packages installed:'
- print 'lsb-core lsb-cxx lsb-graphics lsb-desktop lsb-qt4 lsb-languages lsb-multimedia lsb-printing'
- print 'Converting and installing LSB packages'
- self.dtk_manager_dpkg = package.convert(self.dtk_manager_pkg, 'dpkg')
- package.install(self.dtk_manager_dpkg)
- for self.lsb_rpm in self.lsb_pkg_list:
- self.lsb_dpkg = package.convert(self.lsb_rpm, 'dpkg')
- package.install(self.lsb_dpkg, nodeps = True)
+ # According to the host distro detection, we can install the packages
+ # using the list previously assembled
+ if self.distro_type == 'redhat-based':
+ print 'Installing LSB RPM packages'
+ package.install(self.dtk_manager_pkg)
+ for self.lsb_rpm in self.lsb_pkg_list:
+ package.install(self.lsb_rpm, nodeps = True)
+ elif self.distro_type == 'debian-based':
+ print 'Remember that you must have the following lsb compliance packages installed:'
+ print 'lsb-core lsb-cxx lsb-graphics lsb-desktop lsb-qt4 lsb-languages lsb-multimedia lsb-printing'
+ print 'Converting and installing LSB packages'
+ self.dtk_manager_dpkg = package.convert(self.dtk_manager_pkg, 'dpkg')
+ package.install(self.dtk_manager_dpkg)
+ for self.lsb_rpm in self.lsb_pkg_list:
+ self.lsb_dpkg = package.convert(self.lsb_rpm, 'dpkg')
+ package.install(self.lsb_dpkg, nodeps = True)
- def link_lsb_libraries(self, config):
- print 'Linking LSB libraries'
- self.libdir_key = 'libdir-%s' % self.get_lsb_arch()
- self.os_libdir = config.get('lib', self.libdir_key)
- if not self.os_libdir:
- raise TypeError('Could not find OS lib dir from conf file')
- self.lib_key = 'lib-%s' % self.get_lsb_arch()
- self.lib_list_raw = config.get('lib', self.lib_key)
- if not self.lib_list_raw:
- raise TypeError('Could not find library list from conf file')
- self.lib_list = eval(self.lib_list_raw)
+ def link_lsb_libraries(self, config):
+ print 'Linking LSB libraries'
+ self.libdir_key = 'libdir-%s' % self.get_lsb_arch()
+ self.os_libdir = config.get('lib', self.libdir_key)
+ if not self.os_libdir:
+ raise TypeError('Could not find OS lib dir from conf file')
+ self.lib_key = 'lib-%s' % self.get_lsb_arch()
+ self.lib_list_raw = config.get('lib', self.lib_key)
+ if not self.lib_list_raw:
+ raise TypeError('Could not find library list from conf file')
+ self.lib_list = eval(self.lib_list_raw)
- # Remove any previous ld-lsb*.so symbolic links
- self.lsb_libs = glob.glob('%s/ld-lsb*.so*' % self.os_libdir)
- for self.lib in self.lsb_libs:
- os.remove(self.lib)
+ # Remove any previous ld-lsb*.so symbolic links
+ self.lsb_libs = glob.glob('%s/ld-lsb*.so*' % self.os_libdir)
+ for self.lib in self.lsb_libs:
+ os.remove(self.lib)
- # Get the base library that we'll use to recreate the symbolic links
- self.system_lib = glob.glob('%s/ld-2*.so*' % self.os_libdir)[0]
+ # Get the base library that we'll use to recreate the symbolic links
+ self.system_lib = glob.glob('%s/ld-2*.so*' % self.os_libdir)[0]
- # Now just link the system lib that we just found to each one of the
- # needed LSB libraries that we provided on the conf file
- for self.lsb_lib in self.lib_list:
- # Get the library absolute path
- self.lsb_lib = os.path.join(self.os_libdir, self.lsb_lib)
- # Link the library system_lib -> lsb_lib
- os.symlink(self.system_lib, self.lsb_lib)
+ # Now just link the system lib that we just found to each one of the
+ # needed LSB libraries that we provided on the conf file
+ for self.lsb_lib in self.lib_list:
+ # Get the library absolute path
+ self.lsb_lib = os.path.join(self.os_libdir, self.lsb_lib)
+ # Link the library system_lib -> lsb_lib
+ os.symlink(self.system_lib, self.lsb_lib)
- def execute(self, args = 'all', config = './lsb31.cfg'):
- # Load configuration. Use autotest tmpdir if needed
- my_config = config_loader(config, self.tmpdir)
- # Cache directory, that will store LSB tarball and DTK manager RPM
- self.cachedir = os.path.join(self.bindir, 'cache')
- if not os.path.isdir(self.cachedir):
- os.makedirs(self.cachedir)
+ def execute(self, args = 'all', config = './lsb31.cfg'):
+ # Load configuration. Use autotest tmpdir if needed
+ my_config = config_loader(config, self.tmpdir)
+ # Cache directory, that will store LSB tarball and DTK manager RPM
+ self.cachedir = os.path.join(self.bindir, 'cache')
+ if not os.path.isdir(self.cachedir):
+ os.makedirs(self.cachedir)
- self.install_lsb_packages(self.srcdir, self.cachedir, my_config)
- self.link_lsb_libraries(my_config)
+ self.install_lsb_packages(self.srcdir, self.cachedir, my_config)
+ self.link_lsb_libraries(my_config)
- self.main_script_path = my_config.get('lsb', 'main_script_path')
- logfile = os.path.join(self.resultsdir, 'lsb.log')
- args2 = '-r %s' % (logfile)
- args = args + ' ' + args2
- cmd = os.path.join(self.srcdir, self.main_script_path) + ' ' + args
+ self.main_script_path = my_config.get('lsb', 'main_script_path')
+ logfile = os.path.join(self.resultsdir, 'lsb.log')
+ args2 = '-r %s' % (logfile)
+ args = args + ' ' + args2
+ cmd = os.path.join(self.srcdir, self.main_script_path) + ' ' + args
- profilers = self.job.profilers
- if profilers.present():
- profilers.start(self)
- print 'Executing LSB main test script'
- utils.system(cmd)
- if profilers.present():
- profilers.stop(self)
- profilers.report(self)
+ profilers = self.job.profilers
+ if profilers.present():
+ profilers.start(self)
+ print 'Executing LSB main test script'
+ utils.system(cmd)
+ if profilers.present():
+ profilers.stop(self)
+ profilers.report(self)
diff --git a/client/tests/ltp/ltp-diff.py b/client/tests/ltp/ltp-diff.py
index 12031a3..d0f0f9f 100644
--- a/client/tests/ltp/ltp-diff.py
+++ b/client/tests/ltp/ltp-diff.py
@@ -4,106 +4,106 @@
# Description:
# Input: Two or more files containing results from different executions of
# the LTP. The input can either be file names or the url location
-# of the ltp.results file.
+# of the ltp.results file.
# Output: A report on the following:
-# - The total number of tests executed in each run
-# - The testname, sequence number, and output of each run
+# - The total number of tests executed in each run
+# - The testname, sequence number, and output of each run
# where the results of those runs differ
# Return:
-# 0 if all runs had identical results
-# Non-zero if results differ, or bad input
+# 0 if all runs had identical results
+# Non-zero if results differ, or bad input
import sys, string, re
from autotest_lib.client.comon_lib import utils
def usage():
- print "\nUsage: \n\
- ltp-diff results1 results2 ... locationN \n\
- Note: location[1,2,N] may be local files or URLs of LTP results\n"
- sys.exit(1)
+ print "\nUsage: \n\
+ltp-diff results1 results2 ... locationN \n\
+Note: location[1,2,N] may be local files or URLs of LTP results\n"
+ sys.exit(1)
def get_results(results_files):
- """
- Download the results if needed.
- Return results of each run in a numerically-indexed dictionary
- of dictionaries keyed on testnames.
- Return dictionary keyed on unique testnames across all runs.
- """
- r = re.compile('(\S+\s+\S+)\s+(\S+)\s+:')
- i = 0
- runs = {}
- testnames = {}
- for file in results_files:
- runs[i] = {}
- try:
- fh = utils.urlopen(file)
- results = fh.readlines()
- fh.close()
- except:
- print "ERROR: reading results resource [%s]" % (file)
- usage()
- for line in results:
- try:
- s = r.match(line)
- testname = s.group(1)
- status = s.group(2)
- runs[i][testname] = status
- testnames[testname] = 1
- except:
- pass
- i += 1
- return (runs, testnames)
+ """
+ Download the results if needed.
+ Return results of each run in a numerically-indexed dictionary
+ of dictionaries keyed on testnames.
+ Return dictionary keyed on unique testnames across all runs.
+ """
+ r = re.compile('(\S+\s+\S+)\s+(\S+)\s+:')
+ i = 0
+ runs = {}
+ testnames = {}
+ for file in results_files:
+ runs[i] = {}
+ try:
+ fh = utils.urlopen(file)
+ results = fh.readlines()
+ fh.close()
+ except:
+ print "ERROR: reading results resource [%s]" % (file)
+ usage()
+ for line in results:
+ try:
+ s = r.match(line)
+ testname = s.group(1)
+ status = s.group(2)
+ runs[i][testname] = status
+ testnames[testname] = 1
+ except:
+ pass
+ i += 1
+ return (runs, testnames)
def compare_results(runs):
- """
- Loop through all testnames alpahbetically.
- Print any testnames with differing results across runs.
- Return 1 if any test results across runs differ.
- Return 0 if all test results match.
- """
- rc = 0
- print "LTP Test Results to Compare"
- for i in range(len(runs)):
- print " Run[%d]: %d" % (i, len(runs[i].keys()))
- print ""
- header = 0
- all_testnames = testnames.keys()
- all_testnames.sort()
- for testname in all_testnames:
- differ = 0
- for i in range(1,len(runs)):
- # Must handle testcases that executed in one run
- # but not another by setting status to "null"
- if not runs[i].has_key(testname):
- runs[i][testname] = "null"
- if not runs[i-1].has_key(testname):
- runs[i-1][testname] = "null"
- # Check for the results inconsistencies
- if runs[i][testname] != runs[i-1][testname]:
- differ = 1
- if differ:
- if header == 0:
- # Print the differences header only once
- print "Tests with Inconsistent Results across Runs"
- print " %-35s:\t%s" % ("Testname,Sequence", "Run Results")
- header = 1
+ """
+ Loop through all testnames alpahbetically.
+ Print any testnames with differing results across runs.
+ Return 1 if any test results across runs differ.
+ Return 0 if all test results match.
+ """
+ rc = 0
+ print "LTP Test Results to Compare"
+ for i in range(len(runs)):
+ print " Run[%d]: %d" % (i, len(runs[i].keys()))
+ print ""
+ header = 0
+ all_testnames = testnames.keys()
+ all_testnames.sort()
+ for testname in all_testnames:
+ differ = 0
+ for i in range(1,len(runs)):
+ # Must handle testcases that executed in one run
+ # but not another by setting status to "null"
+ if not runs[i].has_key(testname):
+ runs[i][testname] = "null"
+ if not runs[i-1].has_key(testname):
+ runs[i-1][testname] = "null"
+ # Check for the results inconsistencies
+ if runs[i][testname] != runs[i-1][testname]:
+ differ = 1
+ if differ:
+ if header == 0:
+ # Print the differences header only once
+ print "Tests with Inconsistent Results across Runs"
+ print " %-35s:\t%s" % ("Testname,Sequence", "Run Results")
+ header = 1
- # Print info if results differ
- rc = 1
- testname_cleaned = re.sub('\s+', ',', testname)
- print " %-35s:\t" % (testname_cleaned),
- all_results = ""
- for i in range(len(runs)):
- all_results += runs[i][testname]
- if i+1<len(runs):
- all_results += "/"
- print all_results
- if rc == 0:
- print "All LTP results are identical"
- return rc
+ # Print info if results differ
+ rc = 1
+ testname_cleaned = re.sub('\s+', ',', testname)
+ print " %-35s:\t" % (testname_cleaned),
+ all_results = ""
+ for i in range(len(runs)):
+ all_results += runs[i][testname]
+ if i+1<len(runs):
+ all_results += "/"
+ print all_results
+ if rc == 0:
+ print "All LTP results are identical"
+ return rc
########
@@ -111,7 +111,7 @@
########
sys.argv.pop(0)
if (len(sys.argv) < 2):
- usage()
+ usage()
(runs, testnames) = get_results(sys.argv)
rc = compare_results(runs)
sys.exit(rc)
diff --git a/client/tests/ltp/ltp.py b/client/tests/ltp/ltp.py
index 043ac00..2417f46 100755
--- a/client/tests/ltp/ltp.py
+++ b/client/tests/ltp/ltp.py
@@ -3,46 +3,46 @@
from autotest_lib.client.common_lib import utils, error
class ltp(test.test):
- version = 4
+ version = 4
- # http://prdownloads.sourceforge.net/ltp/ltp-full-20080229.tgz
- def setup(self, tarball = 'ltp-full-20080229.tar.bz2'):
- tarball = utils.unmap_url(self.bindir, tarball,
- self.tmpdir)
- autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
- os.chdir(self.srcdir)
+ # http://prdownloads.sourceforge.net/ltp/ltp-full-20080229.tgz
+ def setup(self, tarball = 'ltp-full-20080229.tar.bz2'):
+ tarball = utils.unmap_url(self.bindir, tarball,
+ self.tmpdir)
+ autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
+ os.chdir(self.srcdir)
- utils.system('patch -p1 < ../ltp.patch')
+ utils.system('patch -p1 < ../ltp.patch')
- # comment the capability tests if we fail to load the capability module
- try:
- utils.system('modprobe capability')
- except error.CmdError, detail:
- utils.system('patch -p1 < ../ltp_capability.patch')
+ # comment the capability tests if we fail to load the capability module
+ try:
+ utils.system('modprobe capability')
+ except error.CmdError, detail:
+ utils.system('patch -p1 < ../ltp_capability.patch')
- utils.system('cp ../scan.c pan/') # saves having lex installed
- utils.system('make -j %d' % autotest_utils.count_cpus())
- utils.system('yes n | make install')
+ utils.system('cp ../scan.c pan/') # saves having lex installed
+ utils.system('make -j %d' % autotest_utils.count_cpus())
+ utils.system('yes n | make install')
- # Note: to run a specific test, try '-f cmdfile -s test' in the
- # in the args (-f for test file and -s for the test case)
- # eg, job.run_test('ltp', '-f math -s float_bessel')
- def execute(self, args = '', script = 'runltp'):
+ # Note: to run a specific test, try '-f cmdfile -s test' in the
+ # in the args (-f for test file and -s for the test case)
+ # eg, job.run_test('ltp', '-f math -s float_bessel')
+ def execute(self, args = '', script = 'runltp'):
- # In case the user wants to run another test script
- if script == 'runltp':
- logfile = os.path.join(self.resultsdir, 'ltp.log')
- failcmdfile = os.path.join(self.debugdir, 'failcmdfile')
- args2 = '-q -l %s -C %s -d %s' % (logfile, failcmdfile, self.tmpdir)
- args = args + ' ' + args2
+ # In case the user wants to run another test script
+ if script == 'runltp':
+ logfile = os.path.join(self.resultsdir, 'ltp.log')
+ failcmdfile = os.path.join(self.debugdir, 'failcmdfile')
+ args2 = '-q -l %s -C %s -d %s' % (logfile, failcmdfile, self.tmpdir)
+ args = args + ' ' + args2
- cmd = os.path.join(self.srcdir, script) + ' ' + args
+ cmd = os.path.join(self.srcdir, script) + ' ' + args
- profilers = self.job.profilers
- if profilers.present():
- profilers.start(self)
- utils.system(cmd)
- if profilers.present():
- profilers.stop(self)
- profilers.report(self)
+ profilers = self.job.profilers
+ if profilers.present():
+ profilers.start(self)
+ utils.system(cmd)
+ if profilers.present():
+ profilers.stop(self)
+ profilers.report(self)
diff --git a/client/tests/netperf2/control.client b/client/tests/netperf2/control.client
index bfff402..7c2ecac 100644
--- a/client/tests/netperf2/control.client
+++ b/client/tests/netperf2/control.client
@@ -1,2 +1 @@
job.run_test('netperf2', '10.10.1.2', '10.10.1.6', 'client', tag='client')
-
diff --git a/client/tests/netperf2/control.parallel b/client/tests/netperf2/control.parallel
index c845879..8414a3a 100644
--- a/client/tests/netperf2/control.parallel
+++ b/client/tests/netperf2/control.parallel
@@ -1,7 +1,7 @@
def client():
- job.run_test('netperf2', '127.0.0.1', '127.0.0.1', 'client', tag='client')
+ job.run_test('netperf2', '127.0.0.1', '127.0.0.1', 'client', tag='client')
def server():
- job.run_test('netperf2', '127.0.0.1', '127.0.0.1', 'server', tag='server')
+ job.run_test('netperf2', '127.0.0.1', '127.0.0.1', 'server', tag='server')
job.parallel([server], [client])
diff --git a/client/tests/netperf2/netperf2.py b/client/tests/netperf2/netperf2.py
index 1f7fb66..e01f815 100755
--- a/client/tests/netperf2/netperf2.py
+++ b/client/tests/netperf2/netperf2.py
@@ -4,75 +4,75 @@
class netperf2(test.test):
- version = 1
+ version = 1
- # ftp://ftp.netperf.org/netperf/netperf-2.4.1.tar.gz
- def setup(self, tarball = 'netperf-2.4.1.tar.gz'):
- tarball = utils.unmap_url(self.bindir, tarball,
- self.tmpdir)
- autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
- os.chdir(self.srcdir)
+ # ftp://ftp.netperf.org/netperf/netperf-2.4.1.tar.gz
+ def setup(self, tarball = 'netperf-2.4.1.tar.gz'):
+ tarball = utils.unmap_url(self.bindir, tarball,
+ self.tmpdir)
+ autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
+ os.chdir(self.srcdir)
- utils.system('./configure')
- utils.system('make')
+ utils.system('./configure')
+ utils.system('make')
- def initialize(self):
- # netserver doesn't detach properly from the console. When
- # it is run from ssh, this causes the ssh command not to
- # return even though netserver meant to be backgrounded.
- # This behavior is remedied by redirecting fd 0, 1 & 2
- self.server_path = ('%s &>/dev/null </dev/null'
- % os.path.join(self.srcdir, 'src/netserver'))
- self.client_path = os.path.join(self.srcdir, 'src/netperf')
+ def initialize(self):
+ # netserver doesn't detach properly from the console. When
+ # it is run from ssh, this causes the ssh command not to
+ # return even though netserver meant to be backgrounded.
+ # This behavior is remedied by redirecting fd 0, 1 & 2
+ self.server_path = ('%s &>/dev/null </dev/null'
+ % os.path.join(self.srcdir, 'src/netserver'))
+ self.client_path = os.path.join(self.srcdir, 'src/netperf')
- def execute(self, server_ip, client_ip, role,
- script='snapshot_script', args=''):
- server_tag = server_ip + '#netperf-server'
- client_tag = client_ip + '#netperf-client'
- all = [server_tag, client_tag]
- job = self.job
- if (role == 'server'):
- self.server_start()
- try:
- job.barrier(server_tag, 'start',
- 600).rendevous(*all)
- job.barrier(server_tag, 'stop',
- 3600).rendevous(*all)
- finally:
- self.server_stop()
- elif (role == 'client'):
- os.environ['NETPERF_CMD'] = self.client_path
- job.barrier(client_tag, 'start', 600).rendevous(*all)
- self.client(script, server_ip, args)
- job.barrier(client_tag, 'stop', 30).rendevous(*all)
- else:
- raise error.UnhandledError('invalid role specified')
+ def execute(self, server_ip, client_ip, role,
+ script='snapshot_script', args=''):
+ server_tag = server_ip + '#netperf-server'
+ client_tag = client_ip + '#netperf-client'
+ all = [server_tag, client_tag]
+ job = self.job
+ if (role == 'server'):
+ self.server_start()
+ try:
+ job.barrier(server_tag, 'start',
+ 600).rendevous(*all)
+ job.barrier(server_tag, 'stop',
+ 3600).rendevous(*all)
+ finally:
+ self.server_stop()
+ elif (role == 'client'):
+ os.environ['NETPERF_CMD'] = self.client_path
+ job.barrier(client_tag, 'start', 600).rendevous(*all)
+ self.client(script, server_ip, args)
+ job.barrier(client_tag, 'stop', 30).rendevous(*all)
+ else:
+ raise error.UnhandledError('invalid role specified')
- def server_start(self):
- # we should really record the pid we forked off, but there
- # was no obvious way to run the daemon in the foreground.
- # Hacked it for now
- system('killall netserver', ignore_status=True)
- system(self.server_path)
+ def server_start(self):
+ # we should really record the pid we forked off, but there
+ # was no obvious way to run the daemon in the foreground.
+ # Hacked it for now
+ system('killall netserver', ignore_status=True)
+ system(self.server_path)
- def server_stop(self):
- # this should really just kill the pid I forked, but ...
- system('killall netserver')
+ def server_stop(self):
+ # this should really just kill the pid I forked, but ...
+ system('killall netserver')
- def client(self, script, server_ip, args = 'CPU'):
- # run some client stuff
- stdout_path = os.path.join(self.resultsdir, script + '.stdout')
- stderr_path = os.path.join(self.resultsdir, script + '.stderr')
- self.job.stdout.tee_redirect(stdout_path)
- self.job.stderr.tee_redirect(stderr_path)
+ def client(self, script, server_ip, args = 'CPU'):
+ # run some client stuff
+ stdout_path = os.path.join(self.resultsdir, script + '.stdout')
+ stderr_path = os.path.join(self.resultsdir, script + '.stderr')
+ self.job.stdout.tee_redirect(stdout_path)
+ self.job.stderr.tee_redirect(stderr_path)
- script_path = os.path.join(self.srcdir, 'doc/examples', script)
- system('%s %s %s' % (script_path, server_ip, args))
+ script_path = os.path.join(self.srcdir, 'doc/examples', script)
+ system('%s %s %s' % (script_path, server_ip, args))
- self.job.stdout.restore()
- self.job.stderr.restore()
+ self.job.stdout.restore()
+ self.job.stderr.restore()
diff --git a/client/tests/parallel_dd/parallel_dd.py b/client/tests/parallel_dd/parallel_dd.py
index a524e6d..5d92667 100755
--- a/client/tests/parallel_dd/parallel_dd.py
+++ b/client/tests/parallel_dd/parallel_dd.py
@@ -4,119 +4,119 @@
class parallel_dd(test.test):
- version = 1
+ version = 1
- def raw_write(self):
- print "Timing raw write of %d megabytes" % self.megabytes
- sys.stdout.flush()
- dd = 'dd if=/dev/zero of=%s bs=4K count=%d' % \
- (self.fs.device, self.blocks)
- print dd
- utils.system(dd + ' > /dev/null')
+ def raw_write(self):
+ print "Timing raw write of %d megabytes" % self.megabytes
+ sys.stdout.flush()
+ dd = 'dd if=/dev/zero of=%s bs=4K count=%d' % \
+ (self.fs.device, self.blocks)
+ print dd
+ utils.system(dd + ' > /dev/null')
- def raw_read(self):
- print "Timing raw read of %d megabytes" % self.megabytes
- sys.stdout.flush()
- dd = 'dd if=%s of=/dev/null bs=4K count=%d' % \
- (self.fs.device, self.blocks)
- print dd
- utils.system(dd + ' > /dev/null')
+ def raw_read(self):
+ print "Timing raw read of %d megabytes" % self.megabytes
+ sys.stdout.flush()
+ dd = 'dd if=%s of=/dev/null bs=4K count=%d' % \
+ (self.fs.device, self.blocks)
+ print dd
+ utils.system(dd + ' > /dev/null')
- def fs_write(self):
- p = []
- # Write out 'streams' files in parallel background tasks
- for i in range(self.streams):
- file = 'poo%d' % (i+1)
- file = os.path.join(self.job.tmpdir, file)
- dd = 'dd if=/dev/zero of=%s bs=4K count=%d' % \
- (file, self.blocks_per_file)
- print dd
- p.append(subprocess.Popen(dd + ' > /dev/null',
- shell=True))
- print "Waiting for %d streams" % self.streams
- # Wait for everyone to complete
- for i in range(self.streams):
- print "Waiting for %d" % p[i].pid
- sys.stdout.flush()
- os.waitpid(p[i].pid, 0)
- sys.stdout.flush()
- sys.stderr.flush()
+ def fs_write(self):
+ p = []
+ # Write out 'streams' files in parallel background tasks
+ for i in range(self.streams):
+ file = 'poo%d' % (i+1)
+ file = os.path.join(self.job.tmpdir, file)
+ dd = 'dd if=/dev/zero of=%s bs=4K count=%d' % \
+ (file, self.blocks_per_file)
+ print dd
+ p.append(subprocess.Popen(dd + ' > /dev/null',
+ shell=True))
+ print "Waiting for %d streams" % self.streams
+ # Wait for everyone to complete
+ for i in range(self.streams):
+ print "Waiting for %d" % p[i].pid
+ sys.stdout.flush()
+ os.waitpid(p[i].pid, 0)
+ sys.stdout.flush()
+ sys.stderr.flush()
- def fs_read(self):
- for i in range(self.streams):
- file = os.path.join(self.job.tmpdir, 'poo%d' % (i+1))
- dd = 'dd if=%s of=/dev/null bs=4K count=%d' % \
- (file, self.blocks_per_file)
- utils.system(dd + ' > /dev/null')
+ def fs_read(self):
+ for i in range(self.streams):
+ file = os.path.join(self.job.tmpdir, 'poo%d' % (i+1))
+ dd = 'dd if=%s of=/dev/null bs=4K count=%d' % \
+ (file, self.blocks_per_file)
+ utils.system(dd + ' > /dev/null')
- def test(self, tag):
- start = time.time()
- self.raw_write()
- self.raw_write_rate = self.megabytes / (time.time() - start)
+ def test(self, tag):
+ start = time.time()
+ self.raw_write()
+ self.raw_write_rate = self.megabytes / (time.time() - start)
- start = time.time()
- self.raw_read()
- self.raw_read_rate = self.megabytes / (time.time() - start)
+ start = time.time()
+ self.raw_read()
+ self.raw_read_rate = self.megabytes / (time.time() - start)
- self.fs.mkfs(self.fstype)
- self.fs.mount()
- start = time.time()
- try:
- self.fs_write()
- except:
- try:
- self.fs.unmount()
- finally:
- raise
- self.fs.unmount()
- self.fs_write_rate = self.megabytes / (time.time() - start)
+ self.fs.mkfs(self.fstype)
+ self.fs.mount()
+ start = time.time()
+ try:
+ self.fs_write()
+ except:
+ try:
+ self.fs.unmount()
+ finally:
+ raise
+ self.fs.unmount()
+ self.fs_write_rate = self.megabytes / (time.time() - start)
- self.fs.mount()
- start = time.time()
- try:
- self.fs_read()
- except:
- try:
- self.fs.unmount()
- finally:
- raise
- read_in()
- self.fs_read_rate = self.megabytes / (time.time() - start)
- self.fs.unmount()
+ self.fs.mount()
+ start = time.time()
+ try:
+ self.fs_read()
+ except:
+ try:
+ self.fs.unmount()
+ finally:
+ raise
+ read_in()
+ self.fs_read_rate = self.megabytes / (time.time() - start)
+ self.fs.unmount()
- def execute(self, fs, fstype = 'ext2', iterations = 2, megabytes = 1000, streams = 2):
- self.megabytes = megabytes
- self.blocks = megabytes * 256
- self.blocks_per_file = self.blocks / streams
- self.fs = fs
- self.fstype = fstype
- self.streams = streams
-
- print "Dumping %d megabytes across %d streams, %d times" % \
- (megabytes, streams, iterations)
+ def execute(self, fs, fstype = 'ext2', iterations = 2, megabytes = 1000, streams = 2):
+ self.megabytes = megabytes
+ self.blocks = megabytes * 256
+ self.blocks_per_file = self.blocks / streams
+ self.fs = fs
+ self.fstype = fstype
+ self.streams = streams
- keyval = open(os.path.join(self.resultsdir, 'keyval'), 'w')
- for i in range(iterations):
- self.test('%d' % i)
- t = "raw_write=%d\n" % self.raw_write_rate
- t += "raw_read=%d\n" % self.raw_read_rate
- t += "fs_write=%d\n" % self.fs_write_rate
- t += "fs_read=%d\n" % self.fs_read_rate
- t += "\n"
- print t
- keyval.write(t)
- keyval.close()
+ print "Dumping %d megabytes across %d streams, %d times" % \
+ (megabytes, streams, iterations)
+
+ keyval = open(os.path.join(self.resultsdir, 'keyval'), 'w')
+ for i in range(iterations):
+ self.test('%d' % i)
+ t = "raw_write=%d\n" % self.raw_write_rate
+ t += "raw_read=%d\n" % self.raw_read_rate
+ t += "fs_write=%d\n" % self.fs_write_rate
+ t += "fs_read=%d\n" % self.fs_read_rate
+ t += "\n"
+ print t
+ keyval.write(t)
+ keyval.close()
- profilers = self.job.profilers
- if profilers.present():
- profilers.start(self)
- self.test('profile')
- profilers.stop(self)
- profilers.report(self)
+ profilers = self.job.profilers
+ if profilers.present():
+ profilers.start(self)
+ self.test('profile')
+ profilers.stop(self)
+ profilers.report(self)
diff --git a/client/tests/pi_tests/pi_tests.py b/client/tests/pi_tests/pi_tests.py
index 54872f4..7f3fcf2 100644
--- a/client/tests/pi_tests/pi_tests.py
+++ b/client/tests/pi_tests/pi_tests.py
@@ -4,19 +4,19 @@
class pi_tests(test.test):
- version = 1
+ version = 1
- # http://www.stardust.webpages.pl/files/patches/autotest/pi_tests.tar.bz2
+ # http://www.stardust.webpages.pl/files/patches/autotest/pi_tests.tar.bz2
- def setup(self, tarball = 'pi_tests.tar.bz2'):
- autotest_utils.check_glibc_ver('2.5')
- tarball = autotest_utils.unmap_url(self.bindir, tarball,
- self.tmpdir)
- autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
- os.chdir(self.srcdir)
+ def setup(self, tarball = 'pi_tests.tar.bz2'):
+ autotest_utils.check_glibc_ver('2.5')
+ tarball = autotest_utils.unmap_url(self.bindir, tarball,
+ self.tmpdir)
+ autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
+ os.chdir(self.srcdir)
- utils.system('make')
+ utils.system('make')
- def execute(self, args = '1 300'):
- os.chdir(self.srcdir)
- utils.system('./start.sh ' + args)
+ def execute(self, args = '1 300'):
+ os.chdir(self.srcdir)
+ utils.system('./start.sh ' + args)
diff --git a/client/tests/pktgen/pktgen.py b/client/tests/pktgen/pktgen.py
index c70cb43..aaef41d 100755
--- a/client/tests/pktgen/pktgen.py
+++ b/client/tests/pktgen/pktgen.py
@@ -4,51 +4,50 @@
class pktgen(test.test):
- version = 1
+ version = 1
- def execute(self, eth='eth0', count=50000, clone_skb=1, \
- dst_ip='192.168.210.210', dst_mac='01:02:03:04:05:07'):
- if not os.path.exists('/proc/net/pktgen'):
- utils.system('/sbin/modprobe pktgen')
- if not os.path.exists('/proc/net/pktgen'):
- raise error.UnhandledError('pktgen not loaded')
+ def execute(self, eth='eth0', count=50000, clone_skb=1, \
+ dst_ip='192.168.210.210', dst_mac='01:02:03:04:05:07'):
+ if not os.path.exists('/proc/net/pktgen'):
+ utils.system('/sbin/modprobe pktgen')
+ if not os.path.exists('/proc/net/pktgen'):
+ raise error.UnhandledError('pktgen not loaded')
- print 'Adding devices to run'
- self.pgdev = '/proc/net/pktgen/kpktgend_0'
+ print 'Adding devices to run'
+ self.pgdev = '/proc/net/pktgen/kpktgend_0'
- self.pgset('rem_device_all')
- self.pgset('add_device ' + eth)
- self.pgset('max_before_softirq 10000')
+ self.pgset('rem_device_all')
+ self.pgset('add_device ' + eth)
+ self.pgset('max_before_softirq 10000')
- # Configure the individual devices
- print 'Configuring devices'
+ # Configure the individual devices
+ print 'Configuring devices'
- self.ethdev='/proc/net/pktgen/' + eth
- self.pgdev=self.ethdev
+ self.ethdev='/proc/net/pktgen/' + eth
+ self.pgdev=self.ethdev
- if clone_skb:
- self.pgset('clone_skb %d' % (count))
- self.pgset('min_pkt_size 60')
- self.pgset('max_pkt_size 60')
- self.pgset('dst ' + dst_ip)
- self.pgset('dst_mac ' + dst_mac)
- self.pgset('count %d' % (count))
+ if clone_skb:
+ self.pgset('clone_skb %d' % (count))
+ self.pgset('min_pkt_size 60')
+ self.pgset('max_pkt_size 60')
+ self.pgset('dst ' + dst_ip)
+ self.pgset('dst_mac ' + dst_mac)
+ self.pgset('count %d' % (count))
- # Time to run
- self.pgdev='/proc/net/pktgen/pgctrl'
- self.pgset('start')
+ # Time to run
+ self.pgdev='/proc/net/pktgen/pgctrl'
+ self.pgset('start')
- output = os.path.join(self.resultsdir, eth)
- utils.system ('cp %s %s' % (self.ethdev, output))
+ output = os.path.join(self.resultsdir, eth)
+ utils.system ('cp %s %s' % (self.ethdev, output))
- def pgset(self, command):
- file = open(self.pgdev, 'w')
- file.write(command + '\n');
- file.close
+ def pgset(self, command):
+ file = open(self.pgdev, 'w')
+ file.write(command + '\n');
+ file.close
- if not autotest_utils.grep('Result: OK', self.pgdev):
- if not autotest_utils.grep('Result: NA', self.pgdev):
- utils.system('cat ' + self.pgdev)
- # raise UnhandledError('Result not OK')
-
+ if not autotest_utils.grep('Result: OK', self.pgdev):
+ if not autotest_utils.grep('Result: NA', self.pgdev):
+ utils.system('cat ' + self.pgdev)
+ # raise UnhandledError('Result not OK')
diff --git a/client/tests/posixtest/posixtest.py b/client/tests/posixtest/posixtest.py
index e11989a..adec69c 100755
--- a/client/tests/posixtest/posixtest.py
+++ b/client/tests/posixtest/posixtest.py
@@ -8,22 +8,21 @@
__author__ = '''mohd.omar@in.ibm.com (Mohammed Omar)'''
class posixtest(test.test):
- version = 1
- # http://ufpr.dl.sourceforge.net/sourceforge/posixtest/posixtestsuite-1.5.2.tar.gz
- def setup(self, tarball = 'posixtestsuite-1.5.2.tar.gz'):
- self.posix_tarball = utils.unmap_url(self.bindir,
- tarball,
- self.tmpdir)
- autotest_utils.extract_tarball_to_dir(self.posix_tarball,
- self.srcdir)
- os.chdir(self.srcdir)
- # Applying a small patch that introduces some linux specific
- # linking options
- utils.system('patch -p1 < ../posix-linux.patch')
- utils.system('make')
+ version = 1
+ # http://ufpr.dl.sourceforge.net/sourceforge/posixtest/posixtestsuite-1.5.2.tar.gz
+ def setup(self, tarball = 'posixtestsuite-1.5.2.tar.gz'):
+ self.posix_tarball = utils.unmap_url(self.bindir,
+ tarball,
+ self.tmpdir)
+ autotest_utils.extract_tarball_to_dir(self.posix_tarball,
+ self.srcdir)
+ os.chdir(self.srcdir)
+ # Applying a small patch that introduces some linux specific
+ # linking options
+ utils.system('patch -p1 < ../posix-linux.patch')
+ utils.system('make')
- def execute(self):
- os.chdir(self.srcdir)
- utils.system('./run_tests THR')
-
+ def execute(self):
+ os.chdir(self.srcdir)
+ utils.system('./run_tests THR')
diff --git a/client/tests/raisetest/raisetest.py b/client/tests/raisetest/raisetest.py
index 1c968b4..e966585 100755
--- a/client/tests/raisetest/raisetest.py
+++ b/client/tests/raisetest/raisetest.py
@@ -3,7 +3,7 @@
class raisetest(test.test):
- version = 1
+ version = 1
- def execute(self):
- raise error.TestError('Arrrrrrrrggggh. You are DOOOMED')
+ def execute(self):
+ raise error.TestError('Arrrrrrrrggggh. You are DOOOMED')
diff --git a/client/tests/reaim/reaim.py b/client/tests/reaim/reaim.py
index 7e43723..9a5e832 100755
--- a/client/tests/reaim/reaim.py
+++ b/client/tests/reaim/reaim.py
@@ -5,87 +5,87 @@
class reaim(test.test):
- version = 1
+ version = 1
- # http://prdownloads.sourceforge.net/re-aim-7/osdl-aim-7.0.1.13.tar.gz
- def setup(self, tarball = 'osdl-aim-7.0.1.13.tar.gz'):
- tarball = utils.unmap_url(self.bindir, tarball,
- self.tmpdir)
- autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
+ # http://prdownloads.sourceforge.net/re-aim-7/osdl-aim-7.0.1.13.tar.gz
+ def setup(self, tarball = 'osdl-aim-7.0.1.13.tar.gz'):
+ tarball = utils.unmap_url(self.bindir, tarball,
+ self.tmpdir)
+ autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
- self.job.setup_dep(['libaio'])
- libs = '-L' + self.autodir + '/deps/libaio/lib -laio'
- cflags = '-I ' + self.autodir + '/deps/libaio/include'
- var_libs = 'LIBS="' + libs + '"'
- var_cflags = 'CFLAGS="' + cflags + '"'
- self.make_flags = var_libs + ' ' + var_cflags
+ self.job.setup_dep(['libaio'])
+ libs = '-L' + self.autodir + '/deps/libaio/lib -laio'
+ cflags = '-I ' + self.autodir + '/deps/libaio/include'
+ var_libs = 'LIBS="' + libs + '"'
+ var_cflags = 'CFLAGS="' + cflags + '"'
+ self.make_flags = var_libs + ' ' + var_cflags
- os_dep.commands('autoconf', 'automake', 'libtoolize')
- os.chdir(self.srcdir)
- utils.system('./bootstrap')
- utils.system('./configure')
- # we can't use patch here, as the Makefile is autogenerated
- # so we can't tell exactly what it looks like.
- # Perform some foul in-place sed hackery instead.
- for file in ('Makefile', 'src/Makefile'):
- utils.system('sed -i "s/^CFLAGS =/CFLAGS +=/" ' + file)
- utils.system('sed -i "s/^LIBS =/LIBS +=/" ' + file)
- utils.system(self.make_flags + ' make')
- os.rename('src/reaim', 'reaim')
+ os_dep.commands('autoconf', 'automake', 'libtoolize')
+ os.chdir(self.srcdir)
+ utils.system('./bootstrap')
+ utils.system('./configure')
+ # we can't use patch here, as the Makefile is autogenerated
+ # so we can't tell exactly what it looks like.
+ # Perform some foul in-place sed hackery instead.
+ for file in ('Makefile', 'src/Makefile'):
+ utils.system('sed -i "s/^CFLAGS =/CFLAGS +=/" ' + file)
+ utils.system('sed -i "s/^LIBS =/LIBS +=/" ' + file)
+ utils.system(self.make_flags + ' make')
+ os.rename('src/reaim', 'reaim')
- def initialize(self):
- self.ldlib = 'LD_LIBRARY_PATH=%s/deps/libaio/lib'%(self.autodir)
+ def initialize(self):
+ self.ldlib = 'LD_LIBRARY_PATH=%s/deps/libaio/lib'%(self.autodir)
- def execute(self, iterations = 1, workfile = 'workfile.short',
- start = 1, end = 10, increment = 2,
- extra_args = '', tmpdir = None):
- if not tmpdir:
- tmpdir = self.tmpdir
+ def execute(self, iterations = 1, workfile = 'workfile.short',
+ start = 1, end = 10, increment = 2,
+ extra_args = '', tmpdir = None):
+ if not tmpdir:
+ tmpdir = self.tmpdir
- # -f workfile
- # -s <number of users to start with>
- # -e <number of users to end with>
- # -i <number of users to increment>
- workfile = os.path.join('data', workfile)
- args = "-f %s -s %d -e %d -i %d" %(workfile,start,end,increment)
- config = os.path.join(self.srcdir, 'reaim.config')
- utils.system('cp -f %s/reaim.config %s' % (self.bindir, config))
- args += ' -c ./reaim.config'
- open(config, 'a+').write("DISKDIR %s\n" % (tmpdir))
- os.chdir(self.srcdir)
- print os.getcwd()
- cmd = self.ldlib + ' ./reaim ' + args + ' ' + extra_args
+ # -f workfile
+ # -s <number of users to start with>
+ # -e <number of users to end with>
+ # -i <number of users to increment>
+ workfile = os.path.join('data', workfile)
+ args = "-f %s -s %d -e %d -i %d" %(workfile,start,end,increment)
+ config = os.path.join(self.srcdir, 'reaim.config')
+ utils.system('cp -f %s/reaim.config %s' % (self.bindir, config))
+ args += ' -c ./reaim.config'
+ open(config, 'a+').write("DISKDIR %s\n" % (tmpdir))
+ os.chdir(self.srcdir)
+ print os.getcwd()
+ cmd = self.ldlib + ' ./reaim ' + args + ' ' + extra_args
- results = []
+ results = []
- profilers = self.job.profilers
- if not profilers.only():
- for i in range(iterations):
- results.append(utils.system_output(cmd,
- retain_output=True))
+ profilers = self.job.profilers
+ if not profilers.only():
+ for i in range(iterations):
+ results.append(utils.system_output(cmd,
+ retain_output=True))
- # Do a profiling run if necessary
- if profilers.present():
- profilers.start(self)
- resuls.append(utils.system_output(cmd,
- retain_output=True))
- profilers.stop(self)
- profilers.report(self)
+ # Do a profiling run if necessary
+ if profilers.present():
+ profilers.start(self)
+ resuls.append(utils.system_output(cmd,
+ retain_output=True))
+ profilers.stop(self)
+ profilers.report(self)
- self.__format_results("\n".join(results))
+ self.__format_results("\n".join(results))
- def __format_results(self, results):
- out = open(self.resultsdir + '/keyval', 'w')
- for line in results.split('\n'):
- m = re.match('Max Jobs per Minute (\d+)', line)
- if m:
- max_jobs_per_min = m.group(1)
- if re.match(r"^[0-9\. ]+$", line):
- fields = line.split()
- print >> out, """\
+ def __format_results(self, results):
+ out = open(self.resultsdir + '/keyval', 'w')
+ for line in results.split('\n'):
+ m = re.match('Max Jobs per Minute (\d+)', line)
+ if m:
+ max_jobs_per_min = m.group(1)
+ if re.match(r"^[0-9\. ]+$", line):
+ fields = line.split()
+ print >> out, """\
max_jobs_per_min=%s
num_forked=%s
parent_time=%s
@@ -97,4 +97,4 @@
std_dev_pct=%s
jti=%s
""" % tuple([max_jobs_per_min] + fields)
- out.close()
+ out.close()
diff --git a/client/tests/rmaptest/rmaptest.py b/client/tests/rmaptest/rmaptest.py
index f4cf3af..9b65547 100644
--- a/client/tests/rmaptest/rmaptest.py
+++ b/client/tests/rmaptest/rmaptest.py
@@ -5,24 +5,24 @@
# tests is a simple array of "cmd" "arguments"
tests = [["rmaptest", "-h -i100 -n100 -s100 -t100 -V10 -v file1.dat"],
- ["rmaptest", "-l -i100 -n100 -s100 -t100 -V10 -v file2.dat"],
- ["rmaptest", "-r -i100 -n100 -s100 -t100 -V10 -v file3.dat"],
- ]
+ ["rmaptest", "-l -i100 -n100 -s100 -t100 -V10 -v file2.dat"],
+ ["rmaptest", "-r -i100 -n100 -s100 -t100 -V10 -v file3.dat"],
+ ]
name = 0
arglist = 1
class rmaptest(test.test):
- version = 1
- preserve_srcdir = True
+ version = 1
+ preserve_srcdir = True
- def setup(self):
- os.chdir(self.srcdir)
- utils.system('gcc -Wall -o rmaptest rmap-test.c')
+ def setup(self):
+ os.chdir(self.srcdir)
+ utils.system('gcc -Wall -o rmaptest rmap-test.c')
- def execute(self, args = ''):
- os.chdir(self.tmpdir)
- for test in tests:
- cmd = self.srcdir + '/' + test[name] + ' ' \
- + args + ' ' + test[arglist]
- utils.system(cmd)
+ def execute(self, args = ''):
+ os.chdir(self.tmpdir)
+ for test in tests:
+ cmd = self.srcdir + '/' + test[name] + ' ' \
+ + args + ' ' + test[arglist]
+ utils.system(cmd)
diff --git a/client/tests/rtlinuxtests/rtlinuxtests.py b/client/tests/rtlinuxtests/rtlinuxtests.py
index e6df6e3..8c27432 100644
--- a/client/tests/rtlinuxtests/rtlinuxtests.py
+++ b/client/tests/rtlinuxtests/rtlinuxtests.py
@@ -5,27 +5,27 @@
# tests is a simple array of "cmd" "arguments"
tests = [["./run.sh", "tests=func"],
- ["./run.sh", "tests=pi-tests"],
- ]
+ ["./run.sh", "tests=pi-tests"],
+ ]
name = 0
arglist = 1
class rtlinuxtests(test.test):
- version = 1
- preserve_srcdir = True
+ version = 1
+ preserve_srcdir = True
- # http://www.kernel.org/pub/linux/kernel/people/dvhart/realtime/tests/tests.tar.bz2
+ # http://www.kernel.org/pub/linux/kernel/people/dvhart/realtime/tests/tests.tar.bz2
- def setup(self, tarball = 'tests.tar.bz2'):
- autotest_utils.check_glibc_ver('2.5')
- self.tarball = utils.unmap_url(self.bindir, tarball,
- self.tmpdir)
- autotest_utils.extract_tarball_to_dir(self.tarball, self.srcdir)
- os.chdir(self.srcdir)
- utils.system('patch -p1 < ../path-fix.patch')
+ def setup(self, tarball = 'tests.tar.bz2'):
+ autotest_utils.check_glibc_ver('2.5')
+ self.tarball = utils.unmap_url(self.bindir, tarball,
+ self.tmpdir)
+ autotest_utils.extract_tarball_to_dir(self.tarball, self.srcdir)
+ os.chdir(self.srcdir)
+ utils.system('patch -p1 < ../path-fix.patch')
- def execute(self, args = ''):
- os.chdir(self.srcdir)
- for test in tests:
- cmd = 'echo y | ' + test[name] + ' ' + args + ' ' + test[arglist]
- utils.system(cmd)
+ def execute(self, args = ''):
+ os.chdir(self.srcdir)
+ for test in tests:
+ cmd = 'echo y | ' + test[name] + ' ' + args + ' ' + test[arglist]
+ utils.system(cmd)
diff --git a/client/tests/rttester/rttester.py b/client/tests/rttester/rttester.py
index 97cea0a..69a2e9f 100644
--- a/client/tests/rttester/rttester.py
+++ b/client/tests/rttester/rttester.py
@@ -4,15 +4,15 @@
class rttester(test.test):
- version = 1
+ version = 1
- # http://www.stardust.webpages.pl/files/patches/autotest/rttester.tar.bz2
+ # http://www.stardust.webpages.pl/files/patches/autotest/rttester.tar.bz2
- def setup(self, tarball = 'rttester.tar.bz2'):
- tarball = utils.unmap_url(self.bindir, tarball,
- self.tmpdir)
- autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
+ def setup(self, tarball = 'rttester.tar.bz2'):
+ tarball = utils.unmap_url(self.bindir, tarball,
+ self.tmpdir)
+ autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
- def execute(self):
- os.chdir(self.srcdir)
- utils.system(self.srcdir + '/check-all.sh')
+ def execute(self):
+ os.chdir(self.srcdir)
+ utils.system(self.srcdir + '/check-all.sh')
diff --git a/client/tests/scrashme/scrashme.py b/client/tests/scrashme/scrashme.py
index 3d8b237..30ace1c 100644
--- a/client/tests/scrashme/scrashme.py
+++ b/client/tests/scrashme/scrashme.py
@@ -4,31 +4,31 @@
class scrashme(test.test):
- version = 1
+ version = 1
- # http://www.codemonkey.org.uk/projects/git-snapshots/scrashme/scrashme-2007-07-08.tar.gz
- def setup(self, tarball = 'scrashme-2007-07-08.tar.gz'):
- tarball = utils.unmap_url(self.bindir, tarball,
- self.tmpdir)
- autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
- os.chdir(self.srcdir)
+ # http://www.codemonkey.org.uk/projects/git-snapshots/scrashme/scrashme-2007-07-08.tar.gz
+ def setup(self, tarball = 'scrashme-2007-07-08.tar.gz'):
+ tarball = utils.unmap_url(self.bindir, tarball,
+ self.tmpdir)
+ autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
+ os.chdir(self.srcdir)
- utils.system('make')
-
- def execute(self, iterations = 1, args_list = ''):
- if len(args_list) != 0:
- args = '' + args_list
- else:
- args = '-c100 -z'
+ utils.system('make')
- profilers = self.job.profilers
- if not profilers.only():
- for i in range(iterations):
- utils.system(self.srcdir + '/scrashme ' + args)
+ def execute(self, iterations = 1, args_list = ''):
+ if len(args_list) != 0:
+ args = '' + args_list
+ else:
+ args = '-c100 -z'
- # Do a profiling run if necessary
- if profilers.present():
- profilers.start(self)
- utils.system(self.srcdir + '/scrashme ' + args)
- profilers.stop(self)
- profilers.report(self)
+ profilers = self.job.profilers
+ if not profilers.only():
+ for i in range(iterations):
+ utils.system(self.srcdir + '/scrashme ' + args)
+
+ # Do a profiling run if necessary
+ if profilers.present():
+ profilers.start(self)
+ utils.system(self.srcdir + '/scrashme ' + args)
+ profilers.stop(self)
+ profilers.report(self)
diff --git a/client/tests/selftest/selftest.py b/client/tests/selftest/selftest.py
index 1baecf9..825b9cf 100644
--- a/client/tests/selftest/selftest.py
+++ b/client/tests/selftest/selftest.py
@@ -4,49 +4,49 @@
class selftest(test.test):
- version = 1
+ version = 1
- def setup(self):
- name = self.job.resultdir + '/sequence'
- if (not os.path.exists(name)):
- fd = file(name, 'w')
- fd.write('0')
- fd.close()
-
- def __mark(self, checkpoint):
- name = self.job.resultdir + '/sequence'
- fd = file(name, 'r')
- current = int(fd.readline())
- fd.close()
+ def setup(self):
+ name = self.job.resultdir + '/sequence'
+ if (not os.path.exists(name)):
+ fd = file(name, 'w')
+ fd.write('0')
+ fd.close()
- current += 1
- fd = file(name + '.new', 'w')
- fd.write('%d' % current)
- fd.close()
+ def __mark(self, checkpoint):
+ name = self.job.resultdir + '/sequence'
+ fd = file(name, 'r')
+ current = int(fd.readline())
+ fd.close()
- os.rename(name + '.new', name)
+ current += 1
+ fd = file(name + '.new', 'w')
+ fd.write('%d' % current)
+ fd.close()
- print "checkpoint %d %d" % (current, checkpoint)
+ os.rename(name + '.new', name)
- if (current != checkpoint):
- raise error.JobError("selftest: sequence was " +
- "%d when %d expected" % (current, checkpoint))
+ print "checkpoint %d %d" % (current, checkpoint)
- def __throw(self):
- __does_not_exist = __does_not_exist_either
+ if (current != checkpoint):
+ raise error.JobError("selftest: sequence was " +
+ "%d when %d expected" % (current, checkpoint))
- def __print(self, msg):
- sys.stdout.write(msg)
+ def __throw(self):
+ __does_not_exist = __does_not_exist_either
- def __warn(self, msg):
- sys.stderr.write(msg)
+ def __print(self, msg):
+ sys.stdout.write(msg)
- def execute(self, cmd, *args):
- if cmd == 'mark':
- self.__mark(*args)
- elif cmd == 'throw':
- self.__throw(*args)
- elif cmd == 'print':
- self.__print(*args)
- elif cmd == 'warn':
- self.__warn(*args)
+ def __warn(self, msg):
+ sys.stderr.write(msg)
+
+ def execute(self, cmd, *args):
+ if cmd == 'mark':
+ self.__mark(*args)
+ elif cmd == 'throw':
+ self.__throw(*args)
+ elif cmd == 'print':
+ self.__print(*args)
+ elif cmd == 'warn':
+ self.__warn(*args)
diff --git a/client/tests/signaltest/signaltest.py b/client/tests/signaltest/signaltest.py
index 9c483c9..be4e095 100644
--- a/client/tests/signaltest/signaltest.py
+++ b/client/tests/signaltest/signaltest.py
@@ -4,14 +4,14 @@
class signaltest(test.test):
- version = 1
- preserve_srcdir = True
+ version = 1
+ preserve_srcdir = True
- # git://git.kernel.org/pub/scm/linux/kernel/git/tglx/rt-tests.git
+ # git://git.kernel.org/pub/scm/linux/kernel/git/tglx/rt-tests.git
- def setup(self):
- os.chdir(self.srcdir)
- utils.system('make')
+ def setup(self):
+ os.chdir(self.srcdir)
+ utils.system('make')
- def execute(self, args = '-t 10 -l 100000'):
- utils.system(self.srcdir + '/signaltest ' + args)
+ def execute(self, args = '-t 10 -l 100000'):
+ utils.system(self.srcdir + '/signaltest ' + args)
diff --git a/client/tests/sleeptest/sleeptest.py b/client/tests/sleeptest/sleeptest.py
index 76c0383..35912f1 100755
--- a/client/tests/sleeptest/sleeptest.py
+++ b/client/tests/sleeptest/sleeptest.py
@@ -2,11 +2,11 @@
from autotest_lib.client.bin import test
class sleeptest(test.test):
- version = 1
+ version = 1
- def execute(self, seconds = 1):
- profilers = self.job.profilers
- profilers.start(self)
- time.sleep(seconds)
- profilers.stop(self)
- profilers.report(self)
+ def execute(self, seconds = 1):
+ profilers = self.job.profilers
+ profilers.start(self)
+ time.sleep(seconds)
+ profilers.stop(self)
+ profilers.report(self)
diff --git a/client/tests/sparse/sparse.py b/client/tests/sparse/sparse.py
index 3a434cd..f6ff11d 100755
--- a/client/tests/sparse/sparse.py
+++ b/client/tests/sparse/sparse.py
@@ -4,25 +4,25 @@
class sparse(test.test):
- version = 1
+ version = 1
- # http://www.codemonkey.org.uk/projects/git-snapshots/sparse/sparse-2006-04-28.tar.gz
- def setup(self, tarball = 'sparse-2006-04-28.tar.gz'):
- tarball = utils.unmap_url(self.bindir, tarball,
- self.tmpdir)
- autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
- os.chdir(self.srcdir)
+ # http://www.codemonkey.org.uk/projects/git-snapshots/sparse/sparse-2006-04-28.tar.gz
+ def setup(self, tarball = 'sparse-2006-04-28.tar.gz'):
+ tarball = utils.unmap_url(self.bindir, tarball,
+ self.tmpdir)
+ autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
+ os.chdir(self.srcdir)
- utils.system('make')
- utils.system('ln check sparse')
-
- self.top_dir = self.job.tmpdir+'/sparse'
-
- def execute(self, base_tree, patches, config, config_list = None):
- kernel = self.job.kernel(base_tree, self.resultsdir)
- kernel.patch(patches)
- kernel.config(config, config_list)
+ utils.system('make')
+ utils.system('ln check sparse')
- os.environ['PATH'] = self.srcdir + ':' + os.environ['PATH']
- results = os.path.join (self.resultsdir, 'sparse')
- kernel.build(make_opts = 'C=1', logfile = results)
+ self.top_dir = self.job.tmpdir+'/sparse'
+
+ def execute(self, base_tree, patches, config, config_list = None):
+ kernel = self.job.kernel(base_tree, self.resultsdir)
+ kernel.patch(patches)
+ kernel.config(config, config_list)
+
+ os.environ['PATH'] = self.srcdir + ':' + os.environ['PATH']
+ results = os.path.join (self.resultsdir, 'sparse')
+ kernel.build(make_opts = 'C=1', logfile = results)
diff --git a/client/tests/spew/spew.py b/client/tests/spew/spew.py
index 7c04cc8..4c1cb55 100755
--- a/client/tests/spew/spew.py
+++ b/client/tests/spew/spew.py
@@ -4,41 +4,41 @@
class spew(test.test):
- version = 1
+ version = 1
- # ftp://ftp.berlios.de/pub/spew/1.0.5/spew-1.0.5.tgz
- def setup(self, tarball = 'spew-1.0.5.tgz'):
- self.tarball = utils.unmap_url(self.bindir, tarball,
- self.tmpdir)
- autotest_utils.extract_tarball_to_dir(self.tarball, self.srcdir)
+ # ftp://ftp.berlios.de/pub/spew/1.0.5/spew-1.0.5.tgz
+ def setup(self, tarball = 'spew-1.0.5.tgz'):
+ self.tarball = utils.unmap_url(self.bindir, tarball,
+ self.tmpdir)
+ autotest_utils.extract_tarball_to_dir(self.tarball, self.srcdir)
- os.chdir(self.srcdir)
- utils.system('./configure')
- utils.system('make')
+ os.chdir(self.srcdir)
+ utils.system('./configure')
+ utils.system('make')
- def execute(self, testdir = None, iterations = 1, filesize='100M', type='write', pattern='random'):
- cmd = os.path.join(self.srcdir, 'src/spew')
- if not testdir:
- testdir = self.tmpdir
- tmpfile = os.path.join(testdir, 'spew-test.%d' % os.getpid())
- results = os.path.join(self.resultsdir, 'stdout')
- args = '--%s -i %d -p %s -b 2k -B 2M %s %s' % \
- (type, iterations, pattern, filesize, tmpfile)
- cmd += ' ' + args
+ def execute(self, testdir = None, iterations = 1, filesize='100M', type='write', pattern='random'):
+ cmd = os.path.join(self.srcdir, 'src/spew')
+ if not testdir:
+ testdir = self.tmpdir
+ tmpfile = os.path.join(testdir, 'spew-test.%d' % os.getpid())
+ results = os.path.join(self.resultsdir, 'stdout')
+ args = '--%s -i %d -p %s -b 2k -B 2M %s %s' % \
+ (type, iterations, pattern, filesize, tmpfile)
+ cmd += ' ' + args
- # Do a profiling run if necessary
- profilers = self.job.profilers
- if profilers.present():
- profilers.start(self)
+ # Do a profiling run if necessary
+ profilers = self.job.profilers
+ if profilers.present():
+ profilers.start(self)
- open(self.resultsdir + '/command', 'w').write(cmd + '\n')
- self.job.stdout.redirect(results)
- try:
- utils.system(cmd)
- finally:
- self.job.stdout.restore()
+ open(self.resultsdir + '/command', 'w').write(cmd + '\n')
+ self.job.stdout.redirect(results)
+ try:
+ utils.system(cmd)
+ finally:
+ self.job.stdout.restore()
- if profilers.present():
- profilers.stop(self)
- profilers.report(self)
+ if profilers.present():
+ profilers.stop(self)
+ profilers.report(self)
diff --git a/client/tests/stress/stress.py b/client/tests/stress/stress.py
index 5714045..d668866 100644
--- a/client/tests/stress/stress.py
+++ b/client/tests/stress/stress.py
@@ -4,56 +4,56 @@
class stress(test.test):
- version = 1
+ version = 1
- # http://weather.ou.edu/~apw/projects/stress/stress-0.18.8.tar.gz
- def setup(self, tarball = 'stress-0.18.8.tar.gz'):
- tarball = utils.unmap_url(self.bindir, tarball,
- self.tmpdir)
- autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
- os.chdir(self.srcdir)
+ # http://weather.ou.edu/~apw/projects/stress/stress-0.18.8.tar.gz
+ def setup(self, tarball = 'stress-0.18.8.tar.gz'):
+ tarball = utils.unmap_url(self.bindir, tarball,
+ self.tmpdir)
+ autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
+ os.chdir(self.srcdir)
- utils.system('./configure')
- utils.system('make')
+ utils.system('./configure')
+ utils.system('make')
- def execute(self, iterations = 1, args = ''):
- if not args:
- threads = 2*autotest_utils.count_cpus()
- args = '-c %d -i %d -m %d -d %d -t 60 -v' % \
- (threads, threads, threads, threads)
+ def execute(self, iterations = 1, args = ''):
+ if not args:
+ threads = 2*autotest_utils.count_cpus()
+ args = '-c %d -i %d -m %d -d %d -t 60 -v' % \
+ (threads, threads, threads, threads)
- profilers = self.job.profilers
- if not profilers.only():
- for i in range(iterations):
- utils.system(self.srcdir + '/src/stress ' + args)
+ profilers = self.job.profilers
+ if not profilers.only():
+ for i in range(iterations):
+ utils.system(self.srcdir + '/src/stress ' + args)
- # Do a profiling run if necessary
- if profilers.present():
- profilers.start(self)
- utils.system(self.srcdir + '/src/stress ' + args)
- profilers.stop(self)
- profilers.report(self)
+ # Do a profiling run if necessary
+ if profilers.present():
+ profilers.start(self)
+ utils.system(self.srcdir + '/src/stress ' + args)
+ profilers.stop(self)
+ profilers.report(self)
-# -v Turn up verbosity.
-# -q Turn down verbosity.
-# -n Show what would have been done (dry-run)
-# -t secs Time out after secs seconds.
-# --backoff usecs Wait for factor of usecs microseconds before starting
-# -c forks Spawn forks processes each spinning on sqrt().
-# -i forks Spawn forks processes each spinning on sync().
-# -m forks Spawn forks processes each spinning on malloc().
-# --vm-bytes bytes Allocate bytes number of bytes. The default is 1.
-# --vm-hang Instruct each vm hog process to go to sleep after
-# allocating memory. This contrasts with their normal
-# behavior, which is to free the memory and reallocate
-# ad infinitum. This is useful for simulating low memory
-# conditions on a machine. For example, the following
-# command allocates 256M of RAM and holds it until killed.
+# -v Turn up verbosity.
+# -q Turn down verbosity.
+# -n Show what would have been done (dry-run)
+# -t secs Time out after secs seconds.
+# --backoff usecs Wait for factor of usecs microseconds before starting
+# -c forks Spawn forks processes each spinning on sqrt().
+# -i forks Spawn forks processes each spinning on sync().
+# -m forks Spawn forks processes each spinning on malloc().
+# --vm-bytes bytes Allocate bytes number of bytes. The default is 1.
+# --vm-hang Instruct each vm hog process to go to sleep after
+# allocating memory. This contrasts with their normal
+# behavior, which is to free the memory and reallocate
+# ad infinitum. This is useful for simulating low memory
+# conditions on a machine. For example, the following
+# command allocates 256M of RAM and holds it until killed.
#
-# % stress --vm 2 --vm-bytes 128M --vm-hang
-# -d forks Spawn forks processes each spinning on write().
-# --hdd-bytes bytes Write bytes number of bytes. The default is 1GB.
-# --hdd-noclean Do not unlink file(s) to which random data is written.
+# % stress --vm 2 --vm-bytes 128M --vm-hang
+# -d forks Spawn forks processes each spinning on write().
+# --hdd-bytes bytes Write bytes number of bytes. The default is 1GB.
+# --hdd-noclean Do not unlink file(s) to which random data is written.
#
-# Note: Suffixes may be s,m,h,d,y (time) or k,m,g (size).
+# Note: Suffixes may be s,m,h,d,y (time) or k,m,g (size).
diff --git a/client/tests/sysbench/sysbench.py b/client/tests/sysbench/sysbench.py
index 58b7d25..571c37f 100644
--- a/client/tests/sysbench/sysbench.py
+++ b/client/tests/sysbench/sysbench.py
@@ -4,188 +4,188 @@
class sysbench(test.test):
- version = 1
+ version = 1
- # http://osdn.dl.sourceforge.net/sourceforge/sysbench/sysbench-0.4.8.tar.gz
- def setup(self, tarball = 'sysbench-0.4.8.tar.bz2'):
- tarball = utils.unmap_url(self.bindir, tarball,
- self.tmpdir)
- autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
- self.job.setup_dep(['pgsql', 'mysql'])
+ # http://osdn.dl.sourceforge.net/sourceforge/sysbench/sysbench-0.4.8.tar.gz
+ def setup(self, tarball = 'sysbench-0.4.8.tar.bz2'):
+ tarball = utils.unmap_url(self.bindir, tarball,
+ self.tmpdir)
+ autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
+ self.job.setup_dep(['pgsql', 'mysql'])
- os.chdir(self.srcdir)
+ os.chdir(self.srcdir)
- pgsql_dir = os.path.join(self.autodir, 'deps/pgsql/pgsql')
- mysql_dir = os.path.join(self.autodir, 'deps/mysql/mysql')
+ pgsql_dir = os.path.join(self.autodir, 'deps/pgsql/pgsql')
+ mysql_dir = os.path.join(self.autodir, 'deps/mysql/mysql')
- # configure wants to get at pg_config, so add its path
- utils.system('PATH=%s/bin:$PATH ./configure --with-mysql=%s --with-pgsql' % (pgsql_dir, mysql_dir))
- utils.system('make -j %d' % autotest_utils.count_cpus())
+ # configure wants to get at pg_config, so add its path
+ utils.system('PATH=%s/bin:$PATH ./configure --with-mysql=%s --with-pgsql' % (pgsql_dir, mysql_dir))
+ utils.system('make -j %d' % autotest_utils.count_cpus())
- def execute(self, db_type = 'pgsql', build = 1, \
- num_threads = autotest_utils.count_cpus(), max_time = 60, \
- read_only = 0, args = ''):
- plib = os.path.join(self.autodir, 'deps/pgsql/pgsql/lib')
- mlib = os.path.join(self.autodir, 'deps/mysql/mysql/lib/mysql')
- ld_path = prepend_path(plib, environ('LD_LIBRARY_PATH'))
- ld_path = prepend_path(mlib, ld_path)
- os.environ['LD_LIBRARY_PATH'] = ld_path
+ def execute(self, db_type = 'pgsql', build = 1, \
+ num_threads = autotest_utils.count_cpus(), max_time = 60, \
+ read_only = 0, args = ''):
+ plib = os.path.join(self.autodir, 'deps/pgsql/pgsql/lib')
+ mlib = os.path.join(self.autodir, 'deps/mysql/mysql/lib/mysql')
+ ld_path = prepend_path(plib, environ('LD_LIBRARY_PATH'))
+ ld_path = prepend_path(mlib, ld_path)
+ os.environ['LD_LIBRARY_PATH'] = ld_path
- # The databases don't want to run as root so run them as nobody
- self.dbuser = 'nobody'
- self.dbuid = pwd.getpwnam(self.dbuser)[2]
- self.sudo = 'sudo -u ' + self.dbuser + ' '
+ # The databases don't want to run as root so run them as nobody
+ self.dbuser = 'nobody'
+ self.dbuid = pwd.getpwnam(self.dbuser)[2]
+ self.sudo = 'sudo -u ' + self.dbuser + ' '
- # Check for nobody user
- try:
- utils.system(self.sudo + '/bin/true')
- except:
- raise TestError('Unable to run as nobody')
+ # Check for nobody user
+ try:
+ utils.system(self.sudo + '/bin/true')
+ except:
+ raise TestError('Unable to run as nobody')
- if (db_type == 'pgsql'):
- self.execute_pgsql(build, num_threads, max_time, \
- read_only, args)
- elif (db_type == 'mysql'):
- self.execute_mysql(build, num_threads, max_time, \
- read_only, args)
+ if (db_type == 'pgsql'):
+ self.execute_pgsql(build, num_threads, max_time, \
+ read_only, args)
+ elif (db_type == 'mysql'):
+ self.execute_mysql(build, num_threads, max_time, \
+ read_only, args)
- def execute_pgsql(self, build, num_threads, max_time, read_only, args):
- bin = os.path.join(self.autodir, 'deps/pgsql/pgsql/bin')
- data = os.path.join(self.autodir, 'deps/pgsql/pgsql/data')
- log = os.path.join(self.debugdir, 'pgsql.log')
+ def execute_pgsql(self, build, num_threads, max_time, read_only, args):
+ bin = os.path.join(self.autodir, 'deps/pgsql/pgsql/bin')
+ data = os.path.join(self.autodir, 'deps/pgsql/pgsql/data')
+ log = os.path.join(self.debugdir, 'pgsql.log')
- if build == 1:
- utils.system('rm -rf ' + data)
- os.mkdir(data)
- os.chown(data, self.dbuid, 0)
- utils.system(self.sudo + bin + '/initdb -D ' + data)
+ if build == 1:
+ utils.system('rm -rf ' + data)
+ os.mkdir(data)
+ os.chown(data, self.dbuid, 0)
+ utils.system(self.sudo + bin + '/initdb -D ' + data)
- # Database must be able to write its output into debugdir
- os.chown(self.debugdir, self.dbuid, 0)
- utils.system(self.sudo + bin + '/pg_ctl -D ' + data + \
- ' -l ' + log + ' start')
+ # Database must be able to write its output into debugdir
+ os.chown(self.debugdir, self.dbuid, 0)
+ utils.system(self.sudo + bin + '/pg_ctl -D ' + data + \
+ ' -l ' + log + ' start')
- # Wait for database to start
- time.sleep(5)
+ # Wait for database to start
+ time.sleep(5)
- try:
- base_cmd = self.srcdir + '/sysbench/sysbench ' + \
- '--test=oltp --db-driver=pgsql ' + \
- '--pgsql-user=' + self.dbuser
+ try:
+ base_cmd = self.srcdir + '/sysbench/sysbench ' + \
+ '--test=oltp --db-driver=pgsql ' + \
+ '--pgsql-user=' + self.dbuser
- if build == 1:
- utils.system(self.sudo + bin + '/createdb sbtest')
- cmd = base_cmd +' prepare'
- utils.system(cmd)
+ if build == 1:
+ utils.system(self.sudo + bin + '/createdb sbtest')
+ cmd = base_cmd +' prepare'
+ utils.system(cmd)
- cmd = base_cmd + \
- ' --num-threads=' + str(num_threads) + \
- ' --max-time=' + str(max_time) + \
- ' --max-requests=0'
+ cmd = base_cmd + \
+ ' --num-threads=' + str(num_threads) + \
+ ' --max-time=' + str(max_time) + \
+ ' --max-requests=0'
- if read_only:
- cmd = cmd + ' --oltp-read-only=on'
+ if read_only:
+ cmd = cmd + ' --oltp-read-only=on'
- results = []
+ results = []
- profilers = self.job.profilers
- if not profilers.only():
- results.append(utils.system_output(cmd + ' run',
- retain_output=True))
+ profilers = self.job.profilers
+ if not profilers.only():
+ results.append(utils.system_output(cmd + ' run',
+ retain_output=True))
- # Do a profiling run if necessary
- if profilers.present():
- profilers.start(self)
- results.append("Profiling run ...")
- results.append(utils.system_output(cmd + ' run',
- retain_output=True))
- profilers.stop(self)
- profilers.report(self)
- except:
- utils.system(self.sudo + bin + '/pg_ctl -D ' + data + ' stop')
- raise
+ # Do a profiling run if necessary
+ if profilers.present():
+ profilers.start(self)
+ results.append("Profiling run ...")
+ results.append(utils.system_output(cmd + ' run',
+ retain_output=True))
+ profilers.stop(self)
+ profilers.report(self)
+ except:
+ utils.system(self.sudo + bin + '/pg_ctl -D ' + data + ' stop')
+ raise
- utils.system(self.sudo + bin + '/pg_ctl -D ' + data + ' stop')
+ utils.system(self.sudo + bin + '/pg_ctl -D ' + data + ' stop')
- self.__format_results("\n".join(results))
+ self.__format_results("\n".join(results))
- def execute_mysql(self, build, num_threads, max_time, read_only, args):
- bin = os.path.join(self.autodir, 'deps/mysql/mysql/bin')
- data = os.path.join(self.autodir, 'deps/mysql/mysql/var')
- log = os.path.join(self.debugdir, 'mysql.log')
+ def execute_mysql(self, build, num_threads, max_time, read_only, args):
+ bin = os.path.join(self.autodir, 'deps/mysql/mysql/bin')
+ data = os.path.join(self.autodir, 'deps/mysql/mysql/var')
+ log = os.path.join(self.debugdir, 'mysql.log')
- if build == 1:
- utils.system('rm -rf ' + data)
- os.mkdir(data)
- os.chown(data, self.dbuid, 0)
- utils.system(bin + '/mysql_install_db --user=' + self.dbuser)
+ if build == 1:
+ utils.system('rm -rf ' + data)
+ os.mkdir(data)
+ os.chown(data, self.dbuid, 0)
+ utils.system(bin + '/mysql_install_db --user=' + self.dbuser)
- utils.system(bin + '/mysqld_safe --log-error=' + log + \
- ' --user=' + self.dbuser + ' &')
+ utils.system(bin + '/mysqld_safe --log-error=' + log + \
+ ' --user=' + self.dbuser + ' &')
- # Wait for database to start
- time.sleep(5)
+ # Wait for database to start
+ time.sleep(5)
- try:
- base_cmd = self.srcdir + '/sysbench/sysbench ' + \
- '--test=oltp --db-driver=mysql ' + \
- '--mysql-user=root'
+ try:
+ base_cmd = self.srcdir + '/sysbench/sysbench ' + \
+ '--test=oltp --db-driver=mysql ' + \
+ '--mysql-user=root'
- if build == 1:
- utils.system('echo "create database sbtest" | ' + \
- bin + '/mysql -u root')
- cmd = base_cmd +' prepare'
- utils.system(cmd)
+ if build == 1:
+ utils.system('echo "create database sbtest" | ' + \
+ bin + '/mysql -u root')
+ cmd = base_cmd +' prepare'
+ utils.system(cmd)
- cmd = base_cmd + \
- ' --num-threads=' + str(num_threads) + \
- ' --max-time=' + str(max_time) + \
- ' --max-requests=0'
+ cmd = base_cmd + \
+ ' --num-threads=' + str(num_threads) + \
+ ' --max-time=' + str(max_time) + \
+ ' --max-requests=0'
- if read_only:
- cmd = cmd + ' --oltp-read-only=on'
+ if read_only:
+ cmd = cmd + ' --oltp-read-only=on'
- results = []
+ results = []
- profilers = self.job.profilers
- if not profilers.only():
- results.append(utils.system_output(cmd + ' run',
- retain_output=True))
+ profilers = self.job.profilers
+ if not profilers.only():
+ results.append(utils.system_output(cmd + ' run',
+ retain_output=True))
- # Do a profiling run if necessary
- if profilers.present():
- profilers.start(self)
- results.append("Profiling run ...")
- results.append(utils.system_output(cmd + ' run',
- retain_output=True))
- profilers.stop(self)
- profilers.report(self)
- except:
- utils.system(bin + '/mysqladmin shutdown')
- raise
+ # Do a profiling run if necessary
+ if profilers.present():
+ profilers.start(self)
+ results.append("Profiling run ...")
+ results.append(utils.system_output(cmd + ' run',
+ retain_output=True))
+ profilers.stop(self)
+ profilers.report(self)
+ except:
+ utils.system(bin + '/mysqladmin shutdown')
+ raise
- utils.system(bin + '/mysqladmin shutdown')
+ utils.system(bin + '/mysqladmin shutdown')
- self.__format_results("\n".join(results))
+ self.__format_results("\n".join(results))
- def __format_results(self, results):
- threads = 0
- tps = 0
+ def __format_results(self, results):
+ threads = 0
+ tps = 0
- out = open(self.resultsdir + '/keyval', 'w')
- for line in results.split('\n'):
- threads_re = re.search('Number of threads: (\d+)', line)
- if threads_re:
- threads = threads_re.group(1)
+ out = open(self.resultsdir + '/keyval', 'w')
+ for line in results.split('\n'):
+ threads_re = re.search('Number of threads: (\d+)', line)
+ if threads_re:
+ threads = threads_re.group(1)
- tps_re = re.search('transactions:\s+\d+\s+\((\S+) per sec.\)', line)
- if tps_re:
- tps = tps_re.group(1)
- break
+ tps_re = re.search('transactions:\s+\d+\s+\((\S+) per sec.\)', line)
+ if tps_re:
+ tps = tps_re.group(1)
+ break
- print >> out, 'threads=%s\ntps=%s' % (threads, tps)
- out.close()
+ print >> out, 'threads=%s\ntps=%s' % (threads, tps)
+ out.close()
diff --git a/client/tests/tbench/tbench.py b/client/tests/tbench/tbench.py
index 3bc0667..0aeb3cd 100755
--- a/client/tests/tbench/tbench.py
+++ b/client/tests/tbench/tbench.py
@@ -4,58 +4,58 @@
class tbench(test.test):
- version = 2
+ version = 2
- # http://samba.org/ftp/tridge/dbench/dbench-3.04.tar.gz
- def setup(self, tarball = 'dbench-3.04.tar.gz'):
- tarball = utils.unmap_url(self.bindir, tarball,
- self.tmpdir)
- autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
- os.chdir(self.srcdir)
+ # http://samba.org/ftp/tridge/dbench/dbench-3.04.tar.gz
+ def setup(self, tarball = 'dbench-3.04.tar.gz'):
+ tarball = utils.unmap_url(self.bindir, tarball,
+ self.tmpdir)
+ autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
+ os.chdir(self.srcdir)
- utils.system('./configure')
- utils.system('make')
+ utils.system('./configure')
+ utils.system('make')
- def execute(self, iterations = 1, nprocs = None, args = ''):
- # only supports combined server+client model at the moment
- # should support separate I suppose, but nobody uses it
- if not nprocs:
- nprocs = self.job.cpu_count()
- args += ' %s' % nprocs
- results = []
- profilers = self.job.profilers
- if not profilers.only():
- for i in range(iterations):
- results.append(self.run_tbench(args))
+ def execute(self, iterations = 1, nprocs = None, args = ''):
+ # only supports combined server+client model at the moment
+ # should support separate I suppose, but nobody uses it
+ if not nprocs:
+ nprocs = self.job.cpu_count()
+ args += ' %s' % nprocs
+ results = []
+ profilers = self.job.profilers
+ if not profilers.only():
+ for i in range(iterations):
+ results.append(self.run_tbench(args))
- # Do a profiling run if necessary
- if profilers.present():
- profilers.start(self)
- results.append(self.run_tbench(args))
- profilers.stop(self)
- profilers.report(self)
+ # Do a profiling run if necessary
+ if profilers.present():
+ profilers.start(self)
+ results.append(self.run_tbench(args))
+ profilers.stop(self)
+ profilers.report(self)
- self.__format_results("\n".join(results))
+ self.__format_results("\n".join(results))
- def run_tbench(self, args):
- pid = os.fork()
- if pid: # parent
- time.sleep(1)
- client = self.srcdir + '/client.txt'
- args = '-c ' + client + ' ' + '%s' % args
- cmd = os.path.join(self.srcdir, "tbench") + " " + args
- results = utils.system_output(cmd, retain_output=True)
- os.kill(pid, signal.SIGTERM) # clean up the server
- else: # child
- server = self.srcdir + '/tbench_srv'
- os.execlp(server, server)
- return results
+ def run_tbench(self, args):
+ pid = os.fork()
+ if pid: # parent
+ time.sleep(1)
+ client = self.srcdir + '/client.txt'
+ args = '-c ' + client + ' ' + '%s' % args
+ cmd = os.path.join(self.srcdir, "tbench") + " " + args
+ results = utils.system_output(cmd, retain_output=True)
+ os.kill(pid, signal.SIGTERM) # clean up the server
+ else: # child
+ server = self.srcdir + '/tbench_srv'
+ os.execlp(server, server)
+ return results
- def __format_results(self, results):
- out = open(self.resultsdir + '/keyval', 'w')
- pattern = re.compile(r"Throughput (.*?) MB/sec (.*?) procs")
- for result in pattern.findall(results):
- print >> out, "throughput=%s\nprocs=%s\n" % result
- out.close()
+ def __format_results(self, results):
+ out = open(self.resultsdir + '/keyval', 'w')
+ pattern = re.compile(r"Throughput (.*?) MB/sec (.*?) procs")
+ for result in pattern.findall(results):
+ print >> out, "throughput=%s\nprocs=%s\n" % result
+ out.close()
diff --git a/client/tests/tiobench/tiobench.py b/client/tests/tiobench/tiobench.py
index 6c55567..8a4d5ba 100644
--- a/client/tests/tiobench/tiobench.py
+++ b/client/tests/tiobench/tiobench.py
@@ -4,32 +4,31 @@
class tiobench(test.test):
- version = 1
+ version = 1
- # http://prdownloads.sourceforge.net/tiobench/tiobench-0.3.3.tar.gz
- def setup(self, tarball = 'tiobench-0.3.3.tar.bz2'):
- tarball = utils.unmap_url(self.bindir, tarball,
- self.tmpdir)
- autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
- os.chdir(self.srcdir)
+ # http://prdownloads.sourceforge.net/tiobench/tiobench-0.3.3.tar.gz
+ def setup(self, tarball = 'tiobench-0.3.3.tar.bz2'):
+ tarball = utils.unmap_url(self.bindir, tarball,
+ self.tmpdir)
+ autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
+ os.chdir(self.srcdir)
- utils.system('make')
-
- def execute(self, dir = None, iterations=1, args = None):
- if not dir:
- dir = self.tmpdir
- os.chdir(self.srcdir)
- if not args:
- args = '--block=4096 --block=8192 --threads=10 --size=1024 --numruns=2'
- profilers = self.job.profilers
- if not profilers.only():
- for i in range(iterations):
- utils.system('./tiobench.pl --dir %s %s' %(dir, args))
+ utils.system('make')
- # Do a profiling run if necessary
- if profilers.present():
- profilers.start(self)
- utils.system('./tiobench.pl --dir %s %s' %(dir, args))
- profilers.stop(self)
- profilers.report(self)
-
+ def execute(self, dir = None, iterations=1, args = None):
+ if not dir:
+ dir = self.tmpdir
+ os.chdir(self.srcdir)
+ if not args:
+ args = '--block=4096 --block=8192 --threads=10 --size=1024 --numruns=2'
+ profilers = self.job.profilers
+ if not profilers.only():
+ for i in range(iterations):
+ utils.system('./tiobench.pl --dir %s %s' %(dir, args))
+
+ # Do a profiling run if necessary
+ if profilers.present():
+ profilers.start(self)
+ utils.system('./tiobench.pl --dir %s %s' %(dir, args))
+ profilers.stop(self)
+ profilers.report(self)
diff --git a/client/tests/tsc/tsc.py b/client/tests/tsc/tsc.py
index 79b5c26..54941f8 100755
--- a/client/tests/tsc/tsc.py
+++ b/client/tests/tsc/tsc.py
@@ -3,16 +3,16 @@
from autotest_lib.client.common_lib import utils
class tsc(test.test):
- version = 1
+ version = 1
- def setup(self, tarball = 'checktsc.tar'):
- tarball = utils.unmap_url(self.bindir, tarball,
- self.tmpdir)
- autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
- os.chdir(self.srcdir)
- utils.system('make')
+ def setup(self, tarball = 'checktsc.tar'):
+ tarball = utils.unmap_url(self.bindir, tarball,
+ self.tmpdir)
+ autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
+ os.chdir(self.srcdir)
+ utils.system('make')
-
- def execute(self, iterations = 1, args = ''):
- for i in range(iterations):
- utils.system(self.srcdir + '/checktsc ' + args)
+
+ def execute(self, iterations = 1, args = ''):
+ for i in range(iterations):
+ utils.system(self.srcdir + '/checktsc ' + args)
diff --git a/client/tests/unixbench/unixbench.py b/client/tests/unixbench/unixbench.py
index 5685404..001ce66 100755
--- a/client/tests/unixbench/unixbench.py
+++ b/client/tests/unixbench/unixbench.py
@@ -4,91 +4,91 @@
class unixbench(test.test):
- version = 2
+ version = 2
- # http://www.tux.org/pub/tux/niemi/unixbench/unixbench-4.1.0.tgz
- def setup(self, tarball = 'unixbench-4.1.0.tar.bz2'):
- tarball = utils.unmap_url(self.bindir, tarball,
- self.tmpdir)
- autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
- os.chdir(self.srcdir)
+ # http://www.tux.org/pub/tux/niemi/unixbench/unixbench-4.1.0.tgz
+ def setup(self, tarball = 'unixbench-4.1.0.tar.bz2'):
+ tarball = utils.unmap_url(self.bindir, tarball,
+ self.tmpdir)
+ autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
+ os.chdir(self.srcdir)
- utils.system('patch -p1 < ../unixbench.patch')
- utils.system('make')
+ utils.system('patch -p1 < ../unixbench.patch')
+ utils.system('make')
- def execute(self, iterations = 1, args = '', stepsecs=0):
- vars = ('TMPDIR=\"%s\" RESULTDIR=\"%s\"' %
- (self.tmpdir, self.resultsdir))
- profilers = self.job.profilers
- keyval = open(self.resultsdir + '/keyval', 'w')
- self.err = None
- if stepsecs:
- # change time per subtest from unixbench's defaults of
- # 10 secs for small tests, 30 secs for bigger tests
- vars += ' systime=%i looper=%i seconds=%i'\
- ' dhrytime=%i arithtime=%i' \
- % ((stepsecs,)*5)
- if not profilers.only():
- for i in range(iterations):
- os.chdir(self.srcdir)
- utils.system(vars + ' ./Run ' + args)
- report = open(self.resultsdir + '/report')
- self.format_results(report, keyval)
-
- # Do a profiling run if necessary
- if profilers.present():
- profilers.start(self)
- utils.system(vars + ' ./Run ' + args)
- profilers.stop(self)
- profilers.report(self)
-
- # check err string and possible throw
- if self.err != None:
- raise error.TestError(self.err)
+ def execute(self, iterations = 1, args = '', stepsecs=0):
+ vars = ('TMPDIR=\"%s\" RESULTDIR=\"%s\"' %
+ (self.tmpdir, self.resultsdir))
+ profilers = self.job.profilers
+ keyval = open(self.resultsdir + '/keyval', 'w')
+ self.err = None
+ if stepsecs:
+ # change time per subtest from unixbench's defaults of
+ # 10 secs for small tests, 30 secs for bigger tests
+ vars += ' systime=%i looper=%i seconds=%i'\
+ ' dhrytime=%i arithtime=%i' \
+ % ((stepsecs,)*5)
+ if not profilers.only():
+ for i in range(iterations):
+ os.chdir(self.srcdir)
+ utils.system(vars + ' ./Run ' + args)
+ report = open(self.resultsdir + '/report')
+ self.format_results(report, keyval)
-
- def check_for_error(self, words):
- l = len(words)
- if l >= 3 and words[-3:l] == ['no', 'measured', 'results']:
- # found a problem so record it in err string
- key = '_'.join(words[:-3])
- if self.err == None:
- self.err = key
- else:
- self.err = self.err + " " + key
- return True
- else:
- return False
+ # Do a profiling run if necessary
+ if profilers.present():
+ profilers.start(self)
+ utils.system(vars + ' ./Run ' + args)
+ profilers.stop(self)
+ profilers.report(self)
+
+ # check err string and possible throw
+ if self.err != None:
+ raise error.TestError(self.err)
- def format_results(self, report, keyval):
- for i in range(9):
- report.next()
- for line in report:
- if not line.strip():
- break
+ def check_for_error(self, words):
+ l = len(words)
+ if l >= 3 and words[-3:l] == ['no', 'measured', 'results']:
+ # found a problem so record it in err string
+ key = '_'.join(words[:-3])
+ if self.err == None:
+ self.err = key
+ else:
+ self.err = self.err + " " + key
+ return True
+ else:
+ return False
- words = line.split()
- # look for problems first
- if self.check_for_error(words):
- continue
-
- # we should make sure that there are at least
- # 6 guys before we start accessing the array
- if len(words) >= 6:
- key = '_'.join(words[:-6])
- value = words[-6]
- print >> keyval, '%s=%s' % (key, value)
- for line in report:
- if 'FINAL SCORE' in line:
- print >> keyval, 'score=%s\n' % line.split()[-1]
- break
+
+ def format_results(self, report, keyval):
+ for i in range(9):
+ report.next()
+ for line in report:
+ if not line.strip():
+ break
+
+ words = line.split()
+ # look for problems first
+ if self.check_for_error(words):
+ continue
+
+ # we should make sure that there are at least
+ # 6 guys before we start accessing the array
+ if len(words) >= 6:
+ key = '_'.join(words[:-6])
+ value = words[-6]
+ print >> keyval, '%s=%s' % (key, value)
+ for line in report:
+ if 'FINAL SCORE' in line:
+ print >> keyval, 'score=%s\n' % line.split()[-1]
+ break
if __name__ == '__main__':
- import sys
- unixbench.format_results(sys.stdin, sys.stdout)
+ import sys
+ unixbench.format_results(sys.stdin, sys.stdout)
""" Here is a sample report file:
diff --git a/client/tests/xmtest/xmtest.py b/client/tests/xmtest/xmtest.py
index 32c649a..e921633 100644
--- a/client/tests/xmtest/xmtest.py
+++ b/client/tests/xmtest/xmtest.py
@@ -1,33 +1,33 @@
# (C) Copyright IBM Corp. 2006
# Author: Paul Larson <pl@us.ibm.com>
# Description:
-# Autotest script for running Xen xm-test
-# This should be run from a Xen domain0
+# Autotest script for running Xen xm-test
+# This should be run from a Xen domain0
import os
from autotest_lib.client.bin import test, autotest_utils
from autotest_lib.client.common_lib import utils
class xmtest(test.test):
- version = 1
+ version = 1
- # This test expects just the xm-test directory, as a tarball
- # from the Xen source tree
- # hg clone http://xenbits.xensource.com/xen-unstable.hg
- # or wget http://www.cl.cam.ac.uk/Research/SRG/netos/xen/downloads/xen-unstable-src.tgz
- # cd tools
- # tar -czf xm-test.tgz xm-test
- def setup(self, tarball = 'xm-test.tar.bz2'):
- tarball = utils.unmap_url(self.bindir, tarball,
- self.tmpdir)
- autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
- os.chdir(self.srcdir)
+ # This test expects just the xm-test directory, as a tarball
+ # from the Xen source tree
+ # hg clone http://xenbits.xensource.com/xen-unstable.hg
+ # or wget http://www.cl.cam.ac.uk/Research/SRG/netos/xen/downloads/xen-unstable-src.tgz
+ # cd tools
+ # tar -czf xm-test.tgz xm-test
+ def setup(self, tarball = 'xm-test.tar.bz2'):
+ tarball = utils.unmap_url(self.bindir, tarball,
+ self.tmpdir)
+ autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
+ os.chdir(self.srcdir)
- utils.system('./autogen')
- utils.system('./configure')
- utils.system('make existing')
+ utils.system('./autogen')
+ utils.system('./configure')
+ utils.system('make existing')
- def execute(self, args = ''):
- os.chdir(self.srcdir)
- utils.system('./runtest.sh ' + args)
- utils.system('mv xmtest.* ' + self.resultsdir)
+ def execute(self, args = ''):
+ os.chdir(self.srcdir)
+ utils.system('./runtest.sh ' + args)
+ utils.system('mv xmtest.* ' + self.resultsdir)
diff --git a/client/tools/autotest b/client/tools/autotest
index 619db5a..2b47b37 100755
--- a/client/tools/autotest
+++ b/client/tools/autotest
@@ -3,21 +3,21 @@
autodir = None
try:
- autodir = os.path.dirname(os.path.realpath('/etc/autotest.conf'))
+ autodir = os.path.dirname(os.path.realpath('/etc/autotest.conf'))
except:
- pass
+ pass
if not autodir:
- for path in ['/usr/local/autotest', '/home/autotest']:
- if os.path.exists(os.path.join(path, '/bin/autotest')):
- autodir = path
+ for path in ['/usr/local/autotest', '/home/autotest']:
+ if os.path.exists(os.path.join(path, '/bin/autotest')):
+ autodir = path
autotest = os.path.join(autodir, 'bin/autotest')
control = os.path.join(autodir, 'control')
state = os.path.join(autodir, 'control.state')
if len(sys.argv) == 1 or sys.argv[1] == 'start':
- if os.path.exists(state):
- print "Restarting partially completed autotest job"
- os.system(autotest + ' --continue ' + control)
- else:
- print "No autotest jobs outstanding"
+ if os.path.exists(state):
+ print "Restarting partially completed autotest job"
+ os.system(autotest + ' --continue ' + control)
+ else:
+ print "No autotest jobs outstanding"
diff --git a/client/tools/avgtime b/client/tools/avgtime
index d78a750..b5a1ed7 100755
--- a/client/tools/avgtime
+++ b/client/tools/avgtime
@@ -2,35 +2,34 @@
import sys, os, re
def avg_deviation(values):
- sum = 0
- count = 0
+ sum = 0
+ count = 0
- if not values:
- return (0, 0)
- for x in values:
- sum += x
- count += 1
- average = sum / count
- sum_sq_dev = 0
- for x in values:
- sum_sq_dev += (x - average) ** 2
- std_dev = (sum_sq_dev / count)**0.5
- return (average, 100 * std_dev / average)
+ if not values:
+ return (0, 0)
+ for x in values:
+ sum += x
+ count += 1
+ average = sum / count
+ sum_sq_dev = 0
+ for x in values:
+ sum_sq_dev += (x - average) ** 2
+ std_dev = (sum_sq_dev / count)**0.5
+ return (average, 100 * std_dev / average)
-list = []
+list = []
for line in sys.stdin.readlines():
- (user, system, elapsed, cpu) = line.split()[0:4]
- user = float(re.match(r'([\d\.]+)', user).group(0))
- system = float(re.match(r'([\d\.]+)', system).group(0))
- m = re.match(r'(\d+):([\d\.]+)', elapsed)
- elapsed = 60*int(m.group(1)) + float(m.group(2))
- cpu = int(re.match(r'(\d+)', cpu).group(0))
+ (user, system, elapsed, cpu) = line.split()[0:4]
+ user = float(re.match(r'([\d\.]+)', user).group(0))
+ system = float(re.match(r'([\d\.]+)', system).group(0))
+ m = re.match(r'(\d+):([\d\.]+)', elapsed)
+ elapsed = 60*int(m.group(1)) + float(m.group(2))
+ cpu = int(re.match(r'(\d+)', cpu).group(0))
- list.append((user, system, elapsed, cpu))
+ list.append((user, system, elapsed, cpu))
print " user: %0.2fs (%0.2f%%)" % avg_deviation([x[0] for x in list])
print " system: %0.2fs (%0.2f%%)" % avg_deviation([x[1] for x in list])
print "elapsed: %0.2fs (%0.2f%%)" % avg_deviation([x[2] for x in list])
print " cpu: %d%% (%0.2f%%)" % avg_deviation([x[3] for x in list])
-
diff --git a/client/tools/diffprofile b/client/tools/diffprofile
index 6bf2b6b..da57eb0 100755
--- a/client/tools/diffprofile
+++ b/client/tools/diffprofile
@@ -7,20 +7,20 @@
results_per_sign = 10
def parse_lines(filename):
- results = []
- start_key = 1
- for line in open(filename).readlines():
- try:
- a = line.split()
- key = ' '.join(a[start_key:])
- count = int(a[0])
- results.append((key, count))
- except: # presumably a header line
- if re.match(r'samples\s*%\s*app name\s*symbol name', line):
- start_key = 2
- elif re.match(r'samples\s*%\s*image name\s*app name\s*symbol name', line):
- start_key = 3
- return results
+ results = []
+ start_key = 1
+ for line in open(filename).readlines():
+ try:
+ a = line.split()
+ key = ' '.join(a[start_key:])
+ count = int(a[0])
+ results.append((key, count))
+ except: # presumably a header line
+ if re.match(r'samples\s*%\s*app name\s*symbol name', line):
+ start_key = 2
+ elif re.match(r'samples\s*%\s*image name\s*app name\s*symbol name', line):
+ start_key = 3
+ return results
# Firstly, suck in both files.
@@ -29,43 +29,43 @@
diff = {}
for (key, count) in parse_lines(sys.argv[1]):
- # Oprofile seems to be ... erm ... broken. Keys can appear > once ;-(
- if orig.has_key(key):
- orig[key] += count
- else:
- orig[key] = count
- if diff.has_key(key):
- diff[key] -= count
- else:
- diff[key] = -count
+ # Oprofile seems to be ... erm ... broken. Keys can appear > once ;-(
+ if orig.has_key(key):
+ orig[key] += count
+ else:
+ orig[key] = count
+ if diff.has_key(key):
+ diff[key] -= count
+ else:
+ diff[key] = -count
for (key, count) in parse_lines(sys.argv[2]):
- if new.has_key(key):
- new[key] += count
- else:
- new[key] = count
- if diff.has_key(key):
- diff[key] += count
- else:
- diff[key] = count
+ if new.has_key(key):
+ new[key] += count
+ else:
+ new[key] = count
+ if diff.has_key(key):
+ diff[key] += count
+ else:
+ diff[key] = count
if len(orig) < 2* results_per_sign or len(new) < 2 * results_per_sign:
- sys.exit(1) # one of the files was blank?
+ sys.exit(1) # one of the files was blank?
# Now sort and print the diffs.
def print_key(key):
- if orig.has_key(key) and orig[key] > 0:
- pct = (100 * diff[key]) / orig[key]
- else:
- pct = 0
- print "%10d %6.1f%% %s" % (diff[key], pct, key)
+ if orig.has_key(key) and orig[key] > 0:
+ pct = (100 * diff[key]) / orig[key]
+ else:
+ pct = 0
+ print "%10d %6.1f%% %s" % (diff[key], pct, key)
keys = sorted(diff.keys(), key=lambda x : diff[x], reverse = True)
for key in keys[:results_per_sign]:
- print_key(key)
+ print_key(key)
print "\n...\n"
for key in keys[len(keys)-results_per_sign:]:
- print_key(key)
+ print_key(key)
diff --git a/client/tools/make_clean b/client/tools/make_clean
index 5f47476..f6853d4 100755
--- a/client/tools/make_clean
+++ b/client/tools/make_clean
@@ -2,25 +2,24 @@
import os
def purge_src(top_dir):
- for dir in os.listdir(top_dir):
- if dir.startswith('.'):
- continue
- py = os.path.join (top_dir, dir, dir + '.py')
- if not os.path.exists(py):
- continue
- ret = os.system('grep -q "preserve_srcdir = " ' + py)
- src_path = os.path.abspath(os.path.join('tests', dir, 'src'))
- if not os.path.exists(src_path):
- continue
- if ret: # This should have a replaceable src dir
- cmd = 'rm -rf ' + src_path
- else:
- cmd = 'cd %s; make clean > /dev/null 2>&1 ' % src_path
+ for dir in os.listdir(top_dir):
+ if dir.startswith('.'):
+ continue
+ py = os.path.join (top_dir, dir, dir + '.py')
+ if not os.path.exists(py):
+ continue
+ ret = os.system('grep -q "preserve_srcdir = " ' + py)
+ src_path = os.path.abspath(os.path.join('tests', dir, 'src'))
+ if not os.path.exists(src_path):
+ continue
+ if ret: # This should have a replaceable src dir
+ cmd = 'rm -rf ' + src_path
+ else:
+ cmd = 'cd %s; make clean > /dev/null 2>&1 ' % src_path
- print cmd
- os.system(cmd)
+ print cmd
+ os.system(cmd)
for dir in ['tests', 'profilers', 'deps']:
- purge_src(dir)
-
+ purge_src(dir)
diff --git a/conmux/contrib/console_check.py b/conmux/contrib/console_check.py
index eb0f37d..7e450e8 100644
--- a/conmux/contrib/console_check.py
+++ b/conmux/contrib/console_check.py
@@ -7,8 +7,8 @@
Checks if machines are not only connected to conmux but also
responding in an expected way
- Supports options to show all, good, bad, unknown and add them
- to autotest as well.
+ Supports options to show all, good, bad, unknown and add them
+ to autotest as well.
*In order for the power update option to work you have to have
access to the etc directory of the conmux server
@@ -19,285 +19,285 @@
def main(argv):
- consoles = {}
- consoles['good'] = []
- consoles['bad'] = []
- consoles['unknown'] = []
- # 0, 1, 2 status
- STATUS = [ 'good', 'bad', 'unknown']
- parser = OptionParser()
- parser.add_option('--conmux-server', dest="conmux_server",
- default='localhost',
- help="Conmux server to connect to")
- parser.add_option('--conmux-dir', dest="conmux_dir",
- default='/usr/local/conmux',
- help="Conmux server to connect to")
- parser.add_option('--console-binary', dest="console_binary",
- default='/usr/local/conmux/bin/console',
- help="Conmux console binary location")
- parser.add_option('--autotest-cli-dir', dest="autotest_cli_dir",
- default='/usr/local/autotest/cli',
- help="Autotest CLI dir")
- parser.add_option('--add-hosts',
- action="store_true", dest="add_hosts",
- default=False,
- help="If host not on autotest server try to add it")
- parser.add_option('--power-label', dest="power_label",
- default='remote-power',
- help="Label to add to hosts that support hard reset")
- parser.add_option('--console-label', dest="console_label",
- default='console',
- help="Label to add to hosts that support console")
- parser.add_option('--update-console-label',
- action="store_true", dest="update_console_label",
- default=False,
- help="Update console label on autotest server")
- parser.add_option('--update-power-label',
- action="store_true", dest="update_power_label",
- default=False,
- help="Update power label on autotest server" +\
- "*Note this runs then exists no consoles are checked")
- parser.add_option('--verbose',
- action="store_true", dest="verbose",
- default=False,
- help="Verbose output")
- parser.add_option('--show-bad',
- action="store_true", dest="show_bad",
- default=False,
- help="Show consoles that are no longer functioning")
- parser.add_option('--show-good',
- action="store_true", dest="show_good",
- default=False,
- help="Show consoles that are functioning properly")
- parser.add_option('--show-unknown',
- action="store_true", dest="show_unknown",
- default=False,
- help="Show consoles that are in an unknown state")
- parser.add_option('--show-all',
- action="store_true", dest="show_all",
- default=False,
- help="Show status of all consoles")
- options, args = parser.parse_args()
- if len(argv) == 2 and options.verbose:
- parser.print_help()
- return 1
- elif len(argv) < 2:
- parser.print_help()
- return 1
+ consoles = {}
+ consoles['good'] = []
+ consoles['bad'] = []
+ consoles['unknown'] = []
+ # 0, 1, 2 status
+ STATUS = [ 'good', 'bad', 'unknown']
+ parser = OptionParser()
+ parser.add_option('--conmux-server', dest="conmux_server",
+ default='localhost',
+ help="Conmux server to connect to")
+ parser.add_option('--conmux-dir', dest="conmux_dir",
+ default='/usr/local/conmux',
+ help="Conmux server to connect to")
+ parser.add_option('--console-binary', dest="console_binary",
+ default='/usr/local/conmux/bin/console',
+ help="Conmux console binary location")
+ parser.add_option('--autotest-cli-dir', dest="autotest_cli_dir",
+ default='/usr/local/autotest/cli',
+ help="Autotest CLI dir")
+ parser.add_option('--add-hosts',
+ action="store_true", dest="add_hosts",
+ default=False,
+ help="If host not on autotest server try to add it")
+ parser.add_option('--power-label', dest="power_label",
+ default='remote-power',
+ help="Label to add to hosts that support hard reset")
+ parser.add_option('--console-label', dest="console_label",
+ default='console',
+ help="Label to add to hosts that support console")
+ parser.add_option('--update-console-label',
+ action="store_true", dest="update_console_label",
+ default=False,
+ help="Update console label on autotest server")
+ parser.add_option('--update-power-label',
+ action="store_true", dest="update_power_label",
+ default=False,
+ help="Update power label on autotest server" +\
+ "*Note this runs then exists no consoles are checked")
+ parser.add_option('--verbose',
+ action="store_true", dest="verbose",
+ default=False,
+ help="Verbose output")
+ parser.add_option('--show-bad',
+ action="store_true", dest="show_bad",
+ default=False,
+ help="Show consoles that are no longer functioning")
+ parser.add_option('--show-good',
+ action="store_true", dest="show_good",
+ default=False,
+ help="Show consoles that are functioning properly")
+ parser.add_option('--show-unknown',
+ action="store_true", dest="show_unknown",
+ default=False,
+ help="Show consoles that are in an unknown state")
+ parser.add_option('--show-all',
+ action="store_true", dest="show_all",
+ default=False,
+ help="Show status of all consoles")
+ options, args = parser.parse_args()
+ if len(argv) == 2 and options.verbose:
+ parser.print_help()
+ return 1
+ elif len(argv) < 2:
+ parser.print_help()
+ return 1
- if options.update_power_label:
- remove_create_label(options.power_label,
- options.autotest_cli_dir)
- update_power_label(options.power_label, options.conmux_dir,
- options.autotest_cli_dir, options.add_hosts)
- return
- print options.console_binary
- if not os.path.exists(options.console_binary):
- print "Error %s does not exist, please specify another path" %\
- options.console_binary
- return 1
- hosts = get_console_hosts(options.console_binary, options.conmux_server)
- for host in hosts:
- rc = check_host(host, options.console_binary)
- if options.verbose is True:
- print "%s status: %s" % (host, STATUS[rc])
- consoles[STATUS[rc]].append(host)
+ if options.update_power_label:
+ remove_create_label(options.power_label,
+ options.autotest_cli_dir)
+ update_power_label(options.power_label, options.conmux_dir,
+ options.autotest_cli_dir, options.add_hosts)
+ return
+ print options.console_binary
+ if not os.path.exists(options.console_binary):
+ print "Error %s does not exist, please specify another path" %\
+ options.console_binary
+ return 1
+ hosts = get_console_hosts(options.console_binary, options.conmux_server)
+ for host in hosts:
+ rc = check_host(host, options.console_binary)
+ if options.verbose is True:
+ print "%s status: %s" % (host, STATUS[rc])
+ consoles[STATUS[rc]].append(host)
- if options.show_all:
- for status in consoles:
- print "--- %s ---" % status
- for host in consoles[status]:
- print host
- if options.show_good:
- print "--- good ---"
- for host in consoles['good']:
- print host
- if options.show_bad:
- print "--- bad ---"
- for host in consoles['bad']:
- print host
- if options.show_unknown:
- print "--- unknown ---"
- for host in consoles['unknown']:
- print host
+ if options.show_all:
+ for status in consoles:
+ print "--- %s ---" % status
+ for host in consoles[status]:
+ print host
+ if options.show_good:
+ print "--- good ---"
+ for host in consoles['good']:
+ print host
+ if options.show_bad:
+ print "--- bad ---"
+ for host in consoles['bad']:
+ print host
+ if options.show_unknown:
+ print "--- unknown ---"
+ for host in consoles['unknown']:
+ print host
- if options.update_console_label:
- remove_create_label(options.console_label,
- options.autotest_cli_dir)
- update_console_label(options.console_label, consoles['good'],
- options.autotest_cli_dir, options.add_hosts)
-
+ if options.update_console_label:
+ remove_create_label(options.console_label,
+ options.autotest_cli_dir)
+ update_console_label(options.console_label, consoles['good'],
+ options.autotest_cli_dir, options.add_hosts)
+
def update_console_label(console_label, consoles, cli_dir, add_hosts=False):
- """Update CONSOLE_LABEL on your autotest server.
- This removes the label and recreates it, then populating the label
- with all the machines your conmux server knows about.
-
- *Note If the hosts do not exist they are created.
- Args:
- console_label:
- string, describes the autotest label to add to machines.
- consoles:
- list, all the consoles that have confirmed console support.
- """
- # TODO: Update to new CLI and change logic until then
- # this is the best way to ensure a machine is added i.e. one at a time
+ """Update CONSOLE_LABEL on your autotest server.
+ This removes the label and recreates it, then populating the label
+ with all the machines your conmux server knows about.
- for host in consoles:
- if not host_label_add(host, console_label, cli_dir):
- # Try to create host
- if add_hosts:
- if host_create(host, cli_dir):
- host_label_add(host, power_label,
- cli_dir)
- else:
- print "Unable to add host " + host
+ *Note If the hosts do not exist they are created.
+ Args:
+ console_label:
+ string, describes the autotest label to add to machines.
+ consoles:
+ list, all the consoles that have confirmed console support.
+ """
+ # TODO: Update to new CLI and change logic until then
+ # this is the best way to ensure a machine is added i.e. one at a time
+
+ for host in consoles:
+ if not host_label_add(host, console_label, cli_dir):
+ # Try to create host
+ if add_hosts:
+ if host_create(host, cli_dir):
+ host_label_add(host, power_label,
+ cli_dir)
+ else:
+ print "Unable to add host " + host
def update_power_label(power_label, conmux_dir, cli_dir, add_hosts=False):
- """Look in CONSOLE_DIR/etc and grab known power commands
- Then remove POWER_LABEL and add machines to that label
- """
- # remove label and add it
- for host in hard_reset_hosts(conmux_dir):
- rc = label_add_host(host, power_label, cli_dir)
- if not rc:
- # Try to create the host
- if add_hosts:
- if host_create(host, cli_dir):
- rc = label_add_host(host, power_label,
- cli_dir)
- else:
- print "Unable to add host " + host
+ """Look in CONSOLE_DIR/etc and grab known power commands
+ Then remove POWER_LABEL and add machines to that label
+ """
+ # remove label and add it
+ for host in hard_reset_hosts(conmux_dir):
+ rc = label_add_host(host, power_label, cli_dir)
+ if not rc:
+ # Try to create the host
+ if add_hosts:
+ if host_create(host, cli_dir):
+ rc = label_add_host(host, power_label,
+ cli_dir)
+ else:
+ print "Unable to add host " + host
-def hard_reset_hosts(conmux_dir):
- """Go through conmux dir and find hosts that have reset commands"""
- config_dir = os.path.join(conmux_dir, "etc")
- hosts = []
- for file in os.listdir(config_dir):
- if not file.endswith(".cf"):
- continue
- file_path = os.path.join(config_dir, file)
- try:
- try:
- f = open(file_path)
- for line in f:
- if "reset" in line:
- hosts.append(file.rstrip(".cf"))
- except IOError:
- pass
- finally:
- f.close()
- return hosts
+def hard_reset_hosts(conmux_dir):
+ """Go through conmux dir and find hosts that have reset commands"""
+ config_dir = os.path.join(conmux_dir, "etc")
+ hosts = []
+ for file in os.listdir(config_dir):
+ if not file.endswith(".cf"):
+ continue
+ file_path = os.path.join(config_dir, file)
+ try:
+ try:
+ f = open(file_path)
+ for line in f:
+ if "reset" in line:
+ hosts.append(file.rstrip(".cf"))
+ except IOError:
+ pass
+ finally:
+ f.close()
+ return hosts
def host_create(host, cli_dir):
- """Create a host
- Return:
- True, if successfuly false if failed
- """
- cmd = "%s/host-create %s" % (cli_dir, host)
- status, output = commands.getstatusoutput(cmd)
- return status == 0
+ """Create a host
+ Return:
+ True, if successfuly false if failed
+ """
+ cmd = "%s/host-create %s" % (cli_dir, host)
+ status, output = commands.getstatusoutput(cmd)
+ return status == 0
def label_add_host(host, label, cli_dir):
- """Add a host to a label"""
- host_cmd = "%s/label-add-hosts %s %s" % (cli_dir, label, host)
- (status, output) = commands.getstatusoutput(host_cmd)
- if status != 0:
- return False
+ """Add a host to a label"""
+ host_cmd = "%s/label-add-hosts %s %s" % (cli_dir, label, host)
+ (status, output) = commands.getstatusoutput(host_cmd)
+ if status != 0:
+ return False
- return True
+ return True
def remove_create_label(label, cli_dir):
- """Remove and recreate a given label"""
- cmd = "%s/label-rm %s" % (cli_dir, label)
- status, output = commands.getstatusoutput(cmd)
- if status != 0:
- raise Exception("Error deleting label: " + label)
+ """Remove and recreate a given label"""
+ cmd = "%s/label-rm %s" % (cli_dir, label)
+ status, output = commands.getstatusoutput(cmd)
+ if status != 0:
+ raise Exception("Error deleting label: " + label)
- cmd = "%s/label-create %s" % (cli_dir, label)
- status, output = commands.getstatusoutput(cmd)
- if status != 0:
- raise Exception("Error creating label: " + label + output)
+ cmd = "%s/label-create %s" % (cli_dir, label)
+ status, output = commands.getstatusoutput(cmd)
+ if status != 0:
+ raise Exception("Error creating label: " + label + output)
- return True
+ return True
def get_console_hosts(console_binary, conmux_server):
- """Use console to collect console hosts and return a list.
-
- Args:
- console_binary:
- string, location of the conmux console binary
- conmux_server:
- string, hostname of the conmux server
-
- Returns:
- A List of console conmux is currently running on.
- """
+ """Use console to collect console hosts and return a list.
- hosts_list = []
- cmd = "%s --list %s" % (console_binary, conmux_server)
- for line in commands.getoutput(cmd).split('\n'):
- host = (line.split(' '))[0]
- hosts_list.append(host)
+ Args:
+ console_binary:
+ string, location of the conmux console binary
+ conmux_server:
+ string, hostname of the conmux server
- return hosts_list
+ Returns:
+ A List of console conmux is currently running on.
+ """
+
+ hosts_list = []
+ cmd = "%s --list %s" % (console_binary, conmux_server)
+ for line in commands.getoutput(cmd).split('\n'):
+ host = (line.split(' '))[0]
+ hosts_list.append(host)
+
+ return hosts_list
def check_host(host, console_binary):
- """Check hosts for common errors and return the status.
+ """Check hosts for common errors and return the status.
- Args:
- host:
- string, the console host identifier
+ Args:
+ host:
+ string, the console host identifier
- console_binary:
- string, location of the conmux console binary
- Returns:
- int, 0: Machine state is good
- int, 1: Machine state is bad
- int, 2: Machine state is unknown
- """
- RESPONSES = [ host + ' login:',
- 'ENOENT entry not found',
- 'login:',
- 'Connection refused',
- '<<<NOT CONNECTED>>>',
- 'Authentication failure',
- 'Give root password for maintenance', ]
+ console_binary:
+ string, location of the conmux console binary
+ Returns:
+ int, 0: Machine state is good
+ int, 1: Machine state is bad
+ int, 2: Machine state is unknown
+ """
+ RESPONSES = [ host + ' login:',
+ 'ENOENT entry not found',
+ 'login:',
+ 'Connection refused',
+ '<<<NOT CONNECTED>>>',
+ 'Authentication failure',
+ 'Give root password for maintenance', ]
- cmd = '%s %s' % (console_binary, host)
- shell = pexpect.spawn(cmd)
+ cmd = '%s %s' % (console_binary, host)
+ shell = pexpect.spawn(cmd)
- shell.send('\r\n')
- shell.send('\r\n')
- shell.send('\r\n')
- try:
- # May need to increase the timeout but good so far
- response = shell.expect(RESPONSES, 1)
- except pexpect.TIMEOUT:
- shell.sendline('~$')
- shell.expect('>')
- shell.sendline('quit')
- return 1
- except pexpect.EOF:
- # unknown error
- shell.sendline('~$')
- shell.expect('>')
- shell.sendline('quit')
- return 2
- # TODO: Change actions based on what server returned
- if response == 0:
- # OK response
- return 0
- else:
- return 1
+ shell.send('\r\n')
+ shell.send('\r\n')
+ shell.send('\r\n')
+ try:
+ # May need to increase the timeout but good so far
+ response = shell.expect(RESPONSES, 1)
+ except pexpect.TIMEOUT:
+ shell.sendline('~$')
+ shell.expect('>')
+ shell.sendline('quit')
+ return 1
+ except pexpect.EOF:
+ # unknown error
+ shell.sendline('~$')
+ shell.expect('>')
+ shell.sendline('quit')
+ return 2
+ # TODO: Change actions based on what server returned
+ if response == 0:
+ # OK response
+ return 0
+ else:
+ return 1
if __name__ == '__main__':
- main(sys.argv)
+ main(sys.argv)
diff --git a/frontend/afe/control_file.py b/frontend/afe/control_file.py
index fd12926..9b8602b 100644
--- a/frontend/afe/control_file.py
+++ b/frontend/afe/control_file.py
@@ -14,11 +14,11 @@
CLIENT_KERNEL_TEMPLATE = """\
kernel = '%(kernel)s'
def step_init():
- job.next_step([step_test])
- testkernel = job.kernel('%(kernel)s')
- %(kernel_config_line)s
- testkernel.install()
- testkernel.boot(args='%(kernel_args)s')
+ job.next_step([step_test])
+ testkernel = job.kernel('%(kernel)s')
+ %(kernel_config_line)s
+ testkernel.install()
+ testkernel.boot(args='%(kernel_args)s')
def step_test():
"""
@@ -26,13 +26,13 @@
SERVER_KERNEL_TEMPLATE = """\
kernel = '%%(kernel)s'
kernel_install_control = \"""
-%s pass
+%s pass
\"""
at = autotest.Autotest()
def install_kernel(machine):
- host = hosts.SSHHost(machine)
- at.run(kernel_install_control, host=host)
+ host = hosts.SSHHost(machine)
+ at.run(kernel_install_control, host=host)
job.parallel_simple(install_kernel, machines)
""" % CLIENT_KERNEL_TEMPLATE
@@ -41,73 +41,73 @@
def kernel_config_line(kernel, platform):
- if (not kernel.endswith('.rpm') and platform and
- platform.kernel_config):
- return "testkernel.config('%s')" % platform.kernel_config
- return ''
+ if (not kernel.endswith('.rpm') and platform and
+ platform.kernel_config):
+ return "testkernel.config('%s')" % platform.kernel_config
+ return ''
def read_control_file(test):
- control_file = open(os.path.join(AUTOTEST_DIR, test.path))
- control_contents = control_file.read()
- control_file.close()
- return control_contents
+ control_file = open(os.path.join(AUTOTEST_DIR, test.path))
+ control_contents = control_file.read()
+ control_file.close()
+ return control_contents
def get_kernel_stanza(kernel, platform=None, kernel_args='', is_server=False):
- if is_server:
- template = SERVER_KERNEL_TEMPLATE
- else:
- template = CLIENT_KERNEL_TEMPLATE
+ if is_server:
+ template = SERVER_KERNEL_TEMPLATE
+ else:
+ template = CLIENT_KERNEL_TEMPLATE
- stanza = template % {
- 'kernel' : kernel,
- 'kernel_config_line' : kernel_config_line(kernel, platform),
- 'kernel_args' : kernel_args}
- return stanza
+ stanza = template % {
+ 'kernel' : kernel,
+ 'kernel_config_line' : kernel_config_line(kernel, platform),
+ 'kernel_args' : kernel_args}
+ return stanza
def add_boilerplate_to_nested_steps(lines):
- # Look for a line that begins with 'def step_init():' while
- # being flexible on spacing. If it's found, this will be
- # a nested set of steps, so add magic to make it work.
- # See client/bin/job.py's step_engine for more info.
- if re.search(r'^(.*\n)*def\s+step_init\s*\(\s*\)\s*:', lines):
- lines += '\nreturn locals() '
- lines += '# Boilerplate magic for nested sets of steps'
- return lines
+ # Look for a line that begins with 'def step_init():' while
+ # being flexible on spacing. If it's found, this will be
+ # a nested set of steps, so add magic to make it work.
+ # See client/bin/job.py's step_engine for more info.
+ if re.search(r'^(.*\n)*def\s+step_init\s*\(\s*\)\s*:', lines):
+ lines += '\nreturn locals() '
+ lines += '# Boilerplate magic for nested sets of steps'
+ return lines
def format_step(item, lines):
- lines = indent_text(lines, '\t')
- lines = 'def step%d():\n%s' % (item, lines)
- return lines
+ lines = indent_text(lines, '\t')
+ lines = 'def step%d():\n%s' % (item, lines)
+ return lines
-
+
def get_tests_stanza(tests, is_server, prepend=[], append=[]):
- raw_control_files = [read_control_file(test) for test in tests]
- if is_server:
- return '\n'.join(raw_control_files)
- raw_steps = prepend + [add_boilerplate_to_nested_steps(step)
- for step in raw_control_files] + append
- steps = [format_step(index, step)
- for index, step in enumerate(raw_steps)]
- header = ''.join(CLIENT_STEP_TEMPLATE % i for i in xrange(len(steps)))
- return header + '\n' + '\n\n'.join(steps)
+ raw_control_files = [read_control_file(test) for test in tests]
+ if is_server:
+ return '\n'.join(raw_control_files)
+ raw_steps = prepend + [add_boilerplate_to_nested_steps(step)
+ for step in raw_control_files] + append
+ steps = [format_step(index, step)
+ for index, step in enumerate(raw_steps)]
+ header = ''.join(CLIENT_STEP_TEMPLATE % i for i in xrange(len(steps)))
+ return header + '\n' + '\n\n'.join(steps)
def indent_text(text, indent):
- lines = [indent + line for line in text.splitlines()]
- return '\n'.join(lines)
+ lines = [indent + line for line in text.splitlines()]
+ return '\n'.join(lines)
def generate_control(tests, kernel=None, platform=None, is_server=False):
- control_file_text = ''
- if kernel:
- control_file_text = get_kernel_stanza(kernel, platform,
- is_server=is_server)
- elif not is_server:
- control_file_text = 'def step_init():\n'
+ control_file_text = ''
+ if kernel:
+ control_file_text = get_kernel_stanza(kernel, platform,
+ is_server=is_server)
+ elif not is_server:
+ control_file_text = 'def step_init():\n'
- control_file_text += get_tests_stanza(tests, is_server)
- return control_file_text
+ control_file_text += get_tests_stanza(tests, is_server)
+ return control_file_text
diff --git a/frontend/afe/enum.py b/frontend/afe/enum.py
index c26ae81..284e04b 100644
--- a/frontend/afe/enum.py
+++ b/frontend/afe/enum.py
@@ -5,69 +5,69 @@
__author__ = 'showard@google.com (Steve Howard)'
class Enum(object):
- """\
- Utility class to implement Enum-like functionality.
+ """\
+ Utility class to implement Enum-like functionality.
- >>> e = Enum('String one', 'String two')
- >>> e.STRING_ONE
- 0
- >>> e.STRING_TWO
- 1
- >>> e.choices()
- [(0, 'String one'), (1, 'String two')]
- >>> e.get_value('String one')
- 0
- >>> e.get_string(0)
- 'String one'
+ >>> e = Enum('String one', 'String two')
+ >>> e.STRING_ONE
+ 0
+ >>> e.STRING_TWO
+ 1
+ >>> e.choices()
+ [(0, 'String one'), (1, 'String two')]
+ >>> e.get_value('String one')
+ 0
+ >>> e.get_string(0)
+ 'String one'
- >>> e = Enum('Hello', 'Goodbye', string_values=True)
- >>> e.HELLO, e.GOODBYE
- ('Hello', 'Goodbye')
+ >>> e = Enum('Hello', 'Goodbye', string_values=True)
+ >>> e.HELLO, e.GOODBYE
+ ('Hello', 'Goodbye')
- >>> e = Enum('One', 'Two', start_value=1)
- >>> e.ONE
- 1
- >>> e.TWO
- 2
- """
- def __init__(self, *names, **kwargs):
- self.string_values = kwargs.get('string_values')
- start_value = kwargs.get('start_value', 0)
- self.names = names
- self.values = []
- for i, name in enumerate(names):
- if self.string_values:
- value = name
- else:
- value = i + start_value
- self.values.append(value)
- setattr(self, self.get_attr_name(name), value)
+ >>> e = Enum('One', 'Two', start_value=1)
+ >>> e.ONE
+ 1
+ >>> e.TWO
+ 2
+ """
+ def __init__(self, *names, **kwargs):
+ self.string_values = kwargs.get('string_values')
+ start_value = kwargs.get('start_value', 0)
+ self.names = names
+ self.values = []
+ for i, name in enumerate(names):
+ if self.string_values:
+ value = name
+ else:
+ value = i + start_value
+ self.values.append(value)
+ setattr(self, self.get_attr_name(name), value)
- @staticmethod
- def get_attr_name(string):
- return string.upper().replace(' ', '_')
+ @staticmethod
+ def get_attr_name(string):
+ return string.upper().replace(' ', '_')
- def choices(self):
- 'Return choice list suitable for Django model choices.'
- return zip(self.values, self.names)
+ def choices(self):
+ 'Return choice list suitable for Django model choices.'
+ return zip(self.values, self.names)
- def get_value(self, name):
- """\
- Convert a string name to it's corresponding value. If a value
- is passed in, it is returned.
- """
- if isinstance(name, int) and not self.string_values:
- # name is already a value
- return name
- return getattr(self, self.get_attr_name(name))
+ def get_value(self, name):
+ """\
+ Convert a string name to it's corresponding value. If a value
+ is passed in, it is returned.
+ """
+ if isinstance(name, int) and not self.string_values:
+ # name is already a value
+ return name
+ return getattr(self, self.get_attr_name(name))
- def get_string(self, value):
- ' Given a value, get the string name for it.'
- if value not in self.values:
- raise ValueError('Value %s not in this enum' % value)
- index = self.values.index(value)
- return self.names[index]
+ def get_string(self, value):
+ ' Given a value, get the string name for it.'
+ if value not in self.values:
+ raise ValueError('Value %s not in this enum' % value)
+ index = self.values.index(value)
+ return self.names[index]
diff --git a/frontend/afe/feeds/feed.py b/frontend/afe/feeds/feed.py
index 801cfe0..d34a0b9 100644
--- a/frontend/afe/feeds/feed.py
+++ b/frontend/afe/feeds/feed.py
@@ -11,57 +11,57 @@
# name changed from feed to feed_view
def feed_view(request, url, feed_dict=None):
- if not feed_dict:
- raise Http404, "No feeds are registered."
+ if not feed_dict:
+ raise Http404, "No feeds are registered."
- try:
- slug, param = url.split('/', 1)
- except ValueError:
- slug, param = url, ''
+ try:
+ slug, param = url.split('/', 1)
+ except ValueError:
+ slug, param = url, ''
- try:
- f = feed_dict[slug]
- except KeyError:
- raise Http404, "Slug %r isn't registered." % slug
+ try:
+ f = feed_dict[slug]
+ except KeyError:
+ raise Http404, "Slug %r isn't registered." % slug
- try:
- # this line is changed from the Django library version to pass
- # in request instead of request.path
- feedgen = f(slug, request).get_feed(param)
- except feeds.FeedDoesNotExist:
- raise Http404, "Invalid feed parameters. Slug %r is valid, but other parameters, or lack thereof, are not." % slug
+ try:
+ # this line is changed from the Django library version to pass
+ # in request instead of request.path
+ feedgen = f(slug, request).get_feed(param)
+ except feeds.FeedDoesNotExist:
+ raise Http404, "Invalid feed parameters. Slug %r is valid, but other parameters, or lack thereof, are not." % slug
- response = HttpResponse(mimetype=feedgen.mime_type)
- feedgen.write(response, 'utf-8')
- return response
+ response = HttpResponse(mimetype=feedgen.mime_type)
+ feedgen.write(response, 'utf-8')
+ return response
# end copied code
class JobFeed(feeds.Feed):
- """\
- Common feed functionality.
- """
- link = "/results"
- title_template = "feeds/job_feed_title.html"
- description_template = "feeds/job_feed_description.html"
+ """\
+ Common feed functionality.
+ """
+ link = "/results"
+ title_template = "feeds/job_feed_title.html"
+ description_template = "feeds/job_feed_description.html"
- NUM_ITEMS = 20
+ NUM_ITEMS = 20
- def __init__(self, slug, request):
- super(JobFeed, self).__init__(slug, request.path)
- server_hostname = django.http.get_host(request)
- self.full_link = 'http://' + server_hostname + self.link
+ def __init__(self, slug, request):
+ super(JobFeed, self).__init__(slug, request.path)
+ server_hostname = django.http.get_host(request)
+ self.full_link = 'http://' + server_hostname + self.link
- def title(self, obj):
- return "Automated Test Framework %s Jobs" % obj.capitalize()
+ def title(self, obj):
+ return "Automated Test Framework %s Jobs" % obj.capitalize()
- def get_object(self, bits):
- # bits[0] should be a job status
- return bits[0]
+ def get_object(self, bits):
+ # bits[0] should be a job status
+ return bits[0]
- def items(self, obj):
- item_list = models.HostQueueEntry.objects.filter(
- status__iexact=obj).select_related()
- return item_list.order_by('-id')[:self.NUM_ITEMS]
+ def items(self, obj):
+ item_list = models.HostQueueEntry.objects.filter(
+ status__iexact=obj).select_related()
+ return item_list.order_by('-id')[:self.NUM_ITEMS]
- def item_link(self, obj):
- return '%s/%s-%s' % (self.full_link, obj.job.id, obj.job.owner)
+ def item_link(self, obj):
+ return '%s/%s-%s' % (self.full_link, obj.job.id, obj.job.owner)
diff --git a/frontend/afe/json_rpc/proxy.py b/frontend/afe/json_rpc/proxy.py
index d33fc96..865ab3c 100644
--- a/frontend/afe/json_rpc/proxy.py
+++ b/frontend/afe/json_rpc/proxy.py
@@ -39,20 +39,20 @@
return ServiceProxy(self.__serviceURL, name, self.__headers)
def __call__(self, *args, **kwargs):
- postdata = json_encoder.encode({"method": self.__serviceName,
- 'params': args + (kwargs,),
- 'id':'jsonrpc'})
- request = urllib2.Request(self.__serviceURL, data=postdata,
- headers=self.__headers)
- respdata = urllib2.urlopen(request).read()
- try:
- resp = json_decoder.decode(respdata)
- except ValueError:
- raise JSONRPCException('Error decoding JSON reponse:\n' + respdata)
- if resp['error'] != None:
- error_message = (resp['error']['name'] + ': ' +
- resp['error']['message'] + '\n' +
- resp['error']['traceback'])
- raise JSONRPCException(error_message)
- else:
- return resp['result']
+ postdata = json_encoder.encode({"method": self.__serviceName,
+ 'params': args + (kwargs,),
+ 'id':'jsonrpc'})
+ request = urllib2.Request(self.__serviceURL, data=postdata,
+ headers=self.__headers)
+ respdata = urllib2.urlopen(request).read()
+ try:
+ resp = json_decoder.decode(respdata)
+ except ValueError:
+ raise JSONRPCException('Error decoding JSON reponse:\n' + respdata)
+ if resp['error'] != None:
+ error_message = (resp['error']['name'] + ': ' +
+ resp['error']['message'] + '\n' +
+ resp['error']['traceback'])
+ raise JSONRPCException(error_message)
+ else:
+ return resp['result']
diff --git a/frontend/afe/json_rpc/serviceHandler.py b/frontend/afe/json_rpc/serviceHandler.py
index afd0b04..630363b 100644
--- a/frontend/afe/json_rpc/serviceHandler.py
+++ b/frontend/afe/json_rpc/serviceHandler.py
@@ -69,7 +69,7 @@
def __init__(self, service):
self.service=service
-
+
def handleRequest(self, json):
err=None
err_traceback = None
@@ -77,7 +77,7 @@
id_=''
#print 'Request:', json
-
+
try:
req = self.translateRequest(json)
except ServiceRequestNotTranslatable, e:
diff --git a/frontend/afe/management.py b/frontend/afe/management.py
index c8f8380..506a218 100644
--- a/frontend/afe/management.py
+++ b/frontend/afe/management.py
@@ -8,32 +8,32 @@
BASIC_ADMIN = 'Basic admin'
def create_admin_group(app, created_models, verbosity, **kwargs):
- """\
- Create a basic admin group with permissions for managing basic autotest
- objects.
- """
- admin_group, created = auth.models.Group.objects.get_or_create(
- name=BASIC_ADMIN)
- admin_group.save() # must save before adding permissions
- PermissionModel = auth.models.Permission
- have_permissions = list(admin_group.permissions.all())
- for model_name in ('host', 'label', 'test', 'acl_group'):
- for permission_type in ('add', 'change', 'delete'):
- codename = permission_type + '_' + model_name
- permissions = list(PermissionModel.objects.filter(
- codename=codename))
- if len(permissions) == 0:
- print ' No permission ' + codename
- continue
- for permission in permissions:
- if permission not in have_permissions:
- print ' Adding permission ' + codename
- admin_group.permissions.add(permission)
- if created:
- print 'Created group "%s"' % BASIC_ADMIN
- else:
- print 'Group "%s" already exists' % BASIC_ADMIN
+ """\
+ Create a basic admin group with permissions for managing basic autotest
+ objects.
+ """
+ admin_group, created = auth.models.Group.objects.get_or_create(
+ name=BASIC_ADMIN)
+ admin_group.save() # must save before adding permissions
+ PermissionModel = auth.models.Permission
+ have_permissions = list(admin_group.permissions.all())
+ for model_name in ('host', 'label', 'test', 'acl_group'):
+ for permission_type in ('add', 'change', 'delete'):
+ codename = permission_type + '_' + model_name
+ permissions = list(PermissionModel.objects.filter(
+ codename=codename))
+ if len(permissions) == 0:
+ print ' No permission ' + codename
+ continue
+ for permission in permissions:
+ if permission not in have_permissions:
+ print ' Adding permission ' + codename
+ admin_group.permissions.add(permission)
+ if created:
+ print 'Created group "%s"' % BASIC_ADMIN
+ else:
+ print 'Group "%s" already exists' % BASIC_ADMIN
dispatcher.connect(create_admin_group, sender=frontend.afe.models,
- signal=signals.post_syncdb)
+ signal=signals.post_syncdb)
diff --git a/frontend/afe/model_logic.py b/frontend/afe/model_logic.py
index f119710..9c248e1 100644
--- a/frontend/afe/model_logic.py
+++ b/frontend/afe/model_logic.py
@@ -7,575 +7,575 @@
class ValidationError(Exception):
- """\
- Data validation error in adding or updating an object. The associated
- value is a dictionary mapping field names to error strings.
- """
+ """\
+ Data validation error in adding or updating an object. The associated
+ value is a dictionary mapping field names to error strings.
+ """
class ExtendedManager(dbmodels.Manager):
- """\
- Extended manager supporting subquery filtering.
- """
+ """\
+ Extended manager supporting subquery filtering.
+ """
- class _RawSqlQ(dbmodels.Q):
- """\
- A Django "Q" object constructed with a raw SQL query.
- """
- def __init__(self, sql, params=[], joins={}):
- """
- sql: the SQL to go into the WHERE clause
+ class _RawSqlQ(dbmodels.Q):
+ """\
+ A Django "Q" object constructed with a raw SQL query.
+ """
+ def __init__(self, sql, params=[], joins={}):
+ """
+ sql: the SQL to go into the WHERE clause
- params: substitution params for the WHERE SQL
+ params: substitution params for the WHERE SQL
- joins: a dict mapping alias to (table, join_type,
- condition). This converts to the SQL:
- "join_type table AS alias ON condition"
- For example:
- alias='host_hqe',
- table='host_queue_entries',
- join_type='INNER JOIN',
- condition='host_hqe.host_id=hosts.id'
- """
- self._sql = sql
- self._params = params[:]
- self._joins = datastructures.SortedDict(joins)
+ joins: a dict mapping alias to (table, join_type,
+ condition). This converts to the SQL:
+ "join_type table AS alias ON condition"
+ For example:
+ alias='host_hqe',
+ table='host_queue_entries',
+ join_type='INNER JOIN',
+ condition='host_hqe.host_id=hosts.id'
+ """
+ self._sql = sql
+ self._params = params[:]
+ self._joins = datastructures.SortedDict(joins)
- def get_sql(self, opts):
- return (self._joins,
- [self._sql],
- self._params)
+ def get_sql(self, opts):
+ return (self._joins,
+ [self._sql],
+ self._params)
- @staticmethod
- def _get_quoted_field(table, field):
- return (backend.quote_name(table) + '.' +
- backend.quote_name(field))
+ @staticmethod
+ def _get_quoted_field(table, field):
+ return (backend.quote_name(table) + '.' +
+ backend.quote_name(field))
- @classmethod
- def _get_sql_string_for(cls, value):
- """
- >>> ExtendedManager._get_sql_string_for((1L, 2L))
- '(1,2)'
- >>> ExtendedManager._get_sql_string_for(['abc', 'def'])
- 'abc,def'
- """
- if isinstance(value, list):
- return ','.join(cls._get_sql_string_for(item)
- for item in value)
- if isinstance(value, tuple):
- return '(%s)' % cls._get_sql_string_for(list(value))
- if isinstance(value, long):
- return str(int(value))
- return str(value)
+ @classmethod
+ def _get_sql_string_for(cls, value):
+ """
+ >>> ExtendedManager._get_sql_string_for((1L, 2L))
+ '(1,2)'
+ >>> ExtendedManager._get_sql_string_for(['abc', 'def'])
+ 'abc,def'
+ """
+ if isinstance(value, list):
+ return ','.join(cls._get_sql_string_for(item)
+ for item in value)
+ if isinstance(value, tuple):
+ return '(%s)' % cls._get_sql_string_for(list(value))
+ if isinstance(value, long):
+ return str(int(value))
+ return str(value)
- @staticmethod
- def _get_sql_query_for(query_object, select_field):
- query_table = query_object.model._meta.db_table
- quoted_field = ExtendedManager._get_quoted_field(query_table,
- select_field)
- _, where, params = query_object._get_sql_clause()
- # where includes the FROM clause
- return '(SELECT DISTINCT ' + quoted_field + where + ')', params
+ @staticmethod
+ def _get_sql_query_for(query_object, select_field):
+ query_table = query_object.model._meta.db_table
+ quoted_field = ExtendedManager._get_quoted_field(query_table,
+ select_field)
+ _, where, params = query_object._get_sql_clause()
+ # where includes the FROM clause
+ return '(SELECT DISTINCT ' + quoted_field + where + ')', params
- def _get_key_on_this_table(self, key_field=None):
- if key_field is None:
- # default to primary key
- key_field = self.model._meta.pk.column
- return self._get_quoted_field(self.model._meta.db_table,
- key_field)
+ def _get_key_on_this_table(self, key_field=None):
+ if key_field is None:
+ # default to primary key
+ key_field = self.model._meta.pk.column
+ return self._get_quoted_field(self.model._meta.db_table,
+ key_field)
- def _do_subquery_filter(self, subquery_key, subquery, subquery_alias,
- this_table_key=None, not_in=False):
- """
- This method constructs SQL queries to accomplish IN/NOT IN
- subquery filtering using explicit joins. It does this by
- LEFT JOINing onto the subquery and then checking to see if
- the joined column is NULL or not.
+ def _do_subquery_filter(self, subquery_key, subquery, subquery_alias,
+ this_table_key=None, not_in=False):
+ """
+ This method constructs SQL queries to accomplish IN/NOT IN
+ subquery filtering using explicit joins. It does this by
+ LEFT JOINing onto the subquery and then checking to see if
+ the joined column is NULL or not.
- We use explicit joins instead of the SQL IN operator because
- MySQL (at least some versions) considers all IN subqueries to be
- dependent, so using explicit joins can be MUCH faster.
+ We use explicit joins instead of the SQL IN operator because
+ MySQL (at least some versions) considers all IN subqueries to be
+ dependent, so using explicit joins can be MUCH faster.
- The query we're going for is:
- SELECT * FROM <this table>
- LEFT JOIN (<subquery>) AS <subquery_alias>
- ON <subquery_alias>.<subquery_key> =
- <this table>.<this_table_key>
- WHERE <subquery_alias>.<subquery_key> IS [NOT] NULL
- """
- subselect, params = self._get_sql_query_for(subquery,
- subquery_key)
+ The query we're going for is:
+ SELECT * FROM <this table>
+ LEFT JOIN (<subquery>) AS <subquery_alias>
+ ON <subquery_alias>.<subquery_key> =
+ <this table>.<this_table_key>
+ WHERE <subquery_alias>.<subquery_key> IS [NOT] NULL
+ """
+ subselect, params = self._get_sql_query_for(subquery,
+ subquery_key)
- this_full_key = self._get_key_on_this_table(this_table_key)
- alias_full_key = self._get_quoted_field(subquery_alias,
- subquery_key)
- join_condition = alias_full_key + ' = ' + this_full_key
- joins = {subquery_alias : (subselect, # join table
- 'LEFT JOIN', # join type
- join_condition)} # join on
+ this_full_key = self._get_key_on_this_table(this_table_key)
+ alias_full_key = self._get_quoted_field(subquery_alias,
+ subquery_key)
+ join_condition = alias_full_key + ' = ' + this_full_key
+ joins = {subquery_alias : (subselect, # join table
+ 'LEFT JOIN', # join type
+ join_condition)} # join on
- if not_in:
- where_sql = alias_full_key + ' IS NULL'
- else:
- where_sql = alias_full_key + ' IS NOT NULL'
- filter_obj = self._RawSqlQ(where_sql, params, joins)
- return self.complex_filter(filter_obj)
+ if not_in:
+ where_sql = alias_full_key + ' IS NULL'
+ else:
+ where_sql = alias_full_key + ' IS NOT NULL'
+ filter_obj = self._RawSqlQ(where_sql, params, joins)
+ return self.complex_filter(filter_obj)
- def filter_in_subquery(self, subquery_key, subquery, subquery_alias,
- this_table_key=None):
- """\
- Construct a filter to perform a subquery match, i.e.
- WHERE id IN (SELECT host_id FROM ... WHERE ...)
- -subquery_key - the field to select in the subquery (host_id
- above)
- -subquery - a query object for the subquery
- -subquery_alias - a logical name for the query, to be used in
- the SQL (i.e. 'valid_hosts')
- -this_table_key - the field to match (id above). Defaults to
- this table's primary key.
- """
- return self._do_subquery_filter(subquery_key, subquery,
- subquery_alias, this_table_key)
+ def filter_in_subquery(self, subquery_key, subquery, subquery_alias,
+ this_table_key=None):
+ """\
+ Construct a filter to perform a subquery match, i.e.
+ WHERE id IN (SELECT host_id FROM ... WHERE ...)
+ -subquery_key - the field to select in the subquery (host_id
+ above)
+ -subquery - a query object for the subquery
+ -subquery_alias - a logical name for the query, to be used in
+ the SQL (i.e. 'valid_hosts')
+ -this_table_key - the field to match (id above). Defaults to
+ this table's primary key.
+ """
+ return self._do_subquery_filter(subquery_key, subquery,
+ subquery_alias, this_table_key)
- def filter_not_in_subquery(self, subquery_key, subquery,
- subquery_alias, this_table_key=None):
- 'Like filter_in_subquery, but use NOT IN rather than IN.'
- return self._do_subquery_filter(subquery_key, subquery,
- subquery_alias, this_table_key,
- not_in=True)
+ def filter_not_in_subquery(self, subquery_key, subquery,
+ subquery_alias, this_table_key=None):
+ 'Like filter_in_subquery, but use NOT IN rather than IN.'
+ return self._do_subquery_filter(subquery_key, subquery,
+ subquery_alias, this_table_key,
+ not_in=True)
- def create_in_bulk(self, fields, values):
- """
- Creates many objects with a single SQL query.
- field - list of field names (model attributes, not actual DB
- field names) for which values will be specified.
- values - list of tuples containing values. Each tuple contains
- the values for the specified fields for a single
- object.
- Example: Host.objects.create_in_bulk(['hostname', 'status'],
- [('host1', 'Ready'), ('host2', 'Running')])
- """
- if not values:
- return
- field_dict = self.model.get_field_dict()
- field_names = [field_dict[field].column for field in fields]
- sql = 'INSERT INTO %s %s' % (
- self.model._meta.db_table,
- self._get_sql_string_for(tuple(field_names)))
- sql += ' VALUES ' + self._get_sql_string_for(list(values))
- cursor = connection.cursor()
- cursor.execute(sql)
- connection._commit()
+ def create_in_bulk(self, fields, values):
+ """
+ Creates many objects with a single SQL query.
+ field - list of field names (model attributes, not actual DB
+ field names) for which values will be specified.
+ values - list of tuples containing values. Each tuple contains
+ the values for the specified fields for a single
+ object.
+ Example: Host.objects.create_in_bulk(['hostname', 'status'],
+ [('host1', 'Ready'), ('host2', 'Running')])
+ """
+ if not values:
+ return
+ field_dict = self.model.get_field_dict()
+ field_names = [field_dict[field].column for field in fields]
+ sql = 'INSERT INTO %s %s' % (
+ self.model._meta.db_table,
+ self._get_sql_string_for(tuple(field_names)))
+ sql += ' VALUES ' + self._get_sql_string_for(list(values))
+ cursor = connection.cursor()
+ cursor.execute(sql)
+ connection._commit()
- def delete_in_bulk(self, ids):
- """
- Deletes many objects with a single SQL query. ids should be a
- list of object ids to delete. Nonexistent ids will be silently
- ignored.
- """
- if not ids:
- return
- sql = 'DELETE FROM %s WHERE id IN %s' % (
- self.model._meta.db_table,
- self._get_sql_string_for(tuple(ids)))
- cursor = connection.cursor()
- cursor.execute(sql)
- connection._commit()
+ def delete_in_bulk(self, ids):
+ """
+ Deletes many objects with a single SQL query. ids should be a
+ list of object ids to delete. Nonexistent ids will be silently
+ ignored.
+ """
+ if not ids:
+ return
+ sql = 'DELETE FROM %s WHERE id IN %s' % (
+ self.model._meta.db_table,
+ self._get_sql_string_for(tuple(ids)))
+ cursor = connection.cursor()
+ cursor.execute(sql)
+ connection._commit()
class ValidObjectsManager(ExtendedManager):
- """
- Manager returning only objects with invalid=False.
- """
- def get_query_set(self):
- queryset = super(ValidObjectsManager, self).get_query_set()
- return queryset.filter(invalid=False)
+ """
+ Manager returning only objects with invalid=False.
+ """
+ def get_query_set(self):
+ queryset = super(ValidObjectsManager, self).get_query_set()
+ return queryset.filter(invalid=False)
class ModelExtensions(object):
- """\
- Mixin with convenience functions for models, built on top of the
- default Django model functions.
- """
- # TODO: at least some of these functions really belong in a custom
- # Manager class
+ """\
+ Mixin with convenience functions for models, built on top of the
+ default Django model functions.
+ """
+ # TODO: at least some of these functions really belong in a custom
+ # Manager class
- field_dict = None
- # subclasses should override if they want to support smart_get() by name
- name_field = None
+ field_dict = None
+ # subclasses should override if they want to support smart_get() by name
+ name_field = None
- @classmethod
- def get_field_dict(cls):
- if cls.field_dict is None:
- cls.field_dict = {}
- for field in cls._meta.fields:
- cls.field_dict[field.name] = field
- return cls.field_dict
+ @classmethod
+ def get_field_dict(cls):
+ if cls.field_dict is None:
+ cls.field_dict = {}
+ for field in cls._meta.fields:
+ cls.field_dict[field.name] = field
+ return cls.field_dict
- @classmethod
- def clean_foreign_keys(cls, data):
- """\
- -Convert foreign key fields in data from <field>_id to just
- <field>.
- -replace foreign key objects with their IDs
- This method modifies data in-place.
- """
- for field in cls._meta.fields:
- if not field.rel:
- continue
- if (field.attname != field.name and
- field.attname in data):
- data[field.name] = data[field.attname]
- del data[field.attname]
- value = data[field.name]
- if isinstance(value, dbmodels.Model):
- data[field.name] = value.id
+ @classmethod
+ def clean_foreign_keys(cls, data):
+ """\
+ -Convert foreign key fields in data from <field>_id to just
+ <field>.
+ -replace foreign key objects with their IDs
+ This method modifies data in-place.
+ """
+ for field in cls._meta.fields:
+ if not field.rel:
+ continue
+ if (field.attname != field.name and
+ field.attname in data):
+ data[field.name] = data[field.attname]
+ del data[field.attname]
+ value = data[field.name]
+ if isinstance(value, dbmodels.Model):
+ data[field.name] = value.id
- # TODO(showard) - is there a way to not have to do this?
- @classmethod
- def provide_default_values(cls, data):
- """\
- Provide default values for fields with default values which have
- nothing passed in.
+ # TODO(showard) - is there a way to not have to do this?
+ @classmethod
+ def provide_default_values(cls, data):
+ """\
+ Provide default values for fields with default values which have
+ nothing passed in.
- For CharField and TextField fields with "blank=True", if nothing
- is passed, we fill in an empty string value, even if there's no
- default set.
- """
- new_data = dict(data)
- field_dict = cls.get_field_dict()
- for name, obj in field_dict.iteritems():
- if data.get(name) is not None:
- continue
- if obj.default is not dbmodels.fields.NOT_PROVIDED:
- new_data[name] = obj.default
- elif (isinstance(obj, dbmodels.CharField) or
- isinstance(obj, dbmodels.TextField)):
- new_data[name] = ''
- return new_data
+ For CharField and TextField fields with "blank=True", if nothing
+ is passed, we fill in an empty string value, even if there's no
+ default set.
+ """
+ new_data = dict(data)
+ field_dict = cls.get_field_dict()
+ for name, obj in field_dict.iteritems():
+ if data.get(name) is not None:
+ continue
+ if obj.default is not dbmodels.fields.NOT_PROVIDED:
+ new_data[name] = obj.default
+ elif (isinstance(obj, dbmodels.CharField) or
+ isinstance(obj, dbmodels.TextField)):
+ new_data[name] = ''
+ return new_data
- @classmethod
- def convert_human_readable_values(cls, data, to_human_readable=False):
- """\
- Performs conversions on user-supplied field data, to make it
- easier for users to pass human-readable data.
+ @classmethod
+ def convert_human_readable_values(cls, data, to_human_readable=False):
+ """\
+ Performs conversions on user-supplied field data, to make it
+ easier for users to pass human-readable data.
- For all fields that have choice sets, convert their values
- from human-readable strings to enum values, if necessary. This
- allows users to pass strings instead of the corresponding
- integer values.
+ For all fields that have choice sets, convert their values
+ from human-readable strings to enum values, if necessary. This
+ allows users to pass strings instead of the corresponding
+ integer values.
- For all foreign key fields, call smart_get with the supplied
- data. This allows the user to pass either an ID value or
- the name of the object as a string.
+ For all foreign key fields, call smart_get with the supplied
+ data. This allows the user to pass either an ID value or
+ the name of the object as a string.
- If to_human_readable=True, perform the inverse - i.e. convert
- numeric values to human readable values.
+ If to_human_readable=True, perform the inverse - i.e. convert
+ numeric values to human readable values.
- This method modifies data in-place.
- """
- field_dict = cls.get_field_dict()
- for field_name in data:
- if data[field_name] is None:
- continue
- field_obj = field_dict[field_name]
- # convert enum values
- if field_obj.choices:
- for choice_data in field_obj.choices:
- # choice_data is (value, name)
- if to_human_readable:
- from_val, to_val = choice_data
- else:
- to_val, from_val = choice_data
- if from_val == data[field_name]:
- data[field_name] = to_val
- break
- # convert foreign key values
- elif field_obj.rel:
- dest_obj = field_obj.rel.to.smart_get(
- data[field_name])
- if (to_human_readable and
- dest_obj.name_field is not None):
- data[field_name] = (
- getattr(dest_obj,
- dest_obj.name_field))
- else:
- data[field_name] = dest_obj.id
+ This method modifies data in-place.
+ """
+ field_dict = cls.get_field_dict()
+ for field_name in data:
+ if data[field_name] is None:
+ continue
+ field_obj = field_dict[field_name]
+ # convert enum values
+ if field_obj.choices:
+ for choice_data in field_obj.choices:
+ # choice_data is (value, name)
+ if to_human_readable:
+ from_val, to_val = choice_data
+ else:
+ to_val, from_val = choice_data
+ if from_val == data[field_name]:
+ data[field_name] = to_val
+ break
+ # convert foreign key values
+ elif field_obj.rel:
+ dest_obj = field_obj.rel.to.smart_get(
+ data[field_name])
+ if (to_human_readable and
+ dest_obj.name_field is not None):
+ data[field_name] = (
+ getattr(dest_obj,
+ dest_obj.name_field))
+ else:
+ data[field_name] = dest_obj.id
- @classmethod
- def validate_field_names(cls, data):
- 'Checks for extraneous fields in data.'
- errors = {}
- field_dict = cls.get_field_dict()
- for field_name in data:
- if field_name not in field_dict:
- errors[field_name] = 'No field of this name'
- return errors
+ @classmethod
+ def validate_field_names(cls, data):
+ 'Checks for extraneous fields in data.'
+ errors = {}
+ field_dict = cls.get_field_dict()
+ for field_name in data:
+ if field_name not in field_dict:
+ errors[field_name] = 'No field of this name'
+ return errors
- @classmethod
- def prepare_data_args(cls, data, kwargs):
- 'Common preparation for add_object and update_object'
- data = dict(data) # don't modify the default keyword arg
- data.update(kwargs)
- # must check for extraneous field names here, while we have the
- # data in a dict
- errors = cls.validate_field_names(data)
- if errors:
- raise ValidationError(errors)
- cls.convert_human_readable_values(data)
- return data
+ @classmethod
+ def prepare_data_args(cls, data, kwargs):
+ 'Common preparation for add_object and update_object'
+ data = dict(data) # don't modify the default keyword arg
+ data.update(kwargs)
+ # must check for extraneous field names here, while we have the
+ # data in a dict
+ errors = cls.validate_field_names(data)
+ if errors:
+ raise ValidationError(errors)
+ cls.convert_human_readable_values(data)
+ return data
- def validate_unique(self):
- """\
- Validate that unique fields are unique. Django manipulators do
- this too, but they're a huge pain to use manually. Trust me.
- """
- errors = {}
- cls = type(self)
- field_dict = self.get_field_dict()
- manager = cls.get_valid_manager()
- for field_name, field_obj in field_dict.iteritems():
- if not field_obj.unique:
- continue
+ def validate_unique(self):
+ """\
+ Validate that unique fields are unique. Django manipulators do
+ this too, but they're a huge pain to use manually. Trust me.
+ """
+ errors = {}
+ cls = type(self)
+ field_dict = self.get_field_dict()
+ manager = cls.get_valid_manager()
+ for field_name, field_obj in field_dict.iteritems():
+ if not field_obj.unique:
+ continue
- value = getattr(self, field_name)
- existing_objs = manager.filter(**{field_name : value})
- num_existing = existing_objs.count()
+ value = getattr(self, field_name)
+ existing_objs = manager.filter(**{field_name : value})
+ num_existing = existing_objs.count()
- if num_existing == 0:
- continue
- if num_existing == 1 and existing_objs[0].id == self.id:
- continue
- errors[field_name] = (
- 'This value must be unique (%s)' % (value))
- return errors
+ if num_existing == 0:
+ continue
+ if num_existing == 1 and existing_objs[0].id == self.id:
+ continue
+ errors[field_name] = (
+ 'This value must be unique (%s)' % (value))
+ return errors
- def do_validate(self):
- errors = self.validate()
- unique_errors = self.validate_unique()
- for field_name, error in unique_errors.iteritems():
- errors.setdefault(field_name, error)
- if errors:
- raise ValidationError(errors)
+ def do_validate(self):
+ errors = self.validate()
+ unique_errors = self.validate_unique()
+ for field_name, error in unique_errors.iteritems():
+ errors.setdefault(field_name, error)
+ if errors:
+ raise ValidationError(errors)
- # actually (externally) useful methods follow
+ # actually (externally) useful methods follow
- @classmethod
- def add_object(cls, data={}, **kwargs):
- """\
- Returns a new object created with the given data (a dictionary
- mapping field names to values). Merges any extra keyword args
- into data.
- """
- data = cls.prepare_data_args(data, kwargs)
- data = cls.provide_default_values(data)
- obj = cls(**data)
- obj.do_validate()
- obj.save()
- return obj
+ @classmethod
+ def add_object(cls, data={}, **kwargs):
+ """\
+ Returns a new object created with the given data (a dictionary
+ mapping field names to values). Merges any extra keyword args
+ into data.
+ """
+ data = cls.prepare_data_args(data, kwargs)
+ data = cls.provide_default_values(data)
+ obj = cls(**data)
+ obj.do_validate()
+ obj.save()
+ return obj
- def update_object(self, data={}, **kwargs):
- """\
- Updates the object with the given data (a dictionary mapping
- field names to values). Merges any extra keyword args into
- data.
- """
- data = self.prepare_data_args(data, kwargs)
- for field_name, value in data.iteritems():
- if value is not None:
- setattr(self, field_name, value)
- self.do_validate()
- self.save()
+ def update_object(self, data={}, **kwargs):
+ """\
+ Updates the object with the given data (a dictionary mapping
+ field names to values). Merges any extra keyword args into
+ data.
+ """
+ data = self.prepare_data_args(data, kwargs)
+ for field_name, value in data.iteritems():
+ if value is not None:
+ setattr(self, field_name, value)
+ self.do_validate()
+ self.save()
- @classmethod
- def query_objects(cls, filter_data, valid_only=True):
- """\
- Returns a QuerySet object for querying the given model_class
- with the given filter_data. Optional special arguments in
- filter_data include:
- -query_start: index of first return to return
- -query_limit: maximum number of results to return
- -sort_by: list of fields to sort on. prefixing a '-' onto a
- field name changes the sort to descending order.
- -extra_args: keyword args to pass to query.extra() (see Django
- DB layer documentation)
- -extra_where: extra WHERE clause to append
- """
- query_start = filter_data.pop('query_start', None)
- query_limit = filter_data.pop('query_limit', None)
- if query_start and not query_limit:
- raise ValueError('Cannot pass query_start without '
- 'query_limit')
- sort_by = filter_data.pop('sort_by', [])
- extra_args = filter_data.pop('extra_args', {})
- extra_where = filter_data.pop('extra_where', None)
- if extra_where:
- extra_args.setdefault('where', []).append(extra_where)
+ @classmethod
+ def query_objects(cls, filter_data, valid_only=True):
+ """\
+ Returns a QuerySet object for querying the given model_class
+ with the given filter_data. Optional special arguments in
+ filter_data include:
+ -query_start: index of first return to return
+ -query_limit: maximum number of results to return
+ -sort_by: list of fields to sort on. prefixing a '-' onto a
+ field name changes the sort to descending order.
+ -extra_args: keyword args to pass to query.extra() (see Django
+ DB layer documentation)
+ -extra_where: extra WHERE clause to append
+ """
+ query_start = filter_data.pop('query_start', None)
+ query_limit = filter_data.pop('query_limit', None)
+ if query_start and not query_limit:
+ raise ValueError('Cannot pass query_start without '
+ 'query_limit')
+ sort_by = filter_data.pop('sort_by', [])
+ extra_args = filter_data.pop('extra_args', {})
+ extra_where = filter_data.pop('extra_where', None)
+ if extra_where:
+ extra_args.setdefault('where', []).append(extra_where)
- # filters
- query_dict = {}
- for field, value in filter_data.iteritems():
- query_dict[field] = value
- if valid_only:
- manager = cls.get_valid_manager()
- else:
- manager = cls.objects
- query = manager.filter(**query_dict).distinct()
+ # filters
+ query_dict = {}
+ for field, value in filter_data.iteritems():
+ query_dict[field] = value
+ if valid_only:
+ manager = cls.get_valid_manager()
+ else:
+ manager = cls.objects
+ query = manager.filter(**query_dict).distinct()
- # other arguments
- if extra_args:
- query = query.extra(**extra_args)
+ # other arguments
+ if extra_args:
+ query = query.extra(**extra_args)
- # sorting + paging
- assert isinstance(sort_by, list) or isinstance(sort_by, tuple)
- query = query.order_by(*sort_by)
- if query_start is not None and query_limit is not None:
- query_limit += query_start
- return query[query_start:query_limit]
+ # sorting + paging
+ assert isinstance(sort_by, list) or isinstance(sort_by, tuple)
+ query = query.order_by(*sort_by)
+ if query_start is not None and query_limit is not None:
+ query_limit += query_start
+ return query[query_start:query_limit]
- @classmethod
- def query_count(cls, filter_data):
- """\
- Like query_objects, but retreive only the count of results.
- """
- filter_data.pop('query_start', None)
- filter_data.pop('query_limit', None)
- return cls.query_objects(filter_data).count()
+ @classmethod
+ def query_count(cls, filter_data):
+ """\
+ Like query_objects, but retreive only the count of results.
+ """
+ filter_data.pop('query_start', None)
+ filter_data.pop('query_limit', None)
+ return cls.query_objects(filter_data).count()
- @classmethod
- def clean_object_dicts(cls, field_dicts):
- """\
- Take a list of dicts corresponding to object (as returned by
- query.values()) and clean the data to be more suitable for
- returning to the user.
- """
- for i in range(len(field_dicts)):
- cls.clean_foreign_keys(field_dicts[i])
- cls.convert_human_readable_values(
- field_dicts[i], to_human_readable=True)
+ @classmethod
+ def clean_object_dicts(cls, field_dicts):
+ """\
+ Take a list of dicts corresponding to object (as returned by
+ query.values()) and clean the data to be more suitable for
+ returning to the user.
+ """
+ for i in range(len(field_dicts)):
+ cls.clean_foreign_keys(field_dicts[i])
+ cls.convert_human_readable_values(
+ field_dicts[i], to_human_readable=True)
- @classmethod
- def list_objects(cls, filter_data):
- """\
- Like query_objects, but return a list of dictionaries.
- """
- query = cls.query_objects(filter_data)
- field_dicts = list(query.values())
- cls.clean_object_dicts(field_dicts)
- return field_dicts
+ @classmethod
+ def list_objects(cls, filter_data):
+ """\
+ Like query_objects, but return a list of dictionaries.
+ """
+ query = cls.query_objects(filter_data)
+ field_dicts = list(query.values())
+ cls.clean_object_dicts(field_dicts)
+ return field_dicts
- @classmethod
- def smart_get(cls, *args, **kwargs):
- """\
- smart_get(integer) -> get object by ID
- smart_get(string) -> get object by name_field
- smart_get(keyword args) -> normal ModelClass.objects.get()
- """
- assert bool(args) ^ bool(kwargs)
- if args:
- assert len(args) == 1
- arg = args[0]
- if isinstance(arg, int) or isinstance(arg, long):
- return cls.objects.get(id=arg)
- if isinstance(arg, str) or isinstance(arg, unicode):
- return cls.objects.get(
- **{cls.name_field : arg})
- raise ValueError(
- 'Invalid positional argument: %s (%s)' % (
- str(arg), type(arg)))
- return cls.objects.get(**kwargs)
+ @classmethod
+ def smart_get(cls, *args, **kwargs):
+ """\
+ smart_get(integer) -> get object by ID
+ smart_get(string) -> get object by name_field
+ smart_get(keyword args) -> normal ModelClass.objects.get()
+ """
+ assert bool(args) ^ bool(kwargs)
+ if args:
+ assert len(args) == 1
+ arg = args[0]
+ if isinstance(arg, int) or isinstance(arg, long):
+ return cls.objects.get(id=arg)
+ if isinstance(arg, str) or isinstance(arg, unicode):
+ return cls.objects.get(
+ **{cls.name_field : arg})
+ raise ValueError(
+ 'Invalid positional argument: %s (%s)' % (
+ str(arg), type(arg)))
+ return cls.objects.get(**kwargs)
- def get_object_dict(self):
- """\
- Return a dictionary mapping fields to this object's values.
- """
- object_dict = dict((field_name, getattr(self, field_name))
- for field_name
- in self.get_field_dict().iterkeys())
- self.clean_object_dicts([object_dict])
- return object_dict
+ def get_object_dict(self):
+ """\
+ Return a dictionary mapping fields to this object's values.
+ """
+ object_dict = dict((field_name, getattr(self, field_name))
+ for field_name
+ in self.get_field_dict().iterkeys())
+ self.clean_object_dicts([object_dict])
+ return object_dict
- @classmethod
- def get_valid_manager(cls):
- return cls.objects
+ @classmethod
+ def get_valid_manager(cls):
+ return cls.objects
class ModelWithInvalid(ModelExtensions):
- """
- Overrides model methods save() and delete() to support invalidation in
- place of actual deletion. Subclasses must have a boolean "invalid"
- field.
- """
+ """
+ Overrides model methods save() and delete() to support invalidation in
+ place of actual deletion. Subclasses must have a boolean "invalid"
+ field.
+ """
- def save(self):
- # see if this object was previously added and invalidated
- my_name = getattr(self, self.name_field)
- filters = {self.name_field : my_name, 'invalid' : True}
- try:
- old_object = self.__class__.objects.get(**filters)
- except self.DoesNotExist:
- # no existing object
- super(ModelWithInvalid, self).save()
- return
+ def save(self):
+ # see if this object was previously added and invalidated
+ my_name = getattr(self, self.name_field)
+ filters = {self.name_field : my_name, 'invalid' : True}
+ try:
+ old_object = self.__class__.objects.get(**filters)
+ except self.DoesNotExist:
+ # no existing object
+ super(ModelWithInvalid, self).save()
+ return
- self.id = old_object.id
- super(ModelWithInvalid, self).save()
+ self.id = old_object.id
+ super(ModelWithInvalid, self).save()
- def clean_object(self):
- """
- This method is called when an object is marked invalid.
- Subclasses should override this to clean up relationships that
- should no longer exist if the object were deleted."""
- pass
+ def clean_object(self):
+ """
+ This method is called when an object is marked invalid.
+ Subclasses should override this to clean up relationships that
+ should no longer exist if the object were deleted."""
+ pass
- def delete(self):
- assert not self.invalid
- self.invalid = True
- self.save()
- self.clean_object()
+ def delete(self):
+ assert not self.invalid
+ self.invalid = True
+ self.save()
+ self.clean_object()
- @classmethod
- def get_valid_manager(cls):
- return cls.valid_objects
+ @classmethod
+ def get_valid_manager(cls):
+ return cls.valid_objects
- class Manipulator(object):
- """
- Force default manipulators to look only at valid objects -
- otherwise they will match against invalid objects when checking
- uniqueness.
- """
- @classmethod
- def _prepare(cls, model):
- super(ModelWithInvalid.Manipulator, cls)._prepare(model)
- cls.manager = model.valid_objects
+ class Manipulator(object):
+ """
+ Force default manipulators to look only at valid objects -
+ otherwise they will match against invalid objects when checking
+ uniqueness.
+ """
+ @classmethod
+ def _prepare(cls, model):
+ super(ModelWithInvalid.Manipulator, cls)._prepare(model)
+ cls.manager = model.valid_objects
diff --git a/frontend/afe/models.py b/frontend/afe/models.py
index 8e7882d..74d19e2 100644
--- a/frontend/afe/models.py
+++ b/frontend/afe/models.py
@@ -4,292 +4,292 @@
class AclAccessViolation(Exception):
- """\
- Raised when an operation is attempted with proper permissions as
- dictated by ACLs.
- """
+ """\
+ Raised when an operation is attempted with proper permissions as
+ dictated by ACLs.
+ """
class Label(model_logic.ModelWithInvalid, dbmodels.Model):
- """\
- Required:
- name: label name
+ """\
+ Required:
+ name: label name
- Optional:
- kernel_config: url/path to kernel config to use for jobs run on this
- label
- platform: if True, this is a platform label (defaults to False)
- """
- name = dbmodels.CharField(maxlength=255, unique=True)
- kernel_config = dbmodels.CharField(maxlength=255, blank=True)
- platform = dbmodels.BooleanField(default=False)
- invalid = dbmodels.BooleanField(default=False,
- editable=settings.FULL_ADMIN)
+ Optional:
+ kernel_config: url/path to kernel config to use for jobs run on this
+ label
+ platform: if True, this is a platform label (defaults to False)
+ """
+ name = dbmodels.CharField(maxlength=255, unique=True)
+ kernel_config = dbmodels.CharField(maxlength=255, blank=True)
+ platform = dbmodels.BooleanField(default=False)
+ invalid = dbmodels.BooleanField(default=False,
+ editable=settings.FULL_ADMIN)
- name_field = 'name'
- objects = model_logic.ExtendedManager()
- valid_objects = model_logic.ValidObjectsManager()
+ name_field = 'name'
+ objects = model_logic.ExtendedManager()
+ valid_objects = model_logic.ValidObjectsManager()
- def clean_object(self):
- self.host_set.clear()
+ def clean_object(self):
+ self.host_set.clear()
- def enqueue_job(self, job):
- 'Enqueue a job on any host of this label.'
- queue_entry = HostQueueEntry(meta_host=self, job=job,
- status=Job.Status.QUEUED,
- priority=job.priority)
- queue_entry.save()
+ def enqueue_job(self, job):
+ 'Enqueue a job on any host of this label.'
+ queue_entry = HostQueueEntry(meta_host=self, job=job,
+ status=Job.Status.QUEUED,
+ priority=job.priority)
+ queue_entry.save()
- class Meta:
- db_table = 'labels'
+ class Meta:
+ db_table = 'labels'
- class Admin:
- list_display = ('name', 'kernel_config')
- # see Host.Admin
- manager = model_logic.ValidObjectsManager()
+ class Admin:
+ list_display = ('name', 'kernel_config')
+ # see Host.Admin
+ manager = model_logic.ValidObjectsManager()
- def __str__(self):
- return self.name
+ def __str__(self):
+ return self.name
class Host(model_logic.ModelWithInvalid, dbmodels.Model):
- """\
- Required:
- hostname
+ """\
+ Required:
+ hostname
- optional:
- locked: host is locked and will not be queued
+ optional:
+ locked: host is locked and will not be queued
- Internal:
- synch_id: currently unused
- status: string describing status of host
- """
- Status = enum.Enum('Verifying', 'Running', 'Ready', 'Repairing',
- 'Repair Failed', 'Dead', 'Rebooting',
- string_values=True)
+ Internal:
+ synch_id: currently unused
+ status: string describing status of host
+ """
+ Status = enum.Enum('Verifying', 'Running', 'Ready', 'Repairing',
+ 'Repair Failed', 'Dead', 'Rebooting',
+ string_values=True)
- hostname = dbmodels.CharField(maxlength=255, unique=True)
- labels = dbmodels.ManyToManyField(Label, blank=True,
- filter_interface=dbmodels.HORIZONTAL)
- locked = dbmodels.BooleanField(default=False)
- synch_id = dbmodels.IntegerField(blank=True, null=True,
- editable=settings.FULL_ADMIN)
- status = dbmodels.CharField(maxlength=255, default=Status.READY,
- choices=Status.choices(),
- editable=settings.FULL_ADMIN)
- invalid = dbmodels.BooleanField(default=False,
- editable=settings.FULL_ADMIN)
+ hostname = dbmodels.CharField(maxlength=255, unique=True)
+ labels = dbmodels.ManyToManyField(Label, blank=True,
+ filter_interface=dbmodels.HORIZONTAL)
+ locked = dbmodels.BooleanField(default=False)
+ synch_id = dbmodels.IntegerField(blank=True, null=True,
+ editable=settings.FULL_ADMIN)
+ status = dbmodels.CharField(maxlength=255, default=Status.READY,
+ choices=Status.choices(),
+ editable=settings.FULL_ADMIN)
+ invalid = dbmodels.BooleanField(default=False,
+ editable=settings.FULL_ADMIN)
- name_field = 'hostname'
- objects = model_logic.ExtendedManager()
- valid_objects = model_logic.ValidObjectsManager()
+ name_field = 'hostname'
+ objects = model_logic.ExtendedManager()
+ valid_objects = model_logic.ValidObjectsManager()
- def clean_object(self):
- self.aclgroup_set.clear()
- self.labels.clear()
+ def clean_object(self):
+ self.aclgroup_set.clear()
+ self.labels.clear()
- def save(self):
- # extra spaces in the hostname can be a sneaky source of errors
- self.hostname = self.hostname.strip()
- # is this a new object being saved for the first time?
- first_time = (self.id is None)
- super(Host, self).save()
- if first_time:
- everyone = AclGroup.objects.get(name='Everyone')
- everyone.hosts.add(self)
+ def save(self):
+ # extra spaces in the hostname can be a sneaky source of errors
+ self.hostname = self.hostname.strip()
+ # is this a new object being saved for the first time?
+ first_time = (self.id is None)
+ super(Host, self).save()
+ if first_time:
+ everyone = AclGroup.objects.get(name='Everyone')
+ everyone.hosts.add(self)
- def enqueue_job(self, job):
- ' Enqueue a job on this host.'
- queue_entry = HostQueueEntry(host=self, job=job,
- status=Job.Status.QUEUED,
- priority=job.priority)
- # allow recovery of dead hosts from the frontend
- if not self.active_queue_entry() and self.is_dead():
- self.status = Host.Status.READY
- self.save()
- queue_entry.save()
+ def enqueue_job(self, job):
+ ' Enqueue a job on this host.'
+ queue_entry = HostQueueEntry(host=self, job=job,
+ status=Job.Status.QUEUED,
+ priority=job.priority)
+ # allow recovery of dead hosts from the frontend
+ if not self.active_queue_entry() and self.is_dead():
+ self.status = Host.Status.READY
+ self.save()
+ queue_entry.save()
- def platform(self):
- # TODO(showard): slighly hacky?
- platforms = self.labels.filter(platform=True)
- if len(platforms) == 0:
- return None
- return platforms[0]
- platform.short_description = 'Platform'
+ def platform(self):
+ # TODO(showard): slighly hacky?
+ platforms = self.labels.filter(platform=True)
+ if len(platforms) == 0:
+ return None
+ return platforms[0]
+ platform.short_description = 'Platform'
- def is_dead(self):
- return self.status == Host.Status.REPAIR_FAILED
+ def is_dead(self):
+ return self.status == Host.Status.REPAIR_FAILED
- def active_queue_entry(self):
- active = list(self.hostqueueentry_set.filter(active=True))
- if not active:
- return None
- assert len(active) == 1, ('More than one active entry for '
- 'host ' + self.hostname)
- return active[0]
+ def active_queue_entry(self):
+ active = list(self.hostqueueentry_set.filter(active=True))
+ if not active:
+ return None
+ assert len(active) == 1, ('More than one active entry for '
+ 'host ' + self.hostname)
+ return active[0]
- class Meta:
- db_table = 'hosts'
+ class Meta:
+ db_table = 'hosts'
- class Admin:
- # TODO(showard) - showing platform requires a SQL query for
- # each row (since labels are many-to-many) - should we remove
- # it?
- list_display = ('hostname', 'platform', 'locked', 'status')
- list_filter = ('labels', 'locked')
- search_fields = ('hostname', 'status')
- # undocumented Django feature - if you set manager here, the
- # admin code will use it, otherwise it'll use a default Manager
- manager = model_logic.ValidObjectsManager()
+ class Admin:
+ # TODO(showard) - showing platform requires a SQL query for
+ # each row (since labels are many-to-many) - should we remove
+ # it?
+ list_display = ('hostname', 'platform', 'locked', 'status')
+ list_filter = ('labels', 'locked')
+ search_fields = ('hostname', 'status')
+ # undocumented Django feature - if you set manager here, the
+ # admin code will use it, otherwise it'll use a default Manager
+ manager = model_logic.ValidObjectsManager()
- def __str__(self):
- return self.hostname
+ def __str__(self):
+ return self.hostname
class Test(dbmodels.Model, model_logic.ModelExtensions):
- """\
- Required:
- name: test name
- test_type: Client or Server
- path: path to pass to run_test()
- synch_type: whether the test should run synchronously or asynchronously
+ """\
+ Required:
+ name: test name
+ test_type: Client or Server
+ path: path to pass to run_test()
+ synch_type: whether the test should run synchronously or asynchronously
- Optional:
- test_class: used for categorization of tests
- description: arbirary text description
- """
- Classes = enum.Enum('Kernel', 'Hardware', 'Canned Test Sets',
- string_values=True)
- SynchType = enum.Enum('Asynchronous', 'Synchronous', start_value=1)
- # TODO(showard) - this should be merged with Job.ControlType (but right
- # now they use opposite values)
- Types = enum.Enum('Client', 'Server', start_value=1)
+ Optional:
+ test_class: used for categorization of tests
+ description: arbirary text description
+ """
+ Classes = enum.Enum('Kernel', 'Hardware', 'Canned Test Sets',
+ string_values=True)
+ SynchType = enum.Enum('Asynchronous', 'Synchronous', start_value=1)
+ # TODO(showard) - this should be merged with Job.ControlType (but right
+ # now they use opposite values)
+ Types = enum.Enum('Client', 'Server', start_value=1)
- name = dbmodels.CharField(maxlength=255, unique=True)
- test_class = dbmodels.CharField(maxlength=255,
- choices=Classes.choices())
- description = dbmodels.TextField(blank=True)
- test_type = dbmodels.SmallIntegerField(choices=Types.choices())
- synch_type = dbmodels.SmallIntegerField(choices=SynchType.choices(),
- default=SynchType.ASYNCHRONOUS)
- path = dbmodels.CharField(maxlength=255)
+ name = dbmodels.CharField(maxlength=255, unique=True)
+ test_class = dbmodels.CharField(maxlength=255,
+ choices=Classes.choices())
+ description = dbmodels.TextField(blank=True)
+ test_type = dbmodels.SmallIntegerField(choices=Types.choices())
+ synch_type = dbmodels.SmallIntegerField(choices=SynchType.choices(),
+ default=SynchType.ASYNCHRONOUS)
+ path = dbmodels.CharField(maxlength=255)
- name_field = 'name'
- objects = model_logic.ExtendedManager()
+ name_field = 'name'
+ objects = model_logic.ExtendedManager()
- class Meta:
- db_table = 'autotests'
+ class Meta:
+ db_table = 'autotests'
- class Admin:
- fields = (
- (None, {'fields' :
- ('name', 'test_class', 'test_type', 'synch_type',
- 'path', 'description')}),
- )
- list_display = ('name', 'test_type', 'synch_type',
- 'description')
- search_fields = ('name',)
+ class Admin:
+ fields = (
+ (None, {'fields' :
+ ('name', 'test_class', 'test_type', 'synch_type',
+ 'path', 'description')}),
+ )
+ list_display = ('name', 'test_type', 'synch_type',
+ 'description')
+ search_fields = ('name',)
- def __str__(self):
- return self.name
+ def __str__(self):
+ return self.name
class User(dbmodels.Model, model_logic.ModelExtensions):
- """\
- Required:
- login :user login name
+ """\
+ Required:
+ login :user login name
- Optional:
- access_level: 0=User (default), 1=Admin, 100=Root
- """
- ACCESS_ROOT = 100
- ACCESS_ADMIN = 1
- ACCESS_USER = 0
+ Optional:
+ access_level: 0=User (default), 1=Admin, 100=Root
+ """
+ ACCESS_ROOT = 100
+ ACCESS_ADMIN = 1
+ ACCESS_USER = 0
- login = dbmodels.CharField(maxlength=255, unique=True)
- access_level = dbmodels.IntegerField(default=ACCESS_USER, blank=True)
+ login = dbmodels.CharField(maxlength=255, unique=True)
+ access_level = dbmodels.IntegerField(default=ACCESS_USER, blank=True)
- name_field = 'login'
- objects = model_logic.ExtendedManager()
+ name_field = 'login'
+ objects = model_logic.ExtendedManager()
- def save(self):
- # is this a new object being saved for the first time?
- first_time = (self.id is None)
- super(User, self).save()
- if first_time:
- everyone = AclGroup.objects.get(name='Everyone')
- everyone.users.add(self)
+ def save(self):
+ # is this a new object being saved for the first time?
+ first_time = (self.id is None)
+ super(User, self).save()
+ if first_time:
+ everyone = AclGroup.objects.get(name='Everyone')
+ everyone.users.add(self)
- def has_access(self, target):
- if self.access_level >= self.ACCESS_ROOT:
- return True
+ def has_access(self, target):
+ if self.access_level >= self.ACCESS_ROOT:
+ return True
- if isinstance(target, int):
- return self.access_level >= target
- if isinstance(target, Job):
- return (target.owner == self.login or
- self.access_level >= self.ACCESS_ADMIN)
- if isinstance(target, Host):
- acl_intersect = [group
- for group in self.aclgroup_set.all()
- if group in target.aclgroup_set.all()]
- return bool(acl_intersect)
- if isinstance(target, User):
- return self.access_level >= target.access_level
- raise ValueError('Invalid target type')
+ if isinstance(target, int):
+ return self.access_level >= target
+ if isinstance(target, Job):
+ return (target.owner == self.login or
+ self.access_level >= self.ACCESS_ADMIN)
+ if isinstance(target, Host):
+ acl_intersect = [group
+ for group in self.aclgroup_set.all()
+ if group in target.aclgroup_set.all()]
+ return bool(acl_intersect)
+ if isinstance(target, User):
+ return self.access_level >= target.access_level
+ raise ValueError('Invalid target type')
- class Meta:
- db_table = 'users'
+ class Meta:
+ db_table = 'users'
- class Admin:
- list_display = ('login', 'access_level')
- search_fields = ('login',)
+ class Admin:
+ list_display = ('login', 'access_level')
+ search_fields = ('login',)
- def __str__(self):
- return self.login
+ def __str__(self):
+ return self.login
class AclGroup(dbmodels.Model, model_logic.ModelExtensions):
- """\
- Required:
- name: name of ACL group
+ """\
+ Required:
+ name: name of ACL group
- Optional:
- description: arbitrary description of group
- """
- name = dbmodels.CharField(maxlength=255, unique=True)
- description = dbmodels.CharField(maxlength=255, blank=True)
- users = dbmodels.ManyToManyField(User,
- filter_interface=dbmodels.HORIZONTAL)
- hosts = dbmodels.ManyToManyField(Host,
- filter_interface=dbmodels.HORIZONTAL)
+ Optional:
+ description: arbitrary description of group
+ """
+ name = dbmodels.CharField(maxlength=255, unique=True)
+ description = dbmodels.CharField(maxlength=255, blank=True)
+ users = dbmodels.ManyToManyField(User,
+ filter_interface=dbmodels.HORIZONTAL)
+ hosts = dbmodels.ManyToManyField(Host,
+ filter_interface=dbmodels.HORIZONTAL)
- name_field = 'name'
- objects = model_logic.ExtendedManager()
+ name_field = 'name'
+ objects = model_logic.ExtendedManager()
- class Meta:
- db_table = 'acl_groups'
+ class Meta:
+ db_table = 'acl_groups'
- class Admin:
- list_display = ('name', 'description')
- search_fields = ('name',)
+ class Admin:
+ list_display = ('name', 'description')
+ search_fields = ('name',)
- def __str__(self):
- return self.name
+ def __str__(self):
+ return self.name
# hack to make the column name in the many-to-many DB tables match the one
# generated by ruby
@@ -297,209 +297,209 @@
class JobManager(model_logic.ExtendedManager):
- 'Custom manager to provide efficient status counts querying.'
- def get_status_counts(self, job_ids):
- """\
- Returns a dictionary mapping the given job IDs to their status
- count dictionaries.
- """
- if not job_ids:
- return {}
- id_list = '(%s)' % ','.join(str(job_id) for job_id in job_ids)
- cursor = connection.cursor()
- cursor.execute("""
- SELECT job_id, status, COUNT(*)
- FROM host_queue_entries
- WHERE job_id IN %s
- GROUP BY job_id, status
- """ % id_list)
- all_job_counts = {}
- for job_id in job_ids:
- all_job_counts[job_id] = {}
- for job_id, status, count in cursor.fetchall():
- all_job_counts[job_id][status] = count
- return all_job_counts
+ 'Custom manager to provide efficient status counts querying.'
+ def get_status_counts(self, job_ids):
+ """\
+ Returns a dictionary mapping the given job IDs to their status
+ count dictionaries.
+ """
+ if not job_ids:
+ return {}
+ id_list = '(%s)' % ','.join(str(job_id) for job_id in job_ids)
+ cursor = connection.cursor()
+ cursor.execute("""
+ SELECT job_id, status, COUNT(*)
+ FROM host_queue_entries
+ WHERE job_id IN %s
+ GROUP BY job_id, status
+ """ % id_list)
+ all_job_counts = {}
+ for job_id in job_ids:
+ all_job_counts[job_id] = {}
+ for job_id, status, count in cursor.fetchall():
+ all_job_counts[job_id][status] = count
+ return all_job_counts
class Job(dbmodels.Model, model_logic.ModelExtensions):
- """\
- owner: username of job owner
- name: job name (does not have to be unique)
- priority: Low, Medium, High, Urgent (or 0-3)
- control_file: contents of control file
- control_type: Client or Server
- created_on: date of job creation
- submitted_on: date of job submission
- synch_type: Asynchronous or Synchronous (i.e. job must run on all hosts
- simultaneously; used for server-side control files)
- synch_count: ???
- synchronizing: for scheduler use
- """
- Priority = enum.Enum('Low', 'Medium', 'High', 'Urgent')
- ControlType = enum.Enum('Server', 'Client', start_value=1)
- Status = enum.Enum('Created', 'Queued', 'Pending', 'Running',
- 'Completed', 'Abort', 'Aborting', 'Aborted',
- 'Failed', string_values=True)
+ """\
+ owner: username of job owner
+ name: job name (does not have to be unique)
+ priority: Low, Medium, High, Urgent (or 0-3)
+ control_file: contents of control file
+ control_type: Client or Server
+ created_on: date of job creation
+ submitted_on: date of job submission
+ synch_type: Asynchronous or Synchronous (i.e. job must run on all hosts
+ simultaneously; used for server-side control files)
+ synch_count: ???
+ synchronizing: for scheduler use
+ """
+ Priority = enum.Enum('Low', 'Medium', 'High', 'Urgent')
+ ControlType = enum.Enum('Server', 'Client', start_value=1)
+ Status = enum.Enum('Created', 'Queued', 'Pending', 'Running',
+ 'Completed', 'Abort', 'Aborting', 'Aborted',
+ 'Failed', string_values=True)
- owner = dbmodels.CharField(maxlength=255)
- name = dbmodels.CharField(maxlength=255)
- priority = dbmodels.SmallIntegerField(choices=Priority.choices(),
- blank=True, # to allow 0
- default=Priority.MEDIUM)
- control_file = dbmodels.TextField()
- control_type = dbmodels.SmallIntegerField(choices=ControlType.choices(),
- blank=True) # to allow 0
- created_on = dbmodels.DateTimeField(auto_now_add=True)
- synch_type = dbmodels.SmallIntegerField(
- blank=True, null=True, choices=Test.SynchType.choices())
- synch_count = dbmodels.IntegerField(blank=True, null=True)
- synchronizing = dbmodels.BooleanField(default=False)
+ owner = dbmodels.CharField(maxlength=255)
+ name = dbmodels.CharField(maxlength=255)
+ priority = dbmodels.SmallIntegerField(choices=Priority.choices(),
+ blank=True, # to allow 0
+ default=Priority.MEDIUM)
+ control_file = dbmodels.TextField()
+ control_type = dbmodels.SmallIntegerField(choices=ControlType.choices(),
+ blank=True) # to allow 0
+ created_on = dbmodels.DateTimeField(auto_now_add=True)
+ synch_type = dbmodels.SmallIntegerField(
+ blank=True, null=True, choices=Test.SynchType.choices())
+ synch_count = dbmodels.IntegerField(blank=True, null=True)
+ synchronizing = dbmodels.BooleanField(default=False)
- # custom manager
- objects = JobManager()
+ # custom manager
+ objects = JobManager()
- def is_server_job(self):
- return self.control_type == self.ControlType.SERVER
+ def is_server_job(self):
+ return self.control_type == self.ControlType.SERVER
- @classmethod
- def create(cls, owner, name, priority, control_file, control_type,
- hosts, synch_type):
- """\
- Creates a job by taking some information (the listed args)
- and filling in the rest of the necessary information.
- """
- job = cls.add_object(
- owner=owner, name=name, priority=priority,
- control_file=control_file, control_type=control_type,
- synch_type=synch_type)
+ @classmethod
+ def create(cls, owner, name, priority, control_file, control_type,
+ hosts, synch_type):
+ """\
+ Creates a job by taking some information (the listed args)
+ and filling in the rest of the necessary information.
+ """
+ job = cls.add_object(
+ owner=owner, name=name, priority=priority,
+ control_file=control_file, control_type=control_type,
+ synch_type=synch_type)
- if job.synch_type == Test.SynchType.SYNCHRONOUS:
- job.synch_count = len(hosts)
- else:
- if len(hosts) == 0:
- errors = {'hosts':
- 'asynchronous jobs require at least'
- + ' one host to run on'}
- raise model_logic.ValidationError(errors)
- job.save()
- return job
+ if job.synch_type == Test.SynchType.SYNCHRONOUS:
+ job.synch_count = len(hosts)
+ else:
+ if len(hosts) == 0:
+ errors = {'hosts':
+ 'asynchronous jobs require at least'
+ + ' one host to run on'}
+ raise model_logic.ValidationError(errors)
+ job.save()
+ return job
- def queue(self, hosts):
- 'Enqueue a job on the given hosts.'
- for host in hosts:
- host.enqueue_job(self)
- self.recompute_blocks()
+ def queue(self, hosts):
+ 'Enqueue a job on the given hosts.'
+ for host in hosts:
+ host.enqueue_job(self)
+ self.recompute_blocks()
- def recompute_blocks(self):
- """\
- Clear out the blocks (ineligible_host_queues) for this job and
- recompute the set. The set of blocks is the union of:
- -all hosts already assigned to this job
- -all hosts not ACL accessible to this job's owner
- """
- job_entries = self.hostqueueentry_set.all()
- query = Host.objects.filter_in_subquery(
- 'host_id', job_entries, subquery_alias='job_entries')
+ def recompute_blocks(self):
+ """\
+ Clear out the blocks (ineligible_host_queues) for this job and
+ recompute the set. The set of blocks is the union of:
+ -all hosts already assigned to this job
+ -all hosts not ACL accessible to this job's owner
+ """
+ job_entries = self.hostqueueentry_set.all()
+ query = Host.objects.filter_in_subquery(
+ 'host_id', job_entries, subquery_alias='job_entries')
- old_ids = [block.id for block in
- self.ineligiblehostqueue_set.all()]
- block_values = [(self.id, host.id) for host in query]
- IneligibleHostQueue.objects.create_in_bulk(('job', 'host'),
- block_values)
- IneligibleHostQueue.objects.delete_in_bulk(old_ids)
+ old_ids = [block.id for block in
+ self.ineligiblehostqueue_set.all()]
+ block_values = [(self.id, host.id) for host in query]
+ IneligibleHostQueue.objects.create_in_bulk(('job', 'host'),
+ block_values)
+ IneligibleHostQueue.objects.delete_in_bulk(old_ids)
- @classmethod
- def recompute_all_blocks(cls):
- 'Recompute blocks for all queued and active jobs.'
- for job in cls.objects.filter(
- hostqueueentry__complete=False).distinct():
- job.recompute_blocks()
+ @classmethod
+ def recompute_all_blocks(cls):
+ 'Recompute blocks for all queued and active jobs.'
+ for job in cls.objects.filter(
+ hostqueueentry__complete=False).distinct():
+ job.recompute_blocks()
- def requeue(self, new_owner):
- 'Creates a new job identical to this one'
- hosts = [queue_entry.meta_host or queue_entry.host
- for queue_entry in self.hostqueueentry_set.all()]
- new_job = Job.create(
- owner=new_owner, name=self.name, priority=self.priority,
- control_file=self.control_file,
- control_type=self.control_type, hosts=hosts,
- synch_type=self.synch_type)
- new_job.queue(hosts)
- return new_job
+ def requeue(self, new_owner):
+ 'Creates a new job identical to this one'
+ hosts = [queue_entry.meta_host or queue_entry.host
+ for queue_entry in self.hostqueueentry_set.all()]
+ new_job = Job.create(
+ owner=new_owner, name=self.name, priority=self.priority,
+ control_file=self.control_file,
+ control_type=self.control_type, hosts=hosts,
+ synch_type=self.synch_type)
+ new_job.queue(hosts)
+ return new_job
- def abort(self):
- for queue_entry in self.hostqueueentry_set.all():
- if queue_entry.active:
- queue_entry.status = Job.Status.ABORT
- elif not queue_entry.complete:
- queue_entry.status = Job.Status.ABORTED
- queue_entry.active = False
- queue_entry.complete = True
- queue_entry.save()
+ def abort(self):
+ for queue_entry in self.hostqueueentry_set.all():
+ if queue_entry.active:
+ queue_entry.status = Job.Status.ABORT
+ elif not queue_entry.complete:
+ queue_entry.status = Job.Status.ABORTED
+ queue_entry.active = False
+ queue_entry.complete = True
+ queue_entry.save()
- def user(self):
- try:
- return User.objects.get(login=self.owner)
- except self.DoesNotExist:
- return None
+ def user(self):
+ try:
+ return User.objects.get(login=self.owner)
+ except self.DoesNotExist:
+ return None
- class Meta:
- db_table = 'jobs'
+ class Meta:
+ db_table = 'jobs'
- if settings.FULL_ADMIN:
- class Admin:
- list_display = ('id', 'owner', 'name', 'control_type')
+ if settings.FULL_ADMIN:
+ class Admin:
+ list_display = ('id', 'owner', 'name', 'control_type')
- def __str__(self):
- return '%s (%s-%s)' % (self.name, self.id, self.owner)
+ def __str__(self):
+ return '%s (%s-%s)' % (self.name, self.id, self.owner)
class IneligibleHostQueue(dbmodels.Model, model_logic.ModelExtensions):
- job = dbmodels.ForeignKey(Job)
- host = dbmodels.ForeignKey(Host)
+ job = dbmodels.ForeignKey(Job)
+ host = dbmodels.ForeignKey(Host)
- objects = model_logic.ExtendedManager()
+ objects = model_logic.ExtendedManager()
- class Meta:
- db_table = 'ineligible_host_queues'
+ class Meta:
+ db_table = 'ineligible_host_queues'
- if settings.FULL_ADMIN:
- class Admin:
- list_display = ('id', 'job', 'host')
+ if settings.FULL_ADMIN:
+ class Admin:
+ list_display = ('id', 'job', 'host')
class HostQueueEntry(dbmodels.Model, model_logic.ModelExtensions):
- job = dbmodels.ForeignKey(Job)
- host = dbmodels.ForeignKey(Host, blank=True, null=True)
- priority = dbmodels.SmallIntegerField()
- status = dbmodels.CharField(maxlength=255)
- meta_host = dbmodels.ForeignKey(Label, blank=True, null=True,
- db_column='meta_host')
- active = dbmodels.BooleanField(default=False)
- complete = dbmodels.BooleanField(default=False)
+ job = dbmodels.ForeignKey(Job)
+ host = dbmodels.ForeignKey(Host, blank=True, null=True)
+ priority = dbmodels.SmallIntegerField()
+ status = dbmodels.CharField(maxlength=255)
+ meta_host = dbmodels.ForeignKey(Label, blank=True, null=True,
+ db_column='meta_host')
+ active = dbmodels.BooleanField(default=False)
+ complete = dbmodels.BooleanField(default=False)
- objects = model_logic.ExtendedManager()
+ objects = model_logic.ExtendedManager()
- def is_meta_host_entry(self):
- 'True if this is a entry has a meta_host instead of a host.'
- return self.host is None and self.meta_host is not None
+ def is_meta_host_entry(self):
+ 'True if this is a entry has a meta_host instead of a host.'
+ return self.host is None and self.meta_host is not None
- class Meta:
- db_table = 'host_queue_entries'
+ class Meta:
+ db_table = 'host_queue_entries'
- if settings.FULL_ADMIN:
- class Admin:
- list_display = ('id', 'job', 'host', 'status',
- 'meta_host')
+ if settings.FULL_ADMIN:
+ class Admin:
+ list_display = ('id', 'job', 'host', 'status',
+ 'meta_host')
diff --git a/frontend/afe/rpc_client_lib.py b/frontend/afe/rpc_client_lib.py
index 0d362b2..1707e04 100644
--- a/frontend/afe/rpc_client_lib.py
+++ b/frontend/afe/rpc_client_lib.py
@@ -8,4 +8,4 @@
from json_rpc import proxy
def get_proxy(*args, **kwargs):
- return proxy.ServiceProxy(*args, **kwargs)
+ return proxy.ServiceProxy(*args, **kwargs)
diff --git a/frontend/afe/rpc_handler.py b/frontend/afe/rpc_handler.py
index f9ba2e2..70ef6cd 100644
--- a/frontend/afe/rpc_handler.py
+++ b/frontend/afe/rpc_handler.py
@@ -13,60 +13,60 @@
class RpcMethodHolder(object):
- 'Dummy class to hold RPC interface methods as attributes.'
+ 'Dummy class to hold RPC interface methods as attributes.'
class RpcHandler(object):
- def __init__(self, rpc_interface_modules, document_module=None):
- self._rpc_methods = RpcMethodHolder()
- self._dispatcher = serviceHandler.ServiceHandler(
- self._rpc_methods)
+ def __init__(self, rpc_interface_modules, document_module=None):
+ self._rpc_methods = RpcMethodHolder()
+ self._dispatcher = serviceHandler.ServiceHandler(
+ self._rpc_methods)
- # store all methods from interface modules
- for module in rpc_interface_modules:
- self._grab_methods_from(module)
+ # store all methods from interface modules
+ for module in rpc_interface_modules:
+ self._grab_methods_from(module)
- # get documentation for rpc_interface we can send back to the
- # user
- if document_module is None:
- document_module = rpc_interface_modules[0]
- self.html_doc = pydoc.html.document(document_module)
+ # get documentation for rpc_interface we can send back to the
+ # user
+ if document_module is None:
+ document_module = rpc_interface_modules[0]
+ self.html_doc = pydoc.html.document(document_module)
- def handle_rpc_request(self, request):
- response = django.http.HttpResponse()
- if len(request.POST):
- response.write(self._dispatcher.handleRequest(
- request.raw_post_data))
- else:
- response.write(self.html_doc)
+ def handle_rpc_request(self, request):
+ response = django.http.HttpResponse()
+ if len(request.POST):
+ response.write(self._dispatcher.handleRequest(
+ request.raw_post_data))
+ else:
+ response.write(self.html_doc)
- response['Content-length'] = str(len(response.content))
- return response
+ response['Content-length'] = str(len(response.content))
+ return response
- @staticmethod
- def _allow_keyword_args(f):
- """\
- Decorator to allow a function to take keyword args even though
- the RPC layer doesn't support that. The decorated function
- assumes its last argument is a dictionary of keyword args and
- passes them to the original function as keyword args.
- """
- def new_fn(*args):
- assert args
- keyword_args = args[-1]
- args = args[:-1]
- return f(*args, **keyword_args)
- new_fn.func_name = f.func_name
- return new_fn
+ @staticmethod
+ def _allow_keyword_args(f):
+ """\
+ Decorator to allow a function to take keyword args even though
+ the RPC layer doesn't support that. The decorated function
+ assumes its last argument is a dictionary of keyword args and
+ passes them to the original function as keyword args.
+ """
+ def new_fn(*args):
+ assert args
+ keyword_args = args[-1]
+ args = args[:-1]
+ return f(*args, **keyword_args)
+ new_fn.func_name = f.func_name
+ return new_fn
- def _grab_methods_from(self, module):
- for name in dir(module):
- attribute = getattr(module, name)
- if not callable(attribute):
- continue
- decorated_function = (
- RpcHandler._allow_keyword_args(attribute))
- setattr(self._rpc_methods, name, decorated_function)
+ def _grab_methods_from(self, module):
+ for name in dir(module):
+ attribute = getattr(module, name)
+ if not callable(attribute):
+ continue
+ decorated_function = (
+ RpcHandler._allow_keyword_args(attribute))
+ setattr(self._rpc_methods, name, decorated_function)
diff --git a/frontend/afe/rpc_interface.py b/frontend/afe/rpc_interface.py
index d82b53e..8015912 100644
--- a/frontend/afe/rpc_interface.py
+++ b/frontend/afe/rpc_interface.py
@@ -34,352 +34,352 @@
# labels
def add_label(name, kernel_config=None, platform=None):
- return models.Label.add_object(name=name, kernel_config=kernel_config,
- platform=platform).id
+ return models.Label.add_object(name=name, kernel_config=kernel_config,
+ platform=platform).id
def modify_label(id, **data):
- models.Label.smart_get(id).update_object(data)
+ models.Label.smart_get(id).update_object(data)
def delete_label(id):
- models.Label.smart_get(id).delete()
+ models.Label.smart_get(id).delete()
def label_add_hosts(id, hosts):
- host_objs = [models.Host.smart_get(host) for host in hosts]
- models.Label.smart_get(id).host_set.add(*host_objs)
+ host_objs = [models.Host.smart_get(host) for host in hosts]
+ models.Label.smart_get(id).host_set.add(*host_objs)
def label_remove_hosts(id, hosts):
- host_objs = [models.Host.smart_get(host) for host in hosts]
- models.Label.smart_get(id).host_set.remove(*host_objs)
+ host_objs = [models.Host.smart_get(host) for host in hosts]
+ models.Label.smart_get(id).host_set.remove(*host_objs)
def get_labels(**filter_data):
- return rpc_utils.prepare_for_serialization(
- models.Label.list_objects(filter_data))
+ return rpc_utils.prepare_for_serialization(
+ models.Label.list_objects(filter_data))
# hosts
def add_host(hostname, status=None, locked=None):
- return models.Host.add_object(hostname=hostname, status=status,
- locked=locked).id
+ return models.Host.add_object(hostname=hostname, status=status,
+ locked=locked).id
def modify_host(id, **data):
- models.Host.smart_get(id).update_object(data)
+ models.Host.smart_get(id).update_object(data)
def host_add_labels(id, labels):
- labels = [models.Label.smart_get(label) for label in labels]
- models.Host.smart_get(id).labels.add(*labels)
+ labels = [models.Label.smart_get(label) for label in labels]
+ models.Host.smart_get(id).labels.add(*labels)
def host_remove_labels(id, labels):
- labels = [models.Label.smart_get(label) for label in labels]
- models.Host.smart_get(id).labels.remove(*labels)
+ labels = [models.Label.smart_get(label) for label in labels]
+ models.Host.smart_get(id).labels.remove(*labels)
def delete_host(id):
- models.Host.smart_get(id).delete()
+ models.Host.smart_get(id).delete()
def get_hosts(multiple_labels=[], **filter_data):
- """\
- multiple_labels: match hosts in all of the labels given. Should be a
- list of label names.
- """
- filter_data['extra_args'] = (
- rpc_utils.extra_host_filters(multiple_labels))
- hosts = models.Host.list_objects(filter_data)
- for host in hosts:
- host_obj = models.Host.objects.get(id=host['id'])
- host['labels'] = [label.name
- for label in host_obj.labels.all()]
- platform = host_obj.platform()
- host['platform'] = platform and platform.name or None
- return rpc_utils.prepare_for_serialization(hosts)
+ """\
+ multiple_labels: match hosts in all of the labels given. Should be a
+ list of label names.
+ """
+ filter_data['extra_args'] = (
+ rpc_utils.extra_host_filters(multiple_labels))
+ hosts = models.Host.list_objects(filter_data)
+ for host in hosts:
+ host_obj = models.Host.objects.get(id=host['id'])
+ host['labels'] = [label.name
+ for label in host_obj.labels.all()]
+ platform = host_obj.platform()
+ host['platform'] = platform and platform.name or None
+ return rpc_utils.prepare_for_serialization(hosts)
def get_num_hosts(multiple_labels=[], **filter_data):
- filter_data['extra_args'] = (
- rpc_utils.extra_host_filters(multiple_labels))
- return models.Host.query_count(filter_data)
+ filter_data['extra_args'] = (
+ rpc_utils.extra_host_filters(multiple_labels))
+ return models.Host.query_count(filter_data)
# tests
def add_test(name, test_type, path, test_class=None, description=None):
- return models.Test.add_object(name=name, test_type=test_type, path=path,
- test_class=test_class,
- description=description).id
+ return models.Test.add_object(name=name, test_type=test_type, path=path,
+ test_class=test_class,
+ description=description).id
def modify_test(id, **data):
- models.Test.smart_get(id).update_object(data)
+ models.Test.smart_get(id).update_object(data)
def delete_test(id):
- models.Test.smart_get(id).delete()
+ models.Test.smart_get(id).delete()
def get_tests(**filter_data):
- return rpc_utils.prepare_for_serialization(
- models.Test.list_objects(filter_data))
+ return rpc_utils.prepare_for_serialization(
+ models.Test.list_objects(filter_data))
# users
def add_user(login, access_level=None):
- return models.User.add_object(login=login, access_level=access_level).id
+ return models.User.add_object(login=login, access_level=access_level).id
def modify_user(id, **data):
- models.User.smart_get(id).update_object(data)
+ models.User.smart_get(id).update_object(data)
def delete_user(id):
- models.User.smart_get(id).delete()
+ models.User.smart_get(id).delete()
def get_users(**filter_data):
- return rpc_utils.prepare_for_serialization(
- models.User.list_objects(filter_data))
+ return rpc_utils.prepare_for_serialization(
+ models.User.list_objects(filter_data))
# acl groups
def add_acl_group(name, description=None):
- return models.AclGroup.add_object(name=name, description=description).id
+ return models.AclGroup.add_object(name=name, description=description).id
def modify_acl_group(id, **data):
- models.AclGroup.smart_get(id).update_object(data)
+ models.AclGroup.smart_get(id).update_object(data)
def acl_group_add_users(id, users):
- users = [models.User.smart_get(user) for user in users]
- group = models.AclGroup.smart_get(id)
- group.users.add(*users)
+ users = [models.User.smart_get(user) for user in users]
+ group = models.AclGroup.smart_get(id)
+ group.users.add(*users)
def acl_group_remove_users(id, users):
- users = [models.User.smart_get(user) for user in users]
- group = models.AclGroup.smart_get(id)
- group.users.remove(*users)
+ users = [models.User.smart_get(user) for user in users]
+ group = models.AclGroup.smart_get(id)
+ group.users.remove(*users)
def acl_group_add_hosts(id, hosts):
- hosts = [models.Host.smart_get(host) for host in hosts]
- group = models.AclGroup.smart_get(id)
- group.hosts.add(*hosts)
+ hosts = [models.Host.smart_get(host) for host in hosts]
+ group = models.AclGroup.smart_get(id)
+ group.hosts.add(*hosts)
def acl_group_remove_hosts(id, hosts):
- hosts = [models.Host.smart_get(host) for host in hosts]
- group = models.AclGroup.smart_get(id)
- group.hosts.remove(*hosts)
+ hosts = [models.Host.smart_get(host) for host in hosts]
+ group = models.AclGroup.smart_get(id)
+ group.hosts.remove(*hosts)
def delete_acl_group(id):
- models.AclGroup.smart_get(id).delete()
+ models.AclGroup.smart_get(id).delete()
def get_acl_groups(**filter_data):
- acl_groups = models.AclGroup.list_objects(filter_data)
- for acl_group in acl_groups:
- acl_group_obj = models.AclGroup.objects.get(id=acl_group['id'])
- acl_group['users'] = [user.login
- for user in acl_group_obj.users.all()]
- acl_group['hosts'] = [host.hostname
- for host in acl_group_obj.hosts.all()]
- return rpc_utils.prepare_for_serialization(acl_groups)
+ acl_groups = models.AclGroup.list_objects(filter_data)
+ for acl_group in acl_groups:
+ acl_group_obj = models.AclGroup.objects.get(id=acl_group['id'])
+ acl_group['users'] = [user.login
+ for user in acl_group_obj.users.all()]
+ acl_group['hosts'] = [host.hostname
+ for host in acl_group_obj.hosts.all()]
+ return rpc_utils.prepare_for_serialization(acl_groups)
# jobs
def generate_control_file(tests, kernel=None, label=None):
- """\
- Generates a client-side control file to load a kernel and run a set of
- tests. Returns a tuple (control_file, is_server, is_synchronous):
- control_file - the control file text
- is_server - is the control file a server-side control file?
- is_synchronous - should the control file be run synchronously?
+ """\
+ Generates a client-side control file to load a kernel and run a set of
+ tests. Returns a tuple (control_file, is_server, is_synchronous):
+ control_file - the control file text
+ is_server - is the control file a server-side control file?
+ is_synchronous - should the control file be run synchronously?
- tests: list of tests to run
- kernel: kernel to install in generated control file
- label: name of label to grab kernel config from
- """
- if not tests:
- return '', False, False
+ tests: list of tests to run
+ kernel: kernel to install in generated control file
+ label: name of label to grab kernel config from
+ """
+ if not tests:
+ return '', False, False
- is_server, is_synchronous, test_objects, label = (
- rpc_utils.prepare_generate_control_file(tests, kernel, label))
- cf_text = control_file.generate_control(test_objects, kernel, label,
- is_server)
- return cf_text, is_server, is_synchronous
+ is_server, is_synchronous, test_objects, label = (
+ rpc_utils.prepare_generate_control_file(tests, kernel, label))
+ cf_text = control_file.generate_control(test_objects, kernel, label,
+ is_server)
+ return cf_text, is_server, is_synchronous
def create_job(name, priority, control_file, control_type, is_synchronous=None,
- hosts=None, meta_hosts=None):
- """\
- Create and enqueue a job.
+ hosts=None, meta_hosts=None):
+ """\
+ Create and enqueue a job.
- priority: Low, Medium, High, Urgent
- control_file: contents of control file
- control_type: type of control file, Client or Server
- is_synchronous: boolean indicating if a job is synchronous
- hosts: list of hosts to run job on
- meta_hosts: list where each entry is a label name, and for each entry
- one host will be chosen from that label to run the job
- on.
- """
- owner = rpc_utils.get_user().login
- # input validation
- if not hosts and not meta_hosts:
- raise models.ValidationError({
- 'arguments' : "You must pass at least one of 'hosts' or "
- "'meta_hosts'"
- })
+ priority: Low, Medium, High, Urgent
+ control_file: contents of control file
+ control_type: type of control file, Client or Server
+ is_synchronous: boolean indicating if a job is synchronous
+ hosts: list of hosts to run job on
+ meta_hosts: list where each entry is a label name, and for each entry
+ one host will be chosen from that label to run the job
+ on.
+ """
+ owner = rpc_utils.get_user().login
+ # input validation
+ if not hosts and not meta_hosts:
+ raise models.ValidationError({
+ 'arguments' : "You must pass at least one of 'hosts' or "
+ "'meta_hosts'"
+ })
- # convert hostnames & meta hosts to host/label objects
- host_objects = []
- for host in hosts or []:
- this_host = models.Host.smart_get(host)
- host_objects.append(this_host)
- for label in meta_hosts or []:
- this_label = models.Label.smart_get(label)
- host_objects.append(this_label)
+ # convert hostnames & meta hosts to host/label objects
+ host_objects = []
+ for host in hosts or []:
+ this_host = models.Host.smart_get(host)
+ host_objects.append(this_host)
+ for label in meta_hosts or []:
+ this_label = models.Label.smart_get(label)
+ host_objects.append(this_label)
- # default is_synchronous to some appropriate value
- ControlType = models.Job.ControlType
- control_type = ControlType.get_value(control_type)
- if is_synchronous is None:
- is_synchronous = (control_type == ControlType.SERVER)
- # convert the synch flag to an actual type
- if is_synchronous:
- synch_type = models.Test.SynchType.SYNCHRONOUS
- else:
- synch_type = models.Test.SynchType.ASYNCHRONOUS
+ # default is_synchronous to some appropriate value
+ ControlType = models.Job.ControlType
+ control_type = ControlType.get_value(control_type)
+ if is_synchronous is None:
+ is_synchronous = (control_type == ControlType.SERVER)
+ # convert the synch flag to an actual type
+ if is_synchronous:
+ synch_type = models.Test.SynchType.SYNCHRONOUS
+ else:
+ synch_type = models.Test.SynchType.ASYNCHRONOUS
- job = models.Job.create(owner=owner, name=name, priority=priority,
- control_file=control_file,
- control_type=control_type,
- synch_type=synch_type,
- hosts=host_objects)
- job.queue(host_objects)
- return job.id
+ job = models.Job.create(owner=owner, name=name, priority=priority,
+ control_file=control_file,
+ control_type=control_type,
+ synch_type=synch_type,
+ hosts=host_objects)
+ job.queue(host_objects)
+ return job.id
def requeue_job(id):
- """\
- Create and enqueue a copy of the given job.
- """
- job = models.Job.objects.get(id=id)
- new_job = job.requeue(rpc_utils.get_user().login)
- return new_job.id
+ """\
+ Create and enqueue a copy of the given job.
+ """
+ job = models.Job.objects.get(id=id)
+ new_job = job.requeue(rpc_utils.get_user().login)
+ return new_job.id
def abort_job(id):
- """\
- Abort the job with the given id number.
- """
- job = models.Job.objects.get(id=id)
- job.abort()
+ """\
+ Abort the job with the given id number.
+ """
+ job = models.Job.objects.get(id=id)
+ job.abort()
def get_jobs(not_yet_run=False, running=False, finished=False, **filter_data):
- """\
- Extra filter args for get_jobs:
- -not_yet_run: Include only jobs that have not yet started running.
- -running: Include only jobs that have start running but for which not
- all hosts have completed.
- -finished: Include only jobs for which all hosts have completed (or
- aborted).
- At most one of these three fields should be specified.
- """
- filter_data['extra_args'] = rpc_utils.extra_job_filters(not_yet_run,
- running,
- finished)
- return rpc_utils.prepare_for_serialization(
- models.Job.list_objects(filter_data))
+ """\
+ Extra filter args for get_jobs:
+ -not_yet_run: Include only jobs that have not yet started running.
+ -running: Include only jobs that have start running but for which not
+ all hosts have completed.
+ -finished: Include only jobs for which all hosts have completed (or
+ aborted).
+ At most one of these three fields should be specified.
+ """
+ filter_data['extra_args'] = rpc_utils.extra_job_filters(not_yet_run,
+ running,
+ finished)
+ return rpc_utils.prepare_for_serialization(
+ models.Job.list_objects(filter_data))
def get_num_jobs(not_yet_run=False, running=False, finished=False,
- **filter_data):
- """\
- See get_jobs() for documentation of extra filter parameters.
- """
- filter_data['extra_args'] = rpc_utils.extra_job_filters(not_yet_run,
- running,
- finished)
- return models.Job.query_count(filter_data)
+ **filter_data):
+ """\
+ See get_jobs() for documentation of extra filter parameters.
+ """
+ filter_data['extra_args'] = rpc_utils.extra_job_filters(not_yet_run,
+ running,
+ finished)
+ return models.Job.query_count(filter_data)
def get_jobs_summary(**filter_data):
- """\
- Like get_jobs(), but adds a 'stauts_counts' field, which is a dictionary
- mapping status strings to the number of hosts currently with that
- status, i.e. {'Queued' : 4, 'Running' : 2}.
- """
- jobs = get_jobs(**filter_data)
- ids = [job['id'] for job in jobs]
- all_status_counts = models.Job.objects.get_status_counts(ids)
- for job in jobs:
- job['status_counts'] = all_status_counts[job['id']]
- return rpc_utils.prepare_for_serialization(jobs)
+ """\
+ Like get_jobs(), but adds a 'stauts_counts' field, which is a dictionary
+ mapping status strings to the number of hosts currently with that
+ status, i.e. {'Queued' : 4, 'Running' : 2}.
+ """
+ jobs = get_jobs(**filter_data)
+ ids = [job['id'] for job in jobs]
+ all_status_counts = models.Job.objects.get_status_counts(ids)
+ for job in jobs:
+ job['status_counts'] = all_status_counts[job['id']]
+ return rpc_utils.prepare_for_serialization(jobs)
# host queue entries
def get_host_queue_entries(**filter_data):
- """\
- TODO
- """
- query = models.HostQueueEntry.query_objects(filter_data)
- all_dicts = []
- for queue_entry in query.select_related():
- entry_dict = queue_entry.get_object_dict()
- if entry_dict['host'] is not None:
- entry_dict['host'] = queue_entry.host.get_object_dict()
- entry_dict['job'] = queue_entry.job.get_object_dict()
- all_dicts.append(entry_dict)
- return rpc_utils.prepare_for_serialization(all_dicts)
+ """\
+ TODO
+ """
+ query = models.HostQueueEntry.query_objects(filter_data)
+ all_dicts = []
+ for queue_entry in query.select_related():
+ entry_dict = queue_entry.get_object_dict()
+ if entry_dict['host'] is not None:
+ entry_dict['host'] = queue_entry.host.get_object_dict()
+ entry_dict['job'] = queue_entry.job.get_object_dict()
+ all_dicts.append(entry_dict)
+ return rpc_utils.prepare_for_serialization(all_dicts)
def get_num_host_queue_entries(**filter_data):
- """\
- Get the number of host queue entries associated with this job.
- """
- return models.HostQueueEntry.query_count(filter_data)
+ """\
+ Get the number of host queue entries associated with this job.
+ """
+ return models.HostQueueEntry.query_count(filter_data)
# other
def get_static_data():
- """\
- Returns a dictionary containing a bunch of data that shouldn't change
- often and is otherwise inaccessible. This includes:
- priorities: list of job priority choices
- default_priority: default priority value for new jobs
- users: sorted list of all users
- labels: sorted list of all labels
- tests: sorted list of all tests
- user_login: logged-in username
- host_statuses: sorted list of possible Host statuses
- job_statuses: sorted list of possible HostQueueEntry statuses
- """
- result = {}
- result['priorities'] = models.Job.Priority.choices()
- default_priority = models.Job.get_field_dict()['priority'].default
- default_string = models.Job.Priority.get_string(default_priority)
- result['default_priority'] = default_string
- result['users'] = get_users(sort_by=['login'])
- result['labels'] = get_labels(sort_by=['-platform', 'name'])
- result['tests'] = get_tests(sort_by=['name'])
- result['user_login'] = rpc_utils.get_user().login
- result['host_statuses'] = rpc_utils.sorted(models.Host.Status.names)
- result['job_statuses'] = rpc_utils.sorted(models.Job.Status.names)
- return result
+ """\
+ Returns a dictionary containing a bunch of data that shouldn't change
+ often and is otherwise inaccessible. This includes:
+ priorities: list of job priority choices
+ default_priority: default priority value for new jobs
+ users: sorted list of all users
+ labels: sorted list of all labels
+ tests: sorted list of all tests
+ user_login: logged-in username
+ host_statuses: sorted list of possible Host statuses
+ job_statuses: sorted list of possible HostQueueEntry statuses
+ """
+ result = {}
+ result['priorities'] = models.Job.Priority.choices()
+ default_priority = models.Job.get_field_dict()['priority'].default
+ default_string = models.Job.Priority.get_string(default_priority)
+ result['default_priority'] = default_string
+ result['users'] = get_users(sort_by=['login'])
+ result['labels'] = get_labels(sort_by=['-platform', 'name'])
+ result['tests'] = get_tests(sort_by=['name'])
+ result['user_login'] = rpc_utils.get_user().login
+ result['host_statuses'] = rpc_utils.sorted(models.Host.Status.names)
+ result['job_statuses'] = rpc_utils.sorted(models.Job.Status.names)
+ return result
diff --git a/frontend/afe/rpc_utils.py b/frontend/afe/rpc_utils.py
index 14fa987..58b524e 100644
--- a/frontend/afe/rpc_utils.py
+++ b/frontend/afe/rpc_utils.py
@@ -9,146 +9,146 @@
from frontend.afe import models
def prepare_for_serialization(objects):
- """
- Prepare Python objects to be returned via RPC.
- """
- if (isinstance(objects, list) and len(objects) and
- isinstance(objects[0], dict) and 'id' in objects[0]):
- objects = gather_unique_dicts(objects)
- return _prepare_data(objects)
+ """
+ Prepare Python objects to be returned via RPC.
+ """
+ if (isinstance(objects, list) and len(objects) and
+ isinstance(objects[0], dict) and 'id' in objects[0]):
+ objects = gather_unique_dicts(objects)
+ return _prepare_data(objects)
def _prepare_data(data):
- """
- Recursively process data structures, performing necessary type
- conversions to values in data to allow for RPC serialization:
- -convert datetimes to strings
- -convert tuples to lists
- """
- if isinstance(data, dict):
- new_data = {}
- for key, value in data.iteritems():
- new_data[key] = _prepare_data(value)
- return new_data
- elif isinstance(data, list) or isinstance(data, tuple):
- return [_prepare_data(item) for item in data]
- elif isinstance(data, datetime.datetime):
- return str(data)
- else:
- return data
+ """
+ Recursively process data structures, performing necessary type
+ conversions to values in data to allow for RPC serialization:
+ -convert datetimes to strings
+ -convert tuples to lists
+ """
+ if isinstance(data, dict):
+ new_data = {}
+ for key, value in data.iteritems():
+ new_data[key] = _prepare_data(value)
+ return new_data
+ elif isinstance(data, list) or isinstance(data, tuple):
+ return [_prepare_data(item) for item in data]
+ elif isinstance(data, datetime.datetime):
+ return str(data)
+ else:
+ return data
def gather_unique_dicts(dict_iterable):
- """\
- Pick out unique objects (by ID) from an iterable of object dicts.
- """
- id_set = set()
- result = []
- for obj in dict_iterable:
- if obj['id'] not in id_set:
- id_set.add(obj['id'])
- result.append(obj)
- return result
+ """\
+ Pick out unique objects (by ID) from an iterable of object dicts.
+ """
+ id_set = set()
+ result = []
+ for obj in dict_iterable:
+ if obj['id'] not in id_set:
+ id_set.add(obj['id'])
+ result.append(obj)
+ return result
def extra_job_filters(not_yet_run=False, running=False, finished=False):
- """\
- Generate a SQL WHERE clause for job status filtering, and return it in
- a dict of keyword args to pass to query.extra(). No more than one of
- the parameters should be passed as True.
- """
- assert not ((not_yet_run and running) or
- (not_yet_run and finished) or
- (running and finished)), ('Cannot specify more than one '
- 'filter to this function')
- if not_yet_run:
- where = ['id NOT IN (SELECT job_id FROM host_queue_entries '
- 'WHERE active OR complete)']
- elif running:
- where = ['(id IN (SELECT job_id FROM host_queue_entries '
- 'WHERE active OR complete)) AND '
- '(id IN (SELECT job_id FROM host_queue_entries '
- 'WHERE not complete OR active))']
- elif finished:
- where = ['id NOT IN (SELECT job_id FROM host_queue_entries '
- 'WHERE not complete OR active)']
- else:
- return None
- return {'where': where}
+ """\
+ Generate a SQL WHERE clause for job status filtering, and return it in
+ a dict of keyword args to pass to query.extra(). No more than one of
+ the parameters should be passed as True.
+ """
+ assert not ((not_yet_run and running) or
+ (not_yet_run and finished) or
+ (running and finished)), ('Cannot specify more than one '
+ 'filter to this function')
+ if not_yet_run:
+ where = ['id NOT IN (SELECT job_id FROM host_queue_entries '
+ 'WHERE active OR complete)']
+ elif running:
+ where = ['(id IN (SELECT job_id FROM host_queue_entries '
+ 'WHERE active OR complete)) AND '
+ '(id IN (SELECT job_id FROM host_queue_entries '
+ 'WHERE not complete OR active))']
+ elif finished:
+ where = ['id NOT IN (SELECT job_id FROM host_queue_entries '
+ 'WHERE not complete OR active)']
+ else:
+ return None
+ return {'where': where}
def extra_host_filters(multiple_labels=[]):
- """\
- Generate SQL WHERE clauses for matching hosts in an intersection of
- labels.
- """
- extra_args = {}
- where_str = ('hosts.id in (select host_id from hosts_labels '
- 'where label_id=%s)')
- extra_args['where'] = [where_str] * len(multiple_labels)
- extra_args['params'] = [models.Label.smart_get(label).id
- for label in multiple_labels]
- return extra_args
+ """\
+ Generate SQL WHERE clauses for matching hosts in an intersection of
+ labels.
+ """
+ extra_args = {}
+ where_str = ('hosts.id in (select host_id from hosts_labels '
+ 'where label_id=%s)')
+ extra_args['where'] = [where_str] * len(multiple_labels)
+ extra_args['params'] = [models.Label.smart_get(label).id
+ for label in multiple_labels]
+ return extra_args
local_vars = threading.local()
def set_user(user):
- """\
- Sets the current request's logged-in user. user should be a
- afe.models.User object.
- """
- local_vars.user = user
+ """\
+ Sets the current request's logged-in user. user should be a
+ afe.models.User object.
+ """
+ local_vars.user = user
def get_user():
- 'Get the currently logged-in user as a afe.models.User object.'
- return local_vars.user
+ 'Get the currently logged-in user as a afe.models.User object.'
+ return local_vars.user
class InconsistencyException(Exception):
- 'Raised when a list of objects does not have a consistent value'
+ 'Raised when a list of objects does not have a consistent value'
def get_consistent_value(objects, field):
- value = getattr(objects[0], field)
- for obj in objects:
- this_value = getattr(obj, field)
- if this_value != value:
- raise InconsistencyException(objects[0], obj)
- return value
+ value = getattr(objects[0], field)
+ for obj in objects:
+ this_value = getattr(obj, field)
+ if this_value != value:
+ raise InconsistencyException(objects[0], obj)
+ return value
def prepare_generate_control_file(tests, kernel, label):
- test_objects = [models.Test.smart_get(test) for test in tests]
- # ensure tests are all the same type
- try:
- test_type = get_consistent_value(test_objects, 'test_type')
- except InconsistencyException, exc:
- test1, test2 = exc.args
- raise models.ValidationError(
- {'tests' : 'You cannot run both server- and client-side '
- 'tests together (tests %s and %s differ' % (
- test1.name, test2.name)})
+ test_objects = [models.Test.smart_get(test) for test in tests]
+ # ensure tests are all the same type
+ try:
+ test_type = get_consistent_value(test_objects, 'test_type')
+ except InconsistencyException, exc:
+ test1, test2 = exc.args
+ raise models.ValidationError(
+ {'tests' : 'You cannot run both server- and client-side '
+ 'tests together (tests %s and %s differ' % (
+ test1.name, test2.name)})
- try:
- synch_type = get_consistent_value(test_objects, 'synch_type')
- except InconsistencyException, exc:
- test1, test2 = exc.args
- raise models.ValidationError(
- {'tests' : 'You cannot run both synchronous and '
- 'asynchronous tests together (tests %s and %s differ)' % (
- test1.name, test2.name)})
+ try:
+ synch_type = get_consistent_value(test_objects, 'synch_type')
+ except InconsistencyException, exc:
+ test1, test2 = exc.args
+ raise models.ValidationError(
+ {'tests' : 'You cannot run both synchronous and '
+ 'asynchronous tests together (tests %s and %s differ)' % (
+ test1.name, test2.name)})
- is_server = (test_type == models.Test.Types.SERVER)
- is_synchronous = (synch_type == models.Test.SynchType.SYNCHRONOUS)
- if label:
- label = models.Label.smart_get(label)
+ is_server = (test_type == models.Test.Types.SERVER)
+ is_synchronous = (synch_type == models.Test.SynchType.SYNCHRONOUS)
+ if label:
+ label = models.Label.smart_get(label)
- return is_server, is_synchronous, test_objects, label
+ return is_server, is_synchronous, test_objects, label
def sorted(in_list):
- new_list = list(in_list)
- new_list.sort()
- return new_list
+ new_list = list(in_list)
+ new_list.sort()
+ return new_list
diff --git a/frontend/afe/simplejson/__init__.py b/frontend/afe/simplejson/__init__.py
index 181225b..b903c7d 100644
--- a/frontend/afe/simplejson/__init__.py
+++ b/frontend/afe/simplejson/__init__.py
@@ -11,7 +11,7 @@
BSD licensed.
Encoding basic Python object hierarchies::
-
+
>>> import simplejson
>>> simplejson.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}])
'["foo", {"bar": ["baz", null, 1.0, 2]}]'
@@ -40,12 +40,12 @@
>>> import simplejson
>>> print simplejson.dumps({'4': 5, '6': 7}, sort_keys=True, indent=4)
{
- "4": 5,
+ "4": 5,
"6": 7
}
Decoding JSON::
-
+
>>> import simplejson
>>> simplejson.loads('["foo", {"bar":["baz", null, 1.0, 2]}]')
[u'foo', {u'bar': [u'baz', None, 1.0, 2]}]
@@ -63,27 +63,27 @@
... if '__complex__' in dct:
... return complex(dct['real'], dct['imag'])
... return dct
- ...
+ ...
>>> simplejson.loads('{"__complex__": true, "real": 1, "imag": 2}',
... object_hook=as_complex)
(1+2j)
Extending JSONEncoder::
-
+
>>> import simplejson
>>> class ComplexEncoder(simplejson.JSONEncoder):
... def default(self, obj):
... if isinstance(obj, complex):
... return [obj.real, obj.imag]
... return simplejson.JSONEncoder.default(self, obj)
- ...
+ ...
>>> dumps(2 + 1j, cls=ComplexEncoder)
'[2.0, 1.0]'
>>> ComplexEncoder().encode(2 + 1j)
'[2.0, 1.0]'
>>> list(ComplexEncoder().iterencode(2 + 1j))
['[', '2.0', ', ', '1.0', ']']
-
+
Note that the JSON produced by this module's default settings
is a subset of YAML, so it may be used as a serializer for that as well.
@@ -104,7 +104,7 @@
``.write()``-supporting file-like object).
If ``skipkeys`` is ``True`` then ``dict`` keys that are not basic types
- (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``)
+ (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``)
will be skipped instead of raising a ``TypeError``.
If ``ensure_ascii`` is ``False``, then the some chunks written to ``fp``
@@ -146,7 +146,7 @@
Serialize ``obj`` to a JSON formatted ``str``.
If ``skipkeys`` is ``True`` then ``dict`` keys that are not basic types
- (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``)
+ (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``)
will be skipped instead of raising a ``TypeError``.
If ``ensure_ascii`` is ``False``, then the return value will be a
@@ -199,7 +199,7 @@
result of any object literal decode (a ``dict``). The return value of
``object_hook`` will be used instead of the ``dict``. This feature
can be used to implement custom decoders (e.g. JSON-RPC class hinting).
-
+
To use a custom ``JSONDecoder`` subclass, specify it with the ``cls``
kwarg.
"""
diff --git a/frontend/afe/simplejson/decoder.py b/frontend/afe/simplejson/decoder.py
index 63f70cb..46a8fb0 100644
--- a/frontend/afe/simplejson/decoder.py
+++ b/frontend/afe/simplejson/decoder.py
@@ -156,7 +156,7 @@
pairs = object_hook(pairs)
return pairs, end
pattern(r'{')(JSONObject)
-
+
def JSONArray(match, context, _w=WHITESPACE.match):
values = []
s = match.string
@@ -182,7 +182,7 @@
end = _w(s, end).end()
return values, end
pattern(r'\[')(JSONArray)
-
+
ANYTHING = [
JSONObject,
JSONArray,
@@ -198,7 +198,7 @@
Simple JSON <http://json.org> decoder
Performs the following translations in decoding:
-
+
+---------------+-------------------+
| JSON | Python |
+===============+===================+
@@ -231,7 +231,7 @@
``encoding`` determines the encoding used to interpret any ``str``
objects decoded by this instance (utf-8 by default). It has no
effect when decoding ``unicode`` objects.
-
+
Note that currently only encodings that are a superset of ASCII work,
strings of other encodings should be passed in as ``unicode``.
diff --git a/frontend/afe/simplejson/encoder.py b/frontend/afe/simplejson/encoder.py
index c83c687..5271f6b 100644
--- a/frontend/afe/simplejson/encoder.py
+++ b/frontend/afe/simplejson/encoder.py
@@ -58,14 +58,14 @@
except KeyError:
return '\\u%04x' % (ord(s),)
return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"'
-
+
class JSONEncoder(object):
"""
Extensible JSON <http://json.org> encoder for Python data structures.
Supports the following objects and types by default:
-
+
+-------------------+---------------+
| Python | JSON |
+===================+===============+
@@ -287,7 +287,7 @@
For example, to support arbitrary iterators, you could
implement default like this::
-
+
def default(self, o):
try:
iterable = iter(o)
@@ -316,9 +316,9 @@
"""
Encode the given object and yield each string
representation as available.
-
+
For example::
-
+
for chunk in JSONEncoder().iterencode(bigobject):
mysocket.write(chunk)
"""
diff --git a/frontend/afe/simplejson/scanner.py b/frontend/afe/simplejson/scanner.py
index 64f4999..b21ce6c 100644
--- a/frontend/afe/simplejson/scanner.py
+++ b/frontend/afe/simplejson/scanner.py
@@ -54,7 +54,7 @@
match = self.scanner.scanner(string, matchend).match
yield rval, matchend
lastend = matchend
-
+
def pattern(pattern, flags=FLAGS):
def decorator(fn):
fn.pattern = pattern
diff --git a/frontend/afe/test.py b/frontend/afe/test.py
index 686eb91..be4fe2a 100644
--- a/frontend/afe/test.py
+++ b/frontend/afe/test.py
@@ -16,46 +16,46 @@
app_name = 'afe'
doctest_dir = 'doctests'
doctest_paths = [os.path.join(doctest_dir, filename) for filename
- in os.listdir(os.path.join(app_name, doctest_dir))
- if not filename.startswith('.')]
+ in os.listdir(os.path.join(app_name, doctest_dir))
+ if not filename.startswith('.')]
doctest_paths.sort()
def get_modules():
- modules = []
- module_names = [os.path.basename(filename)[:-3]
- for filename in glob.glob(app_name + '/*.py')
- if '__init__' not in filename
- and 'test.py' not in filename]
- for module_name in module_names:
- __import__('frontend.afe', globals(), locals(), [module_name])
- modules.append(getattr(afe, module_name))
- return modules
+ modules = []
+ module_names = [os.path.basename(filename)[:-3]
+ for filename in glob.glob(app_name + '/*.py')
+ if '__init__' not in filename
+ and 'test.py' not in filename]
+ for module_name in module_names:
+ __import__('frontend.afe', globals(), locals(), [module_name])
+ modules.append(getattr(afe, module_name))
+ return modules
print_after = 'Ran %d tests from %s'
def run_tests(module_list, verbosity=1):
- modules = get_modules()
- total_errors = 0
- old_db = settings.DATABASE_NAME
- django.test.utils.setup_test_environment()
- django.test.utils.create_test_db(verbosity)
- try:
- for module in modules:
- failures, test_count = doctest.testmod(module)
- print print_after % (test_count, module.__name__)
- total_errors += failures
- for path in doctest_paths:
- failures, test_count = doctest.testfile(path)
- print print_after % (test_count, path)
- total_errors += failures
- finally:
- django.test.utils.destroy_test_db(old_db)
- django.test.utils.teardown_test_environment()
- print
- if total_errors == 0:
- print 'OK'
- else:
- print 'FAIL: %d errors' % total_errors
- return total_errors
+ modules = get_modules()
+ total_errors = 0
+ old_db = settings.DATABASE_NAME
+ django.test.utils.setup_test_environment()
+ django.test.utils.create_test_db(verbosity)
+ try:
+ for module in modules:
+ failures, test_count = doctest.testmod(module)
+ print print_after % (test_count, module.__name__)
+ total_errors += failures
+ for path in doctest_paths:
+ failures, test_count = doctest.testfile(path)
+ print print_after % (test_count, path)
+ total_errors += failures
+ finally:
+ django.test.utils.destroy_test_db(old_db)
+ django.test.utils.teardown_test_environment()
+ print
+ if total_errors == 0:
+ print 'OK'
+ else:
+ print 'FAIL: %d errors' % total_errors
+ return total_errors
diff --git a/frontend/afe/urls.py b/frontend/afe/urls.py
index fa8cdb6..3a77311 100644
--- a/frontend/afe/urls.py
+++ b/frontend/afe/urls.py
@@ -33,6 +33,6 @@
]
if settings.DEBUG:
- pattern_list += debug_pattern_list
+ pattern_list += debug_pattern_list
urlpatterns = patterns('', *pattern_list)
diff --git a/frontend/afe/views.py b/frontend/afe/views.py
index d448215..947c808 100644
--- a/frontend/afe/views.py
+++ b/frontend/afe/views.py
@@ -7,38 +7,38 @@
# since site_rpc_interface is later in the list, its methods will override those
# of rpc_interface
rpc_handler_obj = rpc_handler.RpcHandler((rpc_interface, site_rpc_interface),
- document_module=rpc_interface)
+ document_module=rpc_interface)
def handle_rpc(request):
- rpc_utils.set_user(request.afe_user)
- return rpc_handler_obj.handle_rpc_request(request)
+ rpc_utils.set_user(request.afe_user)
+ return rpc_handler_obj.handle_rpc_request(request)
def model_documentation(request):
- doc = '<h2>Models</h2>\n'
- for model_name in ('Label', 'Host', 'Test', 'User', 'AclGroup', 'Job'):
- model_class = getattr(models, model_name)
- doc += '<h3>%s</h3>\n' % model_name
- doc += '<pre>\n%s</pre>\n' % model_class.__doc__
- return HttpResponse(doc)
+ doc = '<h2>Models</h2>\n'
+ for model_name in ('Label', 'Host', 'Test', 'User', 'AclGroup', 'Job'):
+ model_class = getattr(models, model_name)
+ doc += '<h3>%s</h3>\n' % model_name
+ doc += '<pre>\n%s</pre>\n' % model_class.__doc__
+ return HttpResponse(doc)
def redirect_with_extra_data(request, url, **kwargs):
- kwargs['getdata'] = request.GET.urlencode()
- kwargs['server_name'] = request.META['SERVER_NAME']
- return HttpResponsePermanentRedirect(url % kwargs)
+ kwargs['getdata'] = request.GET.urlencode()
+ kwargs['server_name'] = request.META['SERVER_NAME']
+ return HttpResponsePermanentRedirect(url % kwargs)
GWT_SERVER = 'http://localhost:8888/'
def gwt_forward(request, forward_addr):
- if len(request.POST) == 0:
- data = None
- else:
- data = request.raw_post_data
- url_response = urllib2.urlopen(GWT_SERVER + forward_addr, data=data)
- http_response = HttpResponse(url_response.read())
- for header, value in url_response.info().items():
- if header not in ('connection',):
- http_response[header] = value
- return http_response
+ if len(request.POST) == 0:
+ data = None
+ else:
+ data = request.raw_post_data
+ url_response = urllib2.urlopen(GWT_SERVER + forward_addr, data=data)
+ http_response = HttpResponse(url_response.read())
+ for header, value in url_response.info().items():
+ if header not in ('connection',):
+ http_response[header] = value
+ return http_response
diff --git a/frontend/apache_auth.py b/frontend/apache_auth.py
index 552a6bf..9ce0da2 100644
--- a/frontend/apache_auth.py
+++ b/frontend/apache_auth.py
@@ -7,59 +7,59 @@
DEBUG_USER = 'debug_user'
class SimpleAuthBackend:
- """
- Automatically allows any login. This backend is for use when Apache is
- doing the real authentication. Also ensures logged-in user exists in
- frontend.afe.models.User database.
- """
- def authenticate(self, username=None, password=None):
- try:
- user = User.objects.get(username=username)
- except User.DoesNotExist:
- # password is meaningless
- user = User(username=username,
- password='apache authentication')
- user.is_staff = True
- user.save() # need to save before adding groups
- user.groups.add(Group.objects.get(
- name=management.BASIC_ADMIN))
+ """
+ Automatically allows any login. This backend is for use when Apache is
+ doing the real authentication. Also ensures logged-in user exists in
+ frontend.afe.models.User database.
+ """
+ def authenticate(self, username=None, password=None):
+ try:
+ user = User.objects.get(username=username)
+ except User.DoesNotExist:
+ # password is meaningless
+ user = User(username=username,
+ password='apache authentication')
+ user.is_staff = True
+ user.save() # need to save before adding groups
+ user.groups.add(Group.objects.get(
+ name=management.BASIC_ADMIN))
- SimpleAuthBackend.check_afe_user(username)
- return user
+ SimpleAuthBackend.check_afe_user(username)
+ return user
- @staticmethod
- def check_afe_user(username):
- user, _ = models.User.objects.get_or_create(login=username)
- user.save()
+ @staticmethod
+ def check_afe_user(username):
+ user, _ = models.User.objects.get_or_create(login=username)
+ user.save()
- def get_user(self, user_id):
- try:
- return User.objects.get(pk=user_id)
- except User.DoesNotExist:
- return None
+ def get_user(self, user_id):
+ try:
+ return User.objects.get(pk=user_id)
+ except User.DoesNotExist:
+ return None
class ApacheAuthMiddleware(object):
- """
- Middleware for use when Apache is doing authentication. Looks for
- REQUEST_USER in requests and logs that user in. If no such header is
- found, looks for HTTP_AUTHORIZATION header with username to login (this
- allows CLI to authenticate).
- """
+ """
+ Middleware for use when Apache is doing authentication. Looks for
+ REQUEST_USER in requests and logs that user in. If no such header is
+ found, looks for HTTP_AUTHORIZATION header with username to login (this
+ allows CLI to authenticate).
+ """
- def process_request(self, request):
- # look for a username from Apache
- user = request.META.get('REMOTE_USER')
- if user is None:
- # look for a user in headers. This is insecure but
- # it's our temporarily solution for CLI auth.
- user = request.META.get('HTTP_AUTHORIZATION')
- if user is None:
- # no user info - assume we're in development mode
- user = DEBUG_USER
- user_object = auth.authenticate(username=user,
- password='')
- auth.login(request, user_object)
- request.afe_user = models.User.objects.get(login=user)
- return None
+ def process_request(self, request):
+ # look for a username from Apache
+ user = request.META.get('REMOTE_USER')
+ if user is None:
+ # look for a user in headers. This is insecure but
+ # it's our temporarily solution for CLI auth.
+ user = request.META.get('HTTP_AUTHORIZATION')
+ if user is None:
+ # no user info - assume we're in development mode
+ user = DEBUG_USER
+ user_object = auth.authenticate(username=user,
+ password='')
+ auth.login(request, user_object)
+ request.afe_user = models.User.objects.get(login=user)
+ return None
diff --git a/frontend/frontend_unittest.py b/frontend/frontend_unittest.py
index 87f18d1..65f9573 100644
--- a/frontend/frontend_unittest.py
+++ b/frontend/frontend_unittest.py
@@ -10,11 +10,11 @@
class FrontendTest(unittest.TestCase):
- def test_all(self):
- manage_dir = os.path.dirname(__file__)
- result = os.system("cd %s && ./manage.py test" % (manage_dir))
- self.assert_(result == 0)
+ def test_all(self):
+ manage_dir = os.path.dirname(__file__)
+ result = os.system("cd %s && ./manage.py test" % (manage_dir))
+ self.assert_(result == 0)
if __name__ == '__main__':
- unittest.main()
+ unittest.main()
diff --git a/frontend/manage.py b/frontend/manage.py
index e70d6d2..5e78ea9 100755
--- a/frontend/manage.py
+++ b/frontend/manage.py
@@ -1,11 +1,11 @@
#!/usr/bin/env python
from django.core.management import execute_manager
try:
- import settings # Assumed to be in the same directory.
+ import settings # Assumed to be in the same directory.
except ImportError:
- import sys
- sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__)
- sys.exit(1)
+ import sys
+ sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__)
+ sys.exit(1)
if __name__ == "__main__":
- execute_manager(settings)
+ execute_manager(settings)
diff --git a/frontend/migrations/001_initial_db.py b/frontend/migrations/001_initial_db.py
index f16e6ec..036754c 100644
--- a/frontend/migrations/001_initial_db.py
+++ b/frontend/migrations/001_initial_db.py
@@ -1,33 +1,33 @@
import os
required_tables = ('acl_groups', 'acl_groups_hosts', 'acl_groups_users',
- 'autotests', 'host_queue_entries', 'hosts', 'hosts_labels',
- 'ineligible_host_queues', 'jobs', 'labels', 'users')
+ 'autotests', 'host_queue_entries', 'hosts', 'hosts_labels',
+ 'ineligible_host_queues', 'jobs', 'labels', 'users')
def migrate_up(manager):
- manager.execute("SHOW TABLES")
- tables = [row[0] for row in manager.cursor.fetchall()]
- db_initialized = True
- for table in required_tables:
- if table not in tables:
- db_initialized = False
- break
- if not db_initialized:
- response = raw_input(
- 'Your autotest_web database does not appear to be '
- 'initialized. Do you want to recreate it (this will '
- 'result in loss of any existing data) (yes/No)? ')
- if response != 'yes':
- raise Exception('User has chosen to abort migration')
+ manager.execute("SHOW TABLES")
+ tables = [row[0] for row in manager.cursor.fetchall()]
+ db_initialized = True
+ for table in required_tables:
+ if table not in tables:
+ db_initialized = False
+ break
+ if not db_initialized:
+ response = raw_input(
+ 'Your autotest_web database does not appear to be '
+ 'initialized. Do you want to recreate it (this will '
+ 'result in loss of any existing data) (yes/No)? ')
+ if response != 'yes':
+ raise Exception('User has chosen to abort migration')
- manager.execute_script(CREATE_DB_SQL)
+ manager.execute_script(CREATE_DB_SQL)
- manager.create_migrate_table()
+ manager.create_migrate_table()
def migrate_down(manager):
- manager.execute_script(DROP_DB_SQL)
+ manager.execute_script(DROP_DB_SQL)
CREATE_DB_SQL = """\
diff --git a/frontend/migrations/002_cleanup_fields.py b/frontend/migrations/002_cleanup_fields.py
index a3656ca..88abc78 100644
--- a/frontend/migrations/002_cleanup_fields.py
+++ b/frontend/migrations/002_cleanup_fields.py
@@ -1,12 +1,12 @@
def migrate_up(manager):
- manager.execute('ALTER TABLE autotests DROP params')
- manager.execute('ALTER TABLE jobs DROP kernel_url, DROP status, '
- 'DROP submitted_on')
- manager.execute('ALTER TABLE host_queue_entries DROP created_on')
+ manager.execute('ALTER TABLE autotests DROP params')
+ manager.execute('ALTER TABLE jobs DROP kernel_url, DROP status, '
+ 'DROP submitted_on')
+ manager.execute('ALTER TABLE host_queue_entries DROP created_on')
def migrate_down(manager):
- manager.execute('ALTER TABLE autotests ADD params VARCHAR(255)')
- manager.execute('ALTER TABLE jobs ADD kernel_url VARCHAR(255), '
- 'ADD status VARCHAR(255), ADD submitted_on datetime')
- manager.execute('ALTER TABLE host_queue_entries ADD created_on '
- 'datetime')
+ manager.execute('ALTER TABLE autotests ADD params VARCHAR(255)')
+ manager.execute('ALTER TABLE jobs ADD kernel_url VARCHAR(255), '
+ 'ADD status VARCHAR(255), ADD submitted_on datetime')
+ manager.execute('ALTER TABLE host_queue_entries ADD created_on '
+ 'datetime')
diff --git a/frontend/migrations/003_test_synch_type.py b/frontend/migrations/003_test_synch_type.py
index b20c01d..939d65f 100644
--- a/frontend/migrations/003_test_synch_type.py
+++ b/frontend/migrations/003_test_synch_type.py
@@ -1,9 +1,9 @@
def migrate_up(manager):
- manager.execute('ALTER TABLE autotests ADD `synch_type` smallint '
- 'NOT NULL')
- # set all to asynchronous by default
- manager.execute('UPDATE autotests SET synch_type=1')
+ manager.execute('ALTER TABLE autotests ADD `synch_type` smallint '
+ 'NOT NULL')
+ # set all to asynchronous by default
+ manager.execute('UPDATE autotests SET synch_type=1')
def migrate_down(manager):
- manager.execute('ALTER TABLE autotests DROP `synch_type`')
+ manager.execute('ALTER TABLE autotests DROP `synch_type`')
diff --git a/frontend/migrations/004_add_indexes.py b/frontend/migrations/004_add_indexes.py
index 7219f23..40b496f 100644
--- a/frontend/migrations/004_add_indexes.py
+++ b/frontend/migrations/004_add_indexes.py
@@ -8,16 +8,16 @@
)
def get_index_name(table, field):
- return table + '_' + field
+ return table + '_' + field
def migrate_up(manager):
- for table, field in INDEXES:
- manager.execute('CREATE INDEX %s ON %s (%s)' %
- (get_index_name(table, field), table, field))
+ for table, field in INDEXES:
+ manager.execute('CREATE INDEX %s ON %s (%s)' %
+ (get_index_name(table, field), table, field))
def migrate_down(manager):
- for table, field in INDEXES:
- manager.execute('DROP INDEX %s ON %s' %
- (get_index_name(table, field), table))
+ for table, field in INDEXES:
+ manager.execute('DROP INDEX %s ON %s' %
+ (get_index_name(table, field), table))
diff --git a/frontend/migrations/005_one_more_index.py b/frontend/migrations/005_one_more_index.py
index aa3a81d..787c1c0 100644
--- a/frontend/migrations/005_one_more_index.py
+++ b/frontend/migrations/005_one_more_index.py
@@ -1,7 +1,7 @@
def migrate_up(manger):
- manger.execute('CREATE INDEX hosts_labels_host_id ON hosts_labels '
- '(host_id)')
+ manger.execute('CREATE INDEX hosts_labels_host_id ON hosts_labels '
+ '(host_id)')
def migrate_down(manger):
- manger.execute('DROP INDEX hosts_labels_host_id ON hosts_labels')
+ manger.execute('DROP INDEX hosts_labels_host_id ON hosts_labels')
diff --git a/frontend/migrations/006_host_label_invalid.py b/frontend/migrations/006_host_label_invalid.py
index 9f6a6f1..20c5e4c 100644
--- a/frontend/migrations/006_host_label_invalid.py
+++ b/frontend/migrations/006_host_label_invalid.py
@@ -1,8 +1,8 @@
def migrate_up(manager):
- manager.execute('ALTER TABLE hosts ADD `invalid` bool NOT NULL')
- manager.execute('ALTER TABLE labels ADD `invalid` bool NOT NULL')
+ manager.execute('ALTER TABLE hosts ADD `invalid` bool NOT NULL')
+ manager.execute('ALTER TABLE labels ADD `invalid` bool NOT NULL')
def migrate_down(manager):
- manager.execute('ALTER TABLE hosts DROP invalid')
- manager.execute('ALTER TABLE labels DROP invalid')
+ manager.execute('ALTER TABLE hosts DROP invalid')
+ manager.execute('ALTER TABLE labels DROP invalid')
diff --git a/frontend/migrations/007_indexes_on_acl_tables.py b/frontend/migrations/007_indexes_on_acl_tables.py
index 71f6a01..1dab8cf 100644
--- a/frontend/migrations/007_indexes_on_acl_tables.py
+++ b/frontend/migrations/007_indexes_on_acl_tables.py
@@ -6,16 +6,16 @@
)
def get_index_name(table, field):
- return table + '_' + field
+ return table + '_' + field
def migrate_up(manager):
- for table, field in INDEXES:
- manager.execute('CREATE INDEX %s ON %s (%s)' %
- (get_index_name(table, field), table, field))
+ for table, field in INDEXES:
+ manager.execute('CREATE INDEX %s ON %s (%s)' %
+ (get_index_name(table, field), table, field))
def migrate_down(manager):
- for table, field in INDEXES:
- manager.execute('DROP INDEX %s ON %s' %
- (get_index_name(table, field), table))
+ for table, field in INDEXES:
+ manager.execute('DROP INDEX %s ON %s' %
+ (get_index_name(table, field), table))
diff --git a/frontend/settings.py b/frontend/settings.py
index 74eb5ec..a47eeca 100644
--- a/frontend/settings.py
+++ b/frontend/settings.py
@@ -14,19 +14,19 @@
MANAGERS = ADMINS
-DATABASE_ENGINE = 'mysql_old' # 'postgresql_psycopg2', 'postgresql',
+DATABASE_ENGINE = 'mysql_old' # 'postgresql_psycopg2', 'postgresql',
# 'mysql', 'sqlite3' or 'ado_mssql'.
-DATABASE_PORT = '' # Set to empty string for default.
+DATABASE_PORT = '' # Set to empty string for default.
# Not used with sqlite3.
c = global_config.global_config
-DATABASE_HOST = c.get_config_value("AUTOTEST_WEB", "host")
+DATABASE_HOST = c.get_config_value("AUTOTEST_WEB", "host")
# Or path to database file if using sqlite3.
DATABASE_NAME = c.get_config_value("AUTOTEST_WEB", "database")
# The following not used with sqlite3.
DATABASE_USER = c.get_config_value("AUTOTEST_WEB", "user")
DATABASE_PASSWORD = c.get_config_value("AUTOTEST_WEB", "password")
-
+
# prefix applied to all URLs - useful if requests are coming through apache,
# and you need this app to coexist with others
diff --git a/migrate/migrate.py b/migrate/migrate.py
index 8bbcb5f..fbed905 100755
--- a/migrate/migrate.py
+++ b/migrate/migrate.py
@@ -10,267 +10,267 @@
DEFAULT_MIGRATIONS_DIR = 'migrations'
class Migration(object):
- def __init__(self, filename):
- self.version = int(filename[:3])
- self.name = filename[:-3]
- self.module = __import__(self.name, globals(), locals(), [])
- assert hasattr(self.module, 'migrate_up')
- assert hasattr(self.module, 'migrate_down')
+ def __init__(self, filename):
+ self.version = int(filename[:3])
+ self.name = filename[:-3]
+ self.module = __import__(self.name, globals(), locals(), [])
+ assert hasattr(self.module, 'migrate_up')
+ assert hasattr(self.module, 'migrate_down')
- def migrate_up(self, manager):
- self.module.migrate_up(manager)
+ def migrate_up(self, manager):
+ self.module.migrate_up(manager)
- def migrate_down(self, manager):
- self.module.migrate_down(manager)
+ def migrate_down(self, manager):
+ self.module.migrate_down(manager)
class MigrationManager(object):
- connection = None
- cursor = None
- migrations_dir = None
+ connection = None
+ cursor = None
+ migrations_dir = None
- def __init__(self, database, migrations_dir=None):
- self.database = database
- if migrations_dir is None:
- migrations_dir = os.path.abspath(DEFAULT_MIGRATIONS_DIR)
- self.migrations_dir = migrations_dir
- sys.path.append(migrations_dir)
- assert os.path.exists(migrations_dir)
+ def __init__(self, database, migrations_dir=None):
+ self.database = database
+ if migrations_dir is None:
+ migrations_dir = os.path.abspath(DEFAULT_MIGRATIONS_DIR)
+ self.migrations_dir = migrations_dir
+ sys.path.append(migrations_dir)
+ assert os.path.exists(migrations_dir)
- self.db_host = None
- self.db_name = None
- self.username = None
- self.password = None
+ self.db_host = None
+ self.db_name = None
+ self.username = None
+ self.password = None
- def read_db_info(self):
- # grab the config file and parse for info
- c = global_config.global_config
- self.db_host = c.get_config_value(self.database, "host")
- self.db_name = c.get_config_value(self.database, "database")
- self.username = c.get_config_value(self.database, "user")
- self.password = c.get_config_value(self.database, "password")
+ def read_db_info(self):
+ # grab the config file and parse for info
+ c = global_config.global_config
+ self.db_host = c.get_config_value(self.database, "host")
+ self.db_name = c.get_config_value(self.database, "database")
+ self.username = c.get_config_value(self.database, "user")
+ self.password = c.get_config_value(self.database, "password")
- def connect(self, host, db_name, username, password):
- return MySQLdb.connect(host=host, db=db_name, user=username,
- passwd=password)
+ def connect(self, host, db_name, username, password):
+ return MySQLdb.connect(host=host, db=db_name, user=username,
+ passwd=password)
- def open_connection(self):
- self.connection = self.connect(self.db_host, self.db_name,
- self.username, self.password)
- self.connection.autocommit(True)
- self.cursor = self.connection.cursor()
+ def open_connection(self):
+ self.connection = self.connect(self.db_host, self.db_name,
+ self.username, self.password)
+ self.connection.autocommit(True)
+ self.cursor = self.connection.cursor()
- def close_connection(self):
- self.connection.close()
+ def close_connection(self):
+ self.connection.close()
- def execute(self, query, *parameters):
- #print 'SQL:', query % parameters
- return self.cursor.execute(query, parameters)
+ def execute(self, query, *parameters):
+ #print 'SQL:', query % parameters
+ return self.cursor.execute(query, parameters)
- def execute_script(self, script):
- sql_statements = [statement.strip() for statement
- in script.split(';')]
- for statement in sql_statements:
- if statement:
- self.execute(statement)
+ def execute_script(self, script):
+ sql_statements = [statement.strip() for statement
+ in script.split(';')]
+ for statement in sql_statements:
+ if statement:
+ self.execute(statement)
- def check_migrate_table_exists(self):
- try:
- self.execute("SELECT * FROM %s" % MIGRATE_TABLE)
- return True
- except MySQLdb.ProgrammingError, exc:
- error_code, _ = exc.args
- if error_code == MySQLdb.constants.ER.NO_SUCH_TABLE:
- return False
- raise
+ def check_migrate_table_exists(self):
+ try:
+ self.execute("SELECT * FROM %s" % MIGRATE_TABLE)
+ return True
+ except MySQLdb.ProgrammingError, exc:
+ error_code, _ = exc.args
+ if error_code == MySQLdb.constants.ER.NO_SUCH_TABLE:
+ return False
+ raise
- def create_migrate_table(self):
- if not self.check_migrate_table_exists():
- self.execute("CREATE TABLE %s (`version` integer)" %
- MIGRATE_TABLE)
- else:
- self.execute("DELETE FROM %s" % MIGRATE_TABLE)
- self.execute("INSERT INTO %s VALUES (0)" % MIGRATE_TABLE)
- assert self.cursor.rowcount == 1
+ def create_migrate_table(self):
+ if not self.check_migrate_table_exists():
+ self.execute("CREATE TABLE %s (`version` integer)" %
+ MIGRATE_TABLE)
+ else:
+ self.execute("DELETE FROM %s" % MIGRATE_TABLE)
+ self.execute("INSERT INTO %s VALUES (0)" % MIGRATE_TABLE)
+ assert self.cursor.rowcount == 1
- def set_db_version(self, version):
- assert isinstance(version, int)
- self.execute("UPDATE %s SET version=%%s" % MIGRATE_TABLE,
- version)
- assert self.cursor.rowcount == 1
+ def set_db_version(self, version):
+ assert isinstance(version, int)
+ self.execute("UPDATE %s SET version=%%s" % MIGRATE_TABLE,
+ version)
+ assert self.cursor.rowcount == 1
- def get_db_version(self):
- if not self.check_migrate_table_exists():
- return 0
- self.execute("SELECT * FROM %s" % MIGRATE_TABLE)
- rows = self.cursor.fetchall()
- if len(rows) == 0:
- return 0
- assert len(rows) == 1 and len(rows[0]) == 1
- return rows[0][0]
+ def get_db_version(self):
+ if not self.check_migrate_table_exists():
+ return 0
+ self.execute("SELECT * FROM %s" % MIGRATE_TABLE)
+ rows = self.cursor.fetchall()
+ if len(rows) == 0:
+ return 0
+ assert len(rows) == 1 and len(rows[0]) == 1
+ return rows[0][0]
- def get_migrations(self, minimum_version=None, maximum_version=None):
- migrate_files = [filename for filename
- in os.listdir(self.migrations_dir)
- if re.match(r'^\d\d\d_.*\.py$', filename)]
- migrate_files.sort()
- migrations = [Migration(filename) for filename in migrate_files]
- if minimum_version is not None:
- migrations = [migration for migration in migrations
- if migration.version >= minimum_version]
- if maximum_version is not None:
- migrations = [migration for migration in migrations
- if migration.version <= maximum_version]
- return migrations
+ def get_migrations(self, minimum_version=None, maximum_version=None):
+ migrate_files = [filename for filename
+ in os.listdir(self.migrations_dir)
+ if re.match(r'^\d\d\d_.*\.py$', filename)]
+ migrate_files.sort()
+ migrations = [Migration(filename) for filename in migrate_files]
+ if minimum_version is not None:
+ migrations = [migration for migration in migrations
+ if migration.version >= minimum_version]
+ if maximum_version is not None:
+ migrations = [migration for migration in migrations
+ if migration.version <= maximum_version]
+ return migrations
- def do_migration(self, migration, migrate_up=True):
- print 'Applying migration %s' % migration.name, # no newline
- if migrate_up:
- print 'up'
- assert self.get_db_version() == migration.version - 1
- migration.migrate_up(self)
- new_version = migration.version
- else:
- print 'down'
- assert self.get_db_version() == migration.version
- migration.migrate_down(self)
- new_version = migration.version - 1
- self.set_db_version(new_version)
+ def do_migration(self, migration, migrate_up=True):
+ print 'Applying migration %s' % migration.name, # no newline
+ if migrate_up:
+ print 'up'
+ assert self.get_db_version() == migration.version - 1
+ migration.migrate_up(self)
+ new_version = migration.version
+ else:
+ print 'down'
+ assert self.get_db_version() == migration.version
+ migration.migrate_down(self)
+ new_version = migration.version - 1
+ self.set_db_version(new_version)
- def migrate_to_version(self, version):
- current_version = self.get_db_version()
- if current_version < version:
- lower, upper = current_version, version
- migrate_up = True
- else:
- lower, upper = version, current_version
- migrate_up = False
+ def migrate_to_version(self, version):
+ current_version = self.get_db_version()
+ if current_version < version:
+ lower, upper = current_version, version
+ migrate_up = True
+ else:
+ lower, upper = version, current_version
+ migrate_up = False
- migrations = self.get_migrations(lower + 1, upper)
- if not migrate_up:
- migrations.reverse()
- for migration in migrations:
- self.do_migration(migration, migrate_up)
+ migrations = self.get_migrations(lower + 1, upper)
+ if not migrate_up:
+ migrations.reverse()
+ for migration in migrations:
+ self.do_migration(migration, migrate_up)
- assert self.get_db_version() == version
- print 'At version', version
+ assert self.get_db_version() == version
+ print 'At version', version
- def get_latest_version(self):
- migrations = self.get_migrations()
- return migrations[-1].version
+ def get_latest_version(self):
+ migrations = self.get_migrations()
+ return migrations[-1].version
- def migrate_to_latest(self):
- latest_version = self.get_latest_version()
- self.migrate_to_version(latest_version)
+ def migrate_to_latest(self):
+ latest_version = self.get_latest_version()
+ self.migrate_to_version(latest_version)
- def initialize_test_db(self):
- self.read_db_info()
- test_db_name = 'test_' + self.db_name
- # first, connect to no DB so we can create a test DB
- self.db_name = ''
- self.open_connection()
- print 'Creating test DB', test_db_name
- self.execute('CREATE DATABASE ' + test_db_name)
- self.close_connection()
- # now connect to the test DB
- self.db_name = test_db_name
- self.open_connection()
+ def initialize_test_db(self):
+ self.read_db_info()
+ test_db_name = 'test_' + self.db_name
+ # first, connect to no DB so we can create a test DB
+ self.db_name = ''
+ self.open_connection()
+ print 'Creating test DB', test_db_name
+ self.execute('CREATE DATABASE ' + test_db_name)
+ self.close_connection()
+ # now connect to the test DB
+ self.db_name = test_db_name
+ self.open_connection()
- def remove_test_db(self):
- print 'Removing test DB'
- self.execute('DROP DATABASE ' + self.db_name)
+ def remove_test_db(self):
+ print 'Removing test DB'
+ self.execute('DROP DATABASE ' + self.db_name)
- def get_mysql_args(self):
- return ('-u %(user)s -p%(password)s -h %(host)s %(db)s' % {
- 'user' : self.username,
- 'password' : self.password,
- 'host' : self.db_host,
- 'db' : self.db_name})
+ def get_mysql_args(self):
+ return ('-u %(user)s -p%(password)s -h %(host)s %(db)s' % {
+ 'user' : self.username,
+ 'password' : self.password,
+ 'host' : self.db_host,
+ 'db' : self.db_name})
- def migrate_to_version_or_latest(self, version):
- if version is None:
- self.migrate_to_latest()
- else:
- self.migrate_to_version(version)
+ def migrate_to_version_or_latest(self, version):
+ if version is None:
+ self.migrate_to_latest()
+ else:
+ self.migrate_to_version(version)
- def do_sync_db(self, version=None):
- self.read_db_info()
- self.open_connection()
- print 'Migration starting for database', self.db_name
- self.migrate_to_version_or_latest(version)
- print 'Migration complete'
+ def do_sync_db(self, version=None):
+ self.read_db_info()
+ self.open_connection()
+ print 'Migration starting for database', self.db_name
+ self.migrate_to_version_or_latest(version)
+ print 'Migration complete'
- def test_sync_db(self, version=None):
- """\
- Create a fresh DB and run all migrations on it.
- """
- self.initialize_test_db()
- try:
- print 'Starting migration test on DB', self.db_name
- self.migrate_to_version_or_latest(version)
- # show schema to the user
- os.system('mysqldump %s --no-data=true '
- '--add-drop-table=false' %
- self.get_mysql_args())
- finally:
- self.remove_test_db()
- print 'Test finished successfully'
+ def test_sync_db(self, version=None):
+ """\
+ Create a fresh DB and run all migrations on it.
+ """
+ self.initialize_test_db()
+ try:
+ print 'Starting migration test on DB', self.db_name
+ self.migrate_to_version_or_latest(version)
+ # show schema to the user
+ os.system('mysqldump %s --no-data=true '
+ '--add-drop-table=false' %
+ self.get_mysql_args())
+ finally:
+ self.remove_test_db()
+ print 'Test finished successfully'
- def simulate_sync_db(self, version=None):
- """\
- Create a fresh DB, copy the existing DB to it, and then
- try to synchronize it.
- """
- self.read_db_info()
- self.open_connection()
- db_version = self.get_db_version()
- self.close_connection()
- # don't do anything if we're already at the latest version
- if db_version == self.get_latest_version():
- print 'Skipping simulation, already at latest version'
- return
- # get existing data
- self.read_db_info()
- print 'Dumping existing data'
- dump_fd, dump_file = tempfile.mkstemp('.migrate_dump')
- os.close(dump_fd)
- os.system('mysqldump %s >%s' %
- (self.get_mysql_args(), dump_file))
- # fill in test DB
- self.initialize_test_db()
- print 'Filling in test DB'
- os.system('mysql %s <%s' % (self.get_mysql_args(), dump_file))
- os.remove(dump_file)
- try:
- print 'Starting migration test on DB', self.db_name
- self.migrate_to_version_or_latest(version)
- finally:
- self.remove_test_db()
- print 'Test finished successfully'
+ def simulate_sync_db(self, version=None):
+ """\
+ Create a fresh DB, copy the existing DB to it, and then
+ try to synchronize it.
+ """
+ self.read_db_info()
+ self.open_connection()
+ db_version = self.get_db_version()
+ self.close_connection()
+ # don't do anything if we're already at the latest version
+ if db_version == self.get_latest_version():
+ print 'Skipping simulation, already at latest version'
+ return
+ # get existing data
+ self.read_db_info()
+ print 'Dumping existing data'
+ dump_fd, dump_file = tempfile.mkstemp('.migrate_dump')
+ os.close(dump_fd)
+ os.system('mysqldump %s >%s' %
+ (self.get_mysql_args(), dump_file))
+ # fill in test DB
+ self.initialize_test_db()
+ print 'Filling in test DB'
+ os.system('mysql %s <%s' % (self.get_mysql_args(), dump_file))
+ os.remove(dump_file)
+ try:
+ print 'Starting migration test on DB', self.db_name
+ self.migrate_to_version_or_latest(version)
+ finally:
+ self.remove_test_db()
+ print 'Test finished successfully'
USAGE = """\
@@ -282,37 +282,37 @@
def main():
- parser = OptionParser()
- parser.add_option("-d", "--database",
- help="which database to act on",
- dest="database")
- parser.add_option("-a", "--action", help="what action to perform",
- dest="action")
- (options, args) = parser.parse_args()
- manager = MigrationManager(options.database)
-
- if len(args) > 0:
- if len(args) > 1:
- version = int(args[1])
- else:
- version = None
- if args[0] == 'sync':
- manager.do_sync_db(version)
- elif args[0] == 'test':
- manager.test_sync_db(version)
- elif args[0] == 'simulate':
- manager.simulate_sync_db(version)
- elif args[0] == 'safesync':
- print 'Simluating migration'
- manager.simulate_sync_db(version)
- print 'Performing real migration'
- manager.do_sync_db(version)
- else:
- print USAGE
- return
-
- print USAGE
+ parser = OptionParser()
+ parser.add_option("-d", "--database",
+ help="which database to act on",
+ dest="database")
+ parser.add_option("-a", "--action", help="what action to perform",
+ dest="action")
+ (options, args) = parser.parse_args()
+ manager = MigrationManager(options.database)
+
+ if len(args) > 0:
+ if len(args) > 1:
+ version = int(args[1])
+ else:
+ version = None
+ if args[0] == 'sync':
+ manager.do_sync_db(version)
+ elif args[0] == 'test':
+ manager.test_sync_db(version)
+ elif args[0] == 'simulate':
+ manager.simulate_sync_db(version)
+ elif args[0] == 'safesync':
+ print 'Simluating migration'
+ manager.simulate_sync_db(version)
+ print 'Performing real migration'
+ manager.do_sync_db(version)
+ else:
+ print USAGE
+ return
+
+ print USAGE
if __name__ == '__main__':
- main()
+ main()
diff --git a/migrate/migrate_unittest.py b/migrate/migrate_unittest.py
index ec4ece1..d33dccd 100644
--- a/migrate/migrate_unittest.py
+++ b/migrate/migrate_unittest.py
@@ -13,140 +13,140 @@
NUM_MIGRATIONS = 3
class DummyMigration(object):
- """\
- Dummy migration class that records all migrations done in a class
- varaible.
- """
+ """\
+ Dummy migration class that records all migrations done in a class
+ varaible.
+ """
- migrations_done = []
+ migrations_done = []
- def __init__(self, version):
- self.version = version
- self.name = '%03d_test' % version
+ def __init__(self, version):
+ self.version = version
+ self.name = '%03d_test' % version
- @classmethod
- def get_migrations_done(cls):
- return cls.migrations_done
+ @classmethod
+ def get_migrations_done(cls):
+ return cls.migrations_done
- @classmethod
- def clear_migrations_done(cls):
- cls.migrations_done = []
+ @classmethod
+ def clear_migrations_done(cls):
+ cls.migrations_done = []
- @classmethod
- def do_migration(cls, version, direction):
- cls.migrations_done.append((version, direction))
+ @classmethod
+ def do_migration(cls, version, direction):
+ cls.migrations_done.append((version, direction))
- def migrate_up(self, manager):
- self.do_migration(self.version, 'up')
- if self.version == 1:
- manager.create_migrate_table()
+ def migrate_up(self, manager):
+ self.do_migration(self.version, 'up')
+ if self.version == 1:
+ manager.create_migrate_table()
- def migrate_down(self, manager):
- self.do_migration(self.version, 'down')
+ def migrate_down(self, manager):
+ self.do_migration(self.version, 'down')
MIGRATIONS = [DummyMigration(n) for n in xrange(1, NUM_MIGRATIONS + 1)]
class TestableMigrationManager(migrate.MigrationManager):
- def __init__(self, database, migrations_dir=None):
- self.database = database
- self.migrations_dir = migrations_dir
- self.db_host = None
- self.db_name = None
- self.username = None
- self.password = None
+ def __init__(self, database, migrations_dir=None):
+ self.database = database
+ self.migrations_dir = migrations_dir
+ self.db_host = None
+ self.db_name = None
+ self.username = None
+ self.password = None
- def read_db_info(self):
- migrate.MigrationManager.read_db_info(self)
- self.db_name = 'test_' + self.db_name
+ def read_db_info(self):
+ migrate.MigrationManager.read_db_info(self)
+ self.db_name = 'test_' + self.db_name
- def get_migrations(self, minimum_version=None, maximum_version=None):
- minimum_version = minimum_version or 1
- maximum_version = maximum_version or len(MIGRATIONS)
- return MIGRATIONS[minimum_version-1:maximum_version]
+ def get_migrations(self, minimum_version=None, maximum_version=None):
+ minimum_version = minimum_version or 1
+ maximum_version = maximum_version or len(MIGRATIONS)
+ return MIGRATIONS[minimum_version-1:maximum_version]
class MigrateManagerTest(unittest.TestCase):
- config = global_config.global_config
- host = config.get_config_value(CONFIG_DB, 'host')
- db_name = 'test_' + config.get_config_value(CONFIG_DB, 'database')
- user = config.get_config_value(CONFIG_DB, 'user')
- password = config.get_config_value(CONFIG_DB, 'password')
+ config = global_config.global_config
+ host = config.get_config_value(CONFIG_DB, 'host')
+ db_name = 'test_' + config.get_config_value(CONFIG_DB, 'database')
+ user = config.get_config_value(CONFIG_DB, 'user')
+ password = config.get_config_value(CONFIG_DB, 'password')
- def do_sql(self, sql):
- self.con = MySQLdb.connect(host=self.host, user=self.user,
- passwd=self.password)
- self.con.autocommit(True)
- self.cur = self.con.cursor()
- try:
- self.cur.execute(sql)
- finally:
- self.con.close()
+ def do_sql(self, sql):
+ self.con = MySQLdb.connect(host=self.host, user=self.user,
+ passwd=self.password)
+ self.con.autocommit(True)
+ self.cur = self.con.cursor()
+ try:
+ self.cur.execute(sql)
+ finally:
+ self.con.close()
- def remove_db(self):
- self.do_sql('DROP DATABASE ' + self.db_name)
+ def remove_db(self):
+ self.do_sql('DROP DATABASE ' + self.db_name)
- def setUp(self):
- self.do_sql('CREATE DATABASE ' + self.db_name)
- try:
- self.manager = TestableMigrationManager(CONFIG_DB)
- except MySQLdb.OperationalError:
- self.remove_db()
- raise
- DummyMigration.clear_migrations_done()
+ def setUp(self):
+ self.do_sql('CREATE DATABASE ' + self.db_name)
+ try:
+ self.manager = TestableMigrationManager(CONFIG_DB)
+ except MySQLdb.OperationalError:
+ self.remove_db()
+ raise
+ DummyMigration.clear_migrations_done()
- def tearDown(self):
- self.remove_db()
+ def tearDown(self):
+ self.remove_db()
- def test_sync(self):
- self.manager.do_sync_db()
- self.assertEquals(self.manager.get_db_version(), NUM_MIGRATIONS)
- self.assertEquals(DummyMigration.get_migrations_done(),
- [(1, 'up'), (2, 'up'), (3, 'up')])
+ def test_sync(self):
+ self.manager.do_sync_db()
+ self.assertEquals(self.manager.get_db_version(), NUM_MIGRATIONS)
+ self.assertEquals(DummyMigration.get_migrations_done(),
+ [(1, 'up'), (2, 'up'), (3, 'up')])
- DummyMigration.clear_migrations_done()
- self.manager.do_sync_db(0)
- self.assertEquals(self.manager.get_db_version(), 0)
- self.assertEquals(DummyMigration.get_migrations_done(),
- [(3, 'down'), (2, 'down'), (1, 'down')])
+ DummyMigration.clear_migrations_done()
+ self.manager.do_sync_db(0)
+ self.assertEquals(self.manager.get_db_version(), 0)
+ self.assertEquals(DummyMigration.get_migrations_done(),
+ [(3, 'down'), (2, 'down'), (1, 'down')])
- def test_sync_one_by_one(self):
- for version in xrange(1, NUM_MIGRATIONS + 1):
- self.manager.do_sync_db(version)
- self.assertEquals(self.manager.get_db_version(),
- version)
- self.assertEquals(
- DummyMigration.get_migrations_done()[-1],
- (version, 'up'))
+ def test_sync_one_by_one(self):
+ for version in xrange(1, NUM_MIGRATIONS + 1):
+ self.manager.do_sync_db(version)
+ self.assertEquals(self.manager.get_db_version(),
+ version)
+ self.assertEquals(
+ DummyMigration.get_migrations_done()[-1],
+ (version, 'up'))
- for version in xrange(NUM_MIGRATIONS - 1, -1, -1):
- self.manager.do_sync_db(version)
- self.assertEquals(self.manager.get_db_version(),
- version)
- self.assertEquals(
- DummyMigration.get_migrations_done()[-1],
- (version + 1, 'down'))
+ for version in xrange(NUM_MIGRATIONS - 1, -1, -1):
+ self.manager.do_sync_db(version)
+ self.assertEquals(self.manager.get_db_version(),
+ version)
+ self.assertEquals(
+ DummyMigration.get_migrations_done()[-1],
+ (version + 1, 'down'))
- def test_null_sync(self):
- self.manager.do_sync_db()
- DummyMigration.clear_migrations_done()
- self.manager.do_sync_db()
- self.assertEquals(DummyMigration.get_migrations_done(), [])
+ def test_null_sync(self):
+ self.manager.do_sync_db()
+ DummyMigration.clear_migrations_done()
+ self.manager.do_sync_db()
+ self.assertEquals(DummyMigration.get_migrations_done(), [])
if __name__ == '__main__':
- unittest.main()
+ unittest.main()
diff --git a/mirror/mirror b/mirror/mirror
index 31be530..16c7701 100755
--- a/mirror/mirror
+++ b/mirror/mirror
@@ -6,12 +6,12 @@
excludes = ('2.6.0-test*/', 'broken-out/', '*.sign', '*.gz')
default_clients_path = os.path.expanduser(
- os.path.join('~','.autotest_mirror_clients'))
+ os.path.join('~','.autotest_mirror_clients'))
default_config_path = os.path.expanduser(
- os.path.join ('~', '.autotest_mirror_config'))
+ os.path.join ('~', '.autotest_mirror_config'))
options = optparse.Values(defaults={'email': None,
- 'clients': default_clients_path,
- 'config': default_config_path})
+ 'clients': default_clients_path,
+ 'config': default_config_path})
# This counter gets appended and incremented for all jobs created
# this ensures that no two jobs of the same name are created during
@@ -25,242 +25,242 @@
"""
AUTOTEST_WRAPPER = """def step_init():
- job.next_step([step_test])
- testkernel = job.kernel(kernel)
+ job.next_step([step_test])
+ testkernel = job.kernel(kernel)
- if patches:
- testkernel.patch(*patches)
- if config:
- testkernel.config(config)
- else:
- testkernel.config('', None, True)
- testkernel.build()
- testkernel.boot()
+ if patches:
+ testkernel.patch(*patches)
+ if config:
+ testkernel.config(config)
+ else:
+ testkernel.config('', None, True)
+ testkernel.build()
+ testkernel.boot()
def step_test():"""
def test_kernel(client, control, kernel, config, patches=None):
- """Creates a control file for testing
+ """Creates a control file for testing
- Args:
- client: A str of the autoteset client hostname
- control: A str filename of the control file to wrap as a
- kernel test or an open file to the same
- kernel: A str of the kernel version (i.e. x.x.xx)
- config: A str filename to the kernel config on the client
- patches: A list of patches to apply during kernel build
+ Args:
+ client: A str of the autoteset client hostname
+ control: A str filename of the control file to wrap as a
+ kernel test or an open file to the same
+ kernel: A str of the kernel version (i.e. x.x.xx)
+ config: A str filename to the kernel config on the client
+ patches: A list of patches to apply during kernel build
- Returns:
- The path to the new control file which may not be needed
- since the control file will be created in the host's
- queue automatically
- """
+ Returns:
+ The path to the new control file which may not be needed
+ since the control file will be created in the host's
+ queue automatically
+ """
- is_autoserv_ctl = control.endswith('.srv')
+ is_autoserv_ctl = control.endswith('.srv')
- # Open a temp file to create the generated control file in
- tmpfile = tempfile.mktemp()
- c = open(tmpfile, 'w')
+ # Open a temp file to create the generated control file in
+ tmpfile = tempfile.mktemp()
+ c = open(tmpfile, 'w')
- print >> c, PREAMBLE % tuple([repr(s) for s in (kernel, config, patches)])
+ print >> c, PREAMBLE % tuple([repr(s) for s in (kernel, config, patches)])
- if not is_autoserv_ctl:
- print >> c, AUTOTEST_WRAPPER
+ if not is_autoserv_ctl:
+ print >> c, AUTOTEST_WRAPPER
- # Open the basis control file and pull its contents into this one
- control = open(os.path.expanduser(control), "r")
+ # Open the basis control file and pull its contents into this one
+ control = open(os.path.expanduser(control), "r")
- # If is an AT file then we need to indent to match wrapper
- # function level indentation, srv files don't need this indentation
- indent = ('\t', '')[is_autoserv_ctl]
- for line in control:
- print >> c, "%s%s" % (indent, line.rstrip())
- c.close()
+ # If is an AT file then we need to indent to match wrapper
+ # function level indentation, srv files don't need this indentation
+ indent = ('\t', '')[is_autoserv_ctl]
+ for line in control:
+ print >> c, "%s%s" % (indent, line.rstrip())
+ c.close()
- # Create a name for the queued file
- # if we came from an .srv file create an .srv file
- extension = ('', '.srv')[is_autoserv_ctl]
- global counter
- output = os.path.join(options.queuedir, client, "%s-%d%.3d%s" % (kernel, \
- int(time.time()), counter, extension))
- counter = (counter + 1) % 1000
- shutil.move(tmpfile, output)
- return output
+ # Create a name for the queued file
+ # if we came from an .srv file create an .srv file
+ extension = ('', '.srv')[is_autoserv_ctl]
+ global counter
+ output = os.path.join(options.queuedir, client, "%s-%d%.3d%s" % (kernel, \
+ int(time.time()), counter, extension))
+ counter = (counter + 1) % 1000
+ shutil.move(tmpfile, output)
+ return output
def load_conf(conf_path):
- """Loads a configuration file in the form
- hostname1.domain.tla /path/to/control ~/another_control
- hostname2.domain.tla ~/another_control
+ """Loads a configuration file in the form
+ hostname1.domain.tla /path/to/control ~/another_control
+ hostname2.domain.tla ~/another_control
- Returns:
- A dictionary keyed on hostname storing the args
- Lines beginning with # are stripped from the conf file
- """
- machine_controls = {}
- if not os.path.exists(conf_path):
- return machine_controls
- conf_file = open(conf_path)
- for line in conf_file.readlines():
- if line.startswith('#'):
- continue
- elts = line.split()
- # Ignore lines that aren't string doubles or more
- if len(elts) < 2:
- continue
- control_file_paths = [os.path.expanduser(e) for e in elts[1:]]
- machine_controls[elts[0]] = control_file_paths
- return machine_controls
+ Returns:
+ A dictionary keyed on hostname storing the args
+ Lines beginning with # are stripped from the conf file
+ """
+ machine_controls = {}
+ if not os.path.exists(conf_path):
+ return machine_controls
+ conf_file = open(conf_path)
+ for line in conf_file.readlines():
+ if line.startswith('#'):
+ continue
+ elts = line.split()
+ # Ignore lines that aren't string doubles or more
+ if len(elts) < 2:
+ continue
+ control_file_paths = [os.path.expanduser(e) for e in elts[1:]]
+ machine_controls[elts[0]] = control_file_paths
+ return machine_controls
def mail(from_address, to_addresses, subject, message_text):
- # if passed a string for the to_addresses convert it to a tuple
- if type(to_addresses) is str:
- to_addresses = (to_addresses,)
+ # if passed a string for the to_addresses convert it to a tuple
+ if type(to_addresses) is str:
+ to_addresses = (to_addresses,)
- message = email.Message.Message()
- message["To"] = ", ".join(to_addresses)
- message["From"] = from_address
- message["Subject"] = subject
- message.set_payload(message_text)
+ message = email.Message.Message()
+ message["To"] = ", ".join(to_addresses)
+ message["From"] = from_address
+ message["Subject"] = subject
+ message.set_payload(message_text)
- try:
- sendmail(message.as_string())
- except SendmailException, e:
- server = smtplib.SMTP("localhost")
- server.sendmail(from_address, to_addresses, message.as_string())
- server.quit()
+ try:
+ sendmail(message.as_string())
+ except SendmailException, e:
+ server = smtplib.SMTP("localhost")
+ server.sendmail(from_address, to_addresses, message.as_string())
+ server.quit()
MAIL = "sendmail"
class SendmailException(Exception):
- pass
+ pass
def sendmail(message):
- """Send an email using sendmail"""
- # open a pipe to the mail program and
- # write the data to the pipe
- p = os.popen("%s -t" % MAIL, 'w')
- p.write(message)
- exitcode = p.close()
- if exitcode:
- raise SendmailException("Exit code: %s" % exitcode)
+ """Send an email using sendmail"""
+ # open a pipe to the mail program and
+ # write the data to the pipe
+ p = os.popen("%s -t" % MAIL, 'w')
+ p.write(message)
+ exitcode = p.close()
+ if exitcode:
+ raise SendmailException("Exit code: %s" % exitcode)
def send_kernel_mail(kernel_list):
- if not options.email:
- return
- if len(kernel_list) < 1:
- return
- message = "\n".join(kernel_list)
- message = "Testing new kernel releases:\n%s" % message
+ if not options.email:
+ return
+ if len(kernel_list) < 1:
+ return
+ message = "\n".join(kernel_list)
+ message = "Testing new kernel releases:\n%s" % message
- mail("autotest-server@localhost", options.email, \
- "autotest new kernel notification", message)
+ mail("autotest-server@localhost", options.email, \
+ "autotest new kernel notification", message)
encode_sep = re.compile('(\D+)')
def kver_encode(version):
- if 'rc' not in version:
- version += '-rc99'
- if 'git' not in version:
- version += '-git99'
- bits = encode_sep.split(version)
- for n in range(0, len(bits), 2):
- if len(bits[n]) < 2:
- bits[n] = '0' + bits[n]
- return ''.join(bits)
+ if 'rc' not in version:
+ version += '-rc99'
+ if 'git' not in version:
+ version += '-git99'
+ bits = encode_sep.split(version)
+ for n in range(0, len(bits), 2):
+ if len(bits[n]) < 2:
+ bits[n] = '0' + bits[n]
+ return ''.join(bits)
def kver_cmp(a, b):
- a, b = kver_encode(a), kver_encode(b)
- return cmp(a, b)
+ a, b = kver_encode(a), kver_encode(b)
+ return cmp(a, b)
def closest_kver_leq(klist, kver):
- """Return the closest kernel ver in the list that is leq kver unless
- kver is the lowest, in which case return the lowest in klist"""
- if kver in klist:
- return kver
- l = list(klist)
- l.append(kver)
- l.sort(cmp=kver_cmp)
- i = l.index(kver)
- if i == 0:
- return l[1]
- return l[i - 1]
+ """Return the closest kernel ver in the list that is leq kver unless
+ kver is the lowest, in which case return the lowest in klist"""
+ if kver in klist:
+ return kver
+ l = list(klist)
+ l.append(kver)
+ l.sort(cmp=kver_cmp)
+ i = l.index(kver)
+ if i == 0:
+ return l[1]
+ return l[i - 1]
def perform_client_tests(kernel_list):
- machines = load_conf(options.clients)
- for kernel in kernel_list:
- # Get a list of all the machines available for testing
- # and the tests that each one is to execute
- for machine, controls in machines.items():
- config_paths = load_conf(os.path.join(options.config,
- machine))
- config = '/boot/config'
- if len(config_paths) > 0:
- kvers = config_paths.keys()
- close = closest_kver_leq(kvers,
- kernel)
- config = config_paths[close][0]
- for control in controls:
- test_kernel(machine,
- control,
- kernel,
- config)
+ machines = load_conf(options.clients)
+ for kernel in kernel_list:
+ # Get a list of all the machines available for testing
+ # and the tests that each one is to execute
+ for machine, controls in machines.items():
+ config_paths = load_conf(os.path.join(options.config,
+ machine))
+ config = '/boot/config'
+ if len(config_paths) > 0:
+ kvers = config_paths.keys()
+ close = closest_kver_leq(kvers,
+ kernel)
+ config = config_paths[close][0]
+ for control in controls:
+ test_kernel(machine,
+ control,
+ kernel,
+ config)
def main():
- """Performs an rsync of kernel.org, sends email on new releases
- and starts testing on them
- """
- source = 'rsync://rsync.kernel.org/pub/linux/kernel'
- mirror = rsync.rsync(source, options.target, excludes)
+ """Performs an rsync of kernel.org, sends email on new releases
+ and starts testing on them
+ """
+ source = 'rsync://rsync.kernel.org/pub/linux/kernel'
+ mirror = rsync.rsync(source, options.target, excludes)
- mirror.sync('v2.6/patch-2.6.*.bz2', 'kernel/v2.6')
- # for some reason 'linux-2.6.[0-9]*.tar.bz2' doesn't work
- mirror.sync('v2.6/linux-2.6.[0-9].tar.bz2', 'kernel/v2.6')
- mirror.sync('v2.6/linux-2.6.[0-9][0-9].tar.bz2', 'kernel/v2.6')
- mirror.sync('v2.6/testing/patch*.bz2', 'kernel/v2.6/testing')
- mirror.sync('v2.6/snapshots/*.bz2', 'kernel/v2.6/snapshots')
- mirror.sync('people/akpm/patches/2.6/*', 'akpm')
+ mirror.sync('v2.6/patch-2.6.*.bz2', 'kernel/v2.6')
+ # for some reason 'linux-2.6.[0-9]*.tar.bz2' doesn't work
+ mirror.sync('v2.6/linux-2.6.[0-9].tar.bz2', 'kernel/v2.6')
+ mirror.sync('v2.6/linux-2.6.[0-9][0-9].tar.bz2', 'kernel/v2.6')
+ mirror.sync('v2.6/testing/patch*.bz2', 'kernel/v2.6/testing')
+ mirror.sync('v2.6/snapshots/*.bz2', 'kernel/v2.6/snapshots')
+ mirror.sync('people/akpm/patches/2.6/*', 'akpm')
- trig = trigger.Trigger()
+ trig = trigger.Trigger()
- trig.add_action(send_kernel_mail)
- trig.add_action(perform_client_tests)
+ trig.add_action(send_kernel_mail)
+ trig.add_action(perform_client_tests)
- trig.scan(mirror.tmpfile)
+ trig.scan(mirror.tmpfile)
if __name__ == '__main__':
- usage = "mirror [options] <mirrordir> <queuedir>"
- parser = optparse.OptionParser(usage=usage)
- parser.add_option("-e", "--email", dest="email",
- help="Email address to alert for new kernels",
- metavar="EMAIL")
- parser.add_option("-c", "--clients", dest="clients",
- help="Location of the file that describes which \
- control file to run on which clients; defaults \
- to %s" % default_clients_path, \
- metavar="CLIENTS", \
- default=default_clients_path)
- parser.add_option("-f", "--config", dest="config",
- help="Location of the directory that describes which \
- config files to use on which clients; defaults \
- to %s" % default_config_path,
- metavar="CONFIG",
- default=default_config_path)
- (options, args) = parser.parse_args()
- if len(args) < 2:
- parser.print_help()
- parser.error("A mirrordir and a queuedir must be specified")
- options.target = args[0]
- options.queuedir = args[1]
+ usage = "mirror [options] <mirrordir> <queuedir>"
+ parser = optparse.OptionParser(usage=usage)
+ parser.add_option("-e", "--email", dest="email",
+ help="Email address to alert for new kernels",
+ metavar="EMAIL")
+ parser.add_option("-c", "--clients", dest="clients",
+ help="Location of the file that describes which \
+ control file to run on which clients; defaults \
+ to %s" % default_clients_path, \
+ metavar="CLIENTS", \
+ default=default_clients_path)
+ parser.add_option("-f", "--config", dest="config",
+ help="Location of the directory that describes which \
+ config files to use on which clients; defaults \
+ to %s" % default_config_path,
+ metavar="CONFIG",
+ default=default_config_path)
+ (options, args) = parser.parse_args()
+ if len(args) < 2:
+ parser.print_help()
+ parser.error("A mirrordir and a queuedir must be specified")
+ options.target = args[0]
+ options.queuedir = args[1]
- main()
+ main()
diff --git a/mirror/rsync.py b/mirror/rsync.py
index 0133cb6..baa643e 100755
--- a/mirror/rsync.py
+++ b/mirror/rsync.py
@@ -2,31 +2,31 @@
import os
class rsync:
- command = '/usr/bin/rsync -rltvz'
+ command = '/usr/bin/rsync -rltvz'
- def __init__(self, prefix, target, excludes = []):
- if not os.path.isdir(target):
- os.makedirs(target)
- self.prefix = prefix
- self.target = target
- # Have to use a tmpfile rather than a pipe, else we could
- # trigger from a file that's still only partially mirrored
- self.tmpfile = '/tmp/mirror.%d' % os.getpid()
- if os.path.exists(self.tmpfile):
- os.remove(self.tmpfile)
- self.exclude = ' '.join(['--exclude ' + x for x in excludes])
+ def __init__(self, prefix, target, excludes = []):
+ if not os.path.isdir(target):
+ os.makedirs(target)
+ self.prefix = prefix
+ self.target = target
+ # Have to use a tmpfile rather than a pipe, else we could
+ # trigger from a file that's still only partially mirrored
+ self.tmpfile = '/tmp/mirror.%d' % os.getpid()
+ if os.path.exists(self.tmpfile):
+ os.remove(self.tmpfile)
+ self.exclude = ' '.join(['--exclude ' + x for x in excludes])
- def __del__(self):
- os.remove(self.tmpfile)
+ def __del__(self):
+ os.remove(self.tmpfile)
- def sync(self, src, dest):
- os.chdir(self.target)
- if not os.path.isdir(dest):
- os.makedirs(dest)
- src = os.path.join(self.prefix, src)
- cmd = self.command + ' %s "%s" "%s"' % (self.exclude, src, dest)
- # print cmd + ' >> %s 2>&1' % self.tmpfile
- if os.system(cmd + ' >> %s 2>&1' % self.tmpfile):
- raise 'rsync command failed'
+ def sync(self, src, dest):
+ os.chdir(self.target)
+ if not os.path.isdir(dest):
+ os.makedirs(dest)
+ src = os.path.join(self.prefix, src)
+ cmd = self.command + ' %s "%s" "%s"' % (self.exclude, src, dest)
+ # print cmd + ' >> %s 2>&1' % self.tmpfile
+ if os.system(cmd + ' >> %s 2>&1' % self.tmpfile):
+ raise 'rsync command failed'
diff --git a/mirror/trigger.py b/mirror/trigger.py
index cccef7c..562ec05 100755
--- a/mirror/trigger.py
+++ b/mirror/trigger.py
@@ -5,55 +5,55 @@
# legitimate releases. Return the simplified triggers for each of those files.
matches = (
- # The major tarballs
- r'linux-(2\.6\.\d+)\.tar\.bz2',
- # Stable releases
- r'patch-(2\.6\.\d+\.\d+)\.bz2',
- # -rc releases
- r'patch-(2\.6\.\d+-rc\d+)\.bz2',
- # -git releases
- r'patch-(2\.6\.\d+(-rc\d+)?-git\d+).bz2',
- # -mm tree
- r'(2\.6\.\d+(-rc\d+)?-mm\d+)\.bz2',
- )
+ # The major tarballs
+ r'linux-(2\.6\.\d+)\.tar\.bz2',
+ # Stable releases
+ r'patch-(2\.6\.\d+\.\d+)\.bz2',
+ # -rc releases
+ r'patch-(2\.6\.\d+-rc\d+)\.bz2',
+ # -git releases
+ r'patch-(2\.6\.\d+(-rc\d+)?-git\d+).bz2',
+ # -mm tree
+ r'(2\.6\.\d+(-rc\d+)?-mm\d+)\.bz2',
+ )
compiled_matches = [re.compile(r) for r in matches]
class Trigger(object):
- def __init__(self):
- self.__actions = []
+ def __init__(self):
+ self.__actions = []
- def __re_scan(self, pattern, line):
- """
- First check to see whether the pattern matches.
- (eg. Does it match "linux-2.6.\d.tar.bz2" ?)
- Then we strip out the actual trigger itself from that,
- and return it.
- (eg. return "2.6.\d")
- Note that the pattern uses match,
- so you need the whole filename
- """
- match = pattern.match(line)
- if match:
- return match.group(1)
- else:
- return None
+ def __re_scan(self, pattern, line):
+ """
+ First check to see whether the pattern matches.
+ (eg. Does it match "linux-2.6.\d.tar.bz2" ?)
+ Then we strip out the actual trigger itself from that,
+ and return it.
+ (eg. return "2.6.\d")
+ Note that the pattern uses match,
+ so you need the whole filename
+ """
+ match = pattern.match(line)
+ if match:
+ return match.group(1)
+ else:
+ return None
- def scan(self, input_file):
- triggers = []
- for line in open(input_file, 'r').readlines():
- for pattern in compiled_matches:
- filename = os.path.basename(line)
- t = self.__re_scan(pattern, filename)
- if t:
- triggers.append(t)
+ def scan(self, input_file):
+ triggers = []
+ for line in open(input_file, 'r').readlines():
+ for pattern in compiled_matches:
+ filename = os.path.basename(line)
+ t = self.__re_scan(pattern, filename)
+ if t:
+ triggers.append(t)
- # Call each of the actions and pass in the kernel list
- for action in self.__actions:
- action(triggers)
+ # Call each of the actions and pass in the kernel list
+ for action in self.__actions:
+ action(triggers)
- def add_action(self, func):
- self.__actions.append(func)
+ def add_action(self, func):
+ self.__actions.append(func)
diff --git a/scheduler/monitor_db.py b/scheduler/monitor_db.py
index 31dba1f..fd3e3e9 100644
--- a/scheduler/monitor_db.py
+++ b/scheduler/monitor_db.py
@@ -15,12 +15,12 @@
AUTOTEST_PATH = os.path.join(os.path.dirname(__file__), '..')
if os.environ.has_key('AUTOTEST_DIR'):
- AUTOTEST_PATH = os.environ['AUTOTEST_DIR']
+ AUTOTEST_PATH = os.environ['AUTOTEST_DIR']
AUTOTEST_SERVER_DIR = os.path.join(AUTOTEST_PATH, 'server')
AUTOTEST_TKO_DIR = os.path.join(AUTOTEST_PATH, 'tko')
if AUTOTEST_SERVER_DIR not in sys.path:
- sys.path.insert(0, AUTOTEST_SERVER_DIR)
+ sys.path.insert(0, AUTOTEST_SERVER_DIR)
AUTOSERV_PID_FILE = '.autoserv_execute'
# how long to wait for autoserv to write a pidfile
@@ -35,1744 +35,1744 @@
def main():
- usage = 'usage: %prog [options] results_dir'
+ usage = 'usage: %prog [options] results_dir'
- parser = optparse.OptionParser(usage)
- parser.add_option('--recover-hosts', help='Try to recover dead hosts',
- action='store_true')
- parser.add_option('--logfile', help='Set a log file that all stdout ' +
- 'should be redirected to. Stderr will go to this ' +
- 'file + ".err"')
- parser.add_option('--test', help='Indicate that scheduler is under ' +
- 'test and should use dummy autoserv and no parsing',
- action='store_true')
- (options, args) = parser.parse_args()
- if len(args) != 1:
- parser.print_usage()
- return
-
- global RESULTS_DIR
- RESULTS_DIR = args[0]
+ parser = optparse.OptionParser(usage)
+ parser.add_option('--recover-hosts', help='Try to recover dead hosts',
+ action='store_true')
+ parser.add_option('--logfile', help='Set a log file that all stdout ' +
+ 'should be redirected to. Stderr will go to this ' +
+ 'file + ".err"')
+ parser.add_option('--test', help='Indicate that scheduler is under ' +
+ 'test and should use dummy autoserv and no parsing',
+ action='store_true')
+ (options, args) = parser.parse_args()
+ if len(args) != 1:
+ parser.print_usage()
+ return
- # read in notify_email from global_config
- c = global_config.global_config
- global _notify_email
- val = c.get_config_value(_global_config_section, "notify_email")
- if val != "":
- _notify_email = val
-
- if options.test:
- global _autoserv_path
- _autoserv_path = 'autoserv_dummy'
- global _testing_mode
- _testing_mode = True
+ global RESULTS_DIR
+ RESULTS_DIR = args[0]
- init(options.logfile)
- dispatcher = Dispatcher()
- dispatcher.do_initial_recovery(recover_hosts=options.recover_hosts)
+ # read in notify_email from global_config
+ c = global_config.global_config
+ global _notify_email
+ val = c.get_config_value(_global_config_section, "notify_email")
+ if val != "":
+ _notify_email = val
- try:
- while not _shutdown:
- dispatcher.tick()
- time.sleep(20)
- except:
- log_stacktrace("Uncaught exception; terminating monitor_db")
+ if options.test:
+ global _autoserv_path
+ _autoserv_path = 'autoserv_dummy'
+ global _testing_mode
+ _testing_mode = True
- email_manager.send_queued_emails()
- _db.disconnect()
+ init(options.logfile)
+ dispatcher = Dispatcher()
+ dispatcher.do_initial_recovery(recover_hosts=options.recover_hosts)
+
+ try:
+ while not _shutdown:
+ dispatcher.tick()
+ time.sleep(20)
+ except:
+ log_stacktrace("Uncaught exception; terminating monitor_db")
+
+ email_manager.send_queued_emails()
+ _db.disconnect()
def handle_sigint(signum, frame):
- global _shutdown
- _shutdown = True
- print "Shutdown request received."
+ global _shutdown
+ _shutdown = True
+ print "Shutdown request received."
def init(logfile):
- if logfile:
- enable_logging(logfile)
- print "%s> dispatcher starting" % time.strftime("%X %x")
- print "My PID is %d" % os.getpid()
+ if logfile:
+ enable_logging(logfile)
+ print "%s> dispatcher starting" % time.strftime("%X %x")
+ print "My PID is %d" % os.getpid()
- os.environ['PATH'] = AUTOTEST_SERVER_DIR + ':' + os.environ['PATH']
- global _db
- _db = DatabaseConn()
- _db.connect()
+ os.environ['PATH'] = AUTOTEST_SERVER_DIR + ':' + os.environ['PATH']
+ global _db
+ _db = DatabaseConn()
+ _db.connect()
- print "Setting signal handler"
- signal.signal(signal.SIGINT, handle_sigint)
-
- print "Connected! Running..."
+ print "Setting signal handler"
+ signal.signal(signal.SIGINT, handle_sigint)
+
+ print "Connected! Running..."
def enable_logging(logfile):
- out_file = logfile
- err_file = "%s.err" % logfile
- print "Enabling logging to %s (%s)" % (out_file, err_file)
- out_fd = open(out_file, "a", buffering=0)
- err_fd = open(err_file, "a", buffering=0)
+ out_file = logfile
+ err_file = "%s.err" % logfile
+ print "Enabling logging to %s (%s)" % (out_file, err_file)
+ out_fd = open(out_file, "a", buffering=0)
+ err_fd = open(err_file, "a", buffering=0)
- os.dup2(out_fd.fileno(), sys.stdout.fileno())
- os.dup2(err_fd.fileno(), sys.stderr.fileno())
+ os.dup2(out_fd.fileno(), sys.stdout.fileno())
+ os.dup2(err_fd.fileno(), sys.stderr.fileno())
- sys.stdout = out_fd
- sys.stderr = err_fd
+ sys.stdout = out_fd
+ sys.stderr = err_fd
def queue_entries_to_abort():
- rows = _db.execute("""
- SELECT * FROM host_queue_entries WHERE status='Abort';
- """)
- qe = [HostQueueEntry(row=i) for i in rows]
- return qe
+ rows = _db.execute("""
+ SELECT * FROM host_queue_entries WHERE status='Abort';
+ """)
+ qe = [HostQueueEntry(row=i) for i in rows]
+ return qe
def remove_file_or_dir(path):
- if stat.S_ISDIR(os.stat(path).st_mode):
- # directory
- shutil.rmtree(path)
- else:
- # file
- os.remove(path)
+ if stat.S_ISDIR(os.stat(path).st_mode):
+ # directory
+ shutil.rmtree(path)
+ else:
+ # file
+ os.remove(path)
class DatabaseConn:
- def __init__(self):
- self.reconnect_wait = 20
- self.conn = None
- self.cur = None
+ def __init__(self):
+ self.reconnect_wait = 20
+ self.conn = None
+ self.cur = None
- import MySQLdb.converters
- self.convert_dict = MySQLdb.converters.conversions
- self.convert_dict.setdefault(bool, self.convert_boolean)
+ import MySQLdb.converters
+ self.convert_dict = MySQLdb.converters.conversions
+ self.convert_dict.setdefault(bool, self.convert_boolean)
- @staticmethod
- def convert_boolean(boolean, conversion_dict):
- 'Convert booleans to integer strings'
- return str(int(boolean))
+ @staticmethod
+ def convert_boolean(boolean, conversion_dict):
+ 'Convert booleans to integer strings'
+ return str(int(boolean))
- def connect(self, db_name=None):
- self.disconnect()
+ def connect(self, db_name=None):
+ self.disconnect()
- # get global config and parse for info
- c = global_config.global_config
- dbase = "AUTOTEST_WEB"
- db_host = c.get_config_value(dbase, "host")
- if db_name is None:
- db_name = c.get_config_value(dbase, "database")
+ # get global config and parse for info
+ c = global_config.global_config
+ dbase = "AUTOTEST_WEB"
+ db_host = c.get_config_value(dbase, "host")
+ if db_name is None:
+ db_name = c.get_config_value(dbase, "database")
- if _testing_mode:
- db_name = 'stresstest_autotest_web'
+ if _testing_mode:
+ db_name = 'stresstest_autotest_web'
- db_user = c.get_config_value(dbase, "user")
- db_pass = c.get_config_value(dbase, "password")
+ db_user = c.get_config_value(dbase, "user")
+ db_pass = c.get_config_value(dbase, "password")
- while not self.conn:
- try:
- self.conn = MySQLdb.connect(
- host=db_host, user=db_user, passwd=db_pass,
- db=db_name, conv=self.convert_dict)
+ while not self.conn:
+ try:
+ self.conn = MySQLdb.connect(
+ host=db_host, user=db_user, passwd=db_pass,
+ db=db_name, conv=self.convert_dict)
- self.conn.autocommit(True)
- self.cur = self.conn.cursor()
- except MySQLdb.OperationalError:
- traceback.print_exc()
- print "Can't connect to MYSQL; reconnecting"
- time.sleep(self.reconnect_wait)
- self.disconnect()
+ self.conn.autocommit(True)
+ self.cur = self.conn.cursor()
+ except MySQLdb.OperationalError:
+ traceback.print_exc()
+ print "Can't connect to MYSQL; reconnecting"
+ time.sleep(self.reconnect_wait)
+ self.disconnect()
- def disconnect(self):
- if self.conn:
- self.conn.close()
- self.conn = None
- self.cur = None
+ def disconnect(self):
+ if self.conn:
+ self.conn.close()
+ self.conn = None
+ self.cur = None
- def execute(self, *args, **dargs):
- while (True):
- try:
- self.cur.execute(*args, **dargs)
- return self.cur.fetchall()
- except MySQLdb.OperationalError:
- traceback.print_exc()
- print "MYSQL connection died; reconnecting"
- time.sleep(self.reconnect_wait)
- self.connect()
+ def execute(self, *args, **dargs):
+ while (True):
+ try:
+ self.cur.execute(*args, **dargs)
+ return self.cur.fetchall()
+ except MySQLdb.OperationalError:
+ traceback.print_exc()
+ print "MYSQL connection died; reconnecting"
+ time.sleep(self.reconnect_wait)
+ self.connect()
def generate_parse_command(results_dir, flags=""):
- parse = os.path.abspath(os.path.join(AUTOTEST_TKO_DIR, 'parse'))
- output = os.path.abspath(os.path.join(results_dir, '.parse.log'))
- cmd = "%s %s -r -o %s > %s 2>&1 &"
- return cmd % (parse, flags, results_dir, output)
+ parse = os.path.abspath(os.path.join(AUTOTEST_TKO_DIR, 'parse'))
+ output = os.path.abspath(os.path.join(results_dir, '.parse.log'))
+ cmd = "%s %s -r -o %s > %s 2>&1 &"
+ return cmd % (parse, flags, results_dir, output)
def parse_results(results_dir, flags=""):
- if _testing_mode:
- return
- os.system(generate_parse_command(results_dir, flags))
+ if _testing_mode:
+ return
+ os.system(generate_parse_command(results_dir, flags))
def log_stacktrace(reason):
- (type, value, tb) = sys.exc_info()
- str = "EXCEPTION: %s\n" % reason
- str += ''.join(traceback.format_exception(type, value, tb))
+ (type, value, tb) = sys.exc_info()
+ str = "EXCEPTION: %s\n" % reason
+ str += ''.join(traceback.format_exception(type, value, tb))
- sys.stderr.write("\n%s\n" % str)
- email_manager.enqueue_notify_email("monitor_db exception", str)
+ sys.stderr.write("\n%s\n" % str)
+ email_manager.enqueue_notify_email("monitor_db exception", str)
def get_proc_poll_fn(pid):
- proc_path = os.path.join('/proc', str(pid))
- def poll_fn():
- if os.path.exists(proc_path):
- return None
- return 0 # we can't get a real exit code
- return poll_fn
+ proc_path = os.path.join('/proc', str(pid))
+ def poll_fn():
+ if os.path.exists(proc_path):
+ return None
+ return 0 # we can't get a real exit code
+ return poll_fn
def kill_autoserv(pid, poll_fn=None):
- print 'killing', pid
- if poll_fn is None:
- poll_fn = get_proc_poll_fn(pid)
- if poll_fn() == None:
- os.kill(pid, signal.SIGCONT)
- os.kill(pid, signal.SIGTERM)
+ print 'killing', pid
+ if poll_fn is None:
+ poll_fn = get_proc_poll_fn(pid)
+ if poll_fn() == None:
+ os.kill(pid, signal.SIGCONT)
+ os.kill(pid, signal.SIGTERM)
class EmailNotificationManager(object):
- def __init__(self):
- self._emails = []
- # see os.getlogin() online docs
- self._sender = pwd.getpwuid(os.getuid())[0]
+ def __init__(self):
+ self._emails = []
+ # see os.getlogin() online docs
+ self._sender = pwd.getpwuid(os.getuid())[0]
- def enqueue_notify_email(self, subject, message):
- if not _notify_email:
- return
+ def enqueue_notify_email(self, subject, message):
+ if not _notify_email:
+ return
- body = 'Subject: ' + subject + '\n'
- body += "%s / %s / %s\n%s" % (socket.gethostname(),
- os.getpid(),
- time.strftime("%X %x"), message)
- self._emails.append(body)
+ body = 'Subject: ' + subject + '\n'
+ body += "%s / %s / %s\n%s" % (socket.gethostname(),
+ os.getpid(),
+ time.strftime("%X %x"), message)
+ self._emails.append(body)
- def send_queued_emails(self):
- if not self._emails:
- return
- subject = 'Scheduler notifications from ' + socket.gethostname()
- separator = '\n' + '-' * 40 + '\n'
- body = separator.join(self._emails)
- msg = "From: %s\nTo: %s\nSubject: %s\n\n%s" % (
- self._sender, _notify_email, subject, body)
+ def send_queued_emails(self):
+ if not self._emails:
+ return
+ subject = 'Scheduler notifications from ' + socket.gethostname()
+ separator = '\n' + '-' * 40 + '\n'
+ body = separator.join(self._emails)
+ msg = "From: %s\nTo: %s\nSubject: %s\n\n%s" % (
+ self._sender, _notify_email, subject, body)
- mailer = smtplib.SMTP('localhost')
- mailer.sendmail(self._sender, _notify_email, msg)
- mailer.quit()
- self._emails = []
+ mailer = smtplib.SMTP('localhost')
+ mailer.sendmail(self._sender, _notify_email, msg)
+ mailer.quit()
+ self._emails = []
email_manager = EmailNotificationManager()
class Dispatcher:
- autoserv_procs_cache = None
- max_running_agents = global_config.global_config.get_config_value(
- _global_config_section, 'max_running_jobs', type=int)
- max_jobs_started_per_cycle = (
- global_config.global_config.get_config_value(
- _global_config_section, 'max_jobs_started_per_cycle', type=int))
+ autoserv_procs_cache = None
+ max_running_agents = global_config.global_config.get_config_value(
+ _global_config_section, 'max_running_jobs', type=int)
+ max_jobs_started_per_cycle = (
+ global_config.global_config.get_config_value(
+ _global_config_section, 'max_jobs_started_per_cycle', type=int))
- def __init__(self):
- self._agents = []
+ def __init__(self):
+ self._agents = []
- def do_initial_recovery(self, recover_hosts=True):
- # always recover processes
- self._recover_processes()
+ def do_initial_recovery(self, recover_hosts=True):
+ # always recover processes
+ self._recover_processes()
- if recover_hosts:
- self._recover_hosts()
+ if recover_hosts:
+ self._recover_hosts()
- def tick(self):
- Dispatcher.autoserv_procs_cache = None
- self._find_aborting()
- self._schedule_new_jobs()
- self._handle_agents()
- self._clear_inactive_blocks()
- email_manager.send_queued_emails()
+ def tick(self):
+ Dispatcher.autoserv_procs_cache = None
+ self._find_aborting()
+ self._schedule_new_jobs()
+ self._handle_agents()
+ self._clear_inactive_blocks()
+ email_manager.send_queued_emails()
- def add_agent(self, agent):
- self._agents.append(agent)
- agent.dispatcher = self
-
- # Find agent corresponding to the specified queue_entry
- def get_agents(self, queue_entry):
- res_agents = []
- for agent in self._agents:
- if queue_entry.id in agent.queue_entry_ids:
- res_agents.append(agent)
- return res_agents
-
+ def add_agent(self, agent):
+ self._agents.append(agent)
+ agent.dispatcher = self
- def remove_agent(self, agent):
- self._agents.remove(agent)
+ # Find agent corresponding to the specified queue_entry
+ def get_agents(self, queue_entry):
+ res_agents = []
+ for agent in self._agents:
+ if queue_entry.id in agent.queue_entry_ids:
+ res_agents.append(agent)
+ return res_agents
- def num_started_agents(self):
- return len([agent for agent in self._agents
- if agent.is_started()])
+ def remove_agent(self, agent):
+ self._agents.remove(agent)
- @classmethod
- def find_autoservs(cls, orphans_only=False):
- """\
- Returns a dict mapping pids to command lines for root autoserv
- processes. If orphans_only=True, return only processes that
- have been orphaned (i.e. parent pid = 1).
- """
- if cls.autoserv_procs_cache is not None:
- return cls.autoserv_procs_cache
-
- proc = subprocess.Popen(
- ['/bin/ps', 'x', '-o', 'pid,pgid,ppid,comm,args'],
- stdout=subprocess.PIPE)
- # split each line into the four columns output by ps
- procs = [line.split(None, 4) for line in
- proc.communicate()[0].splitlines()]
- autoserv_procs = {}
- for proc in procs:
- # check ppid == 1 for orphans
- if orphans_only and proc[2] != 1:
- continue
- # only root autoserv processes have pgid == pid
- if (proc[3] == 'autoserv' and # comm
- proc[1] == proc[0]): # pgid == pid
- # map pid to args
- autoserv_procs[int(proc[0])] = proc[4]
- cls.autoserv_procs_cache = autoserv_procs
- return autoserv_procs
+ def num_started_agents(self):
+ return len([agent for agent in self._agents
+ if agent.is_started()])
- def recover_queue_entry(self, queue_entry, run_monitor):
- job = queue_entry.job
- if job.is_synchronous():
- all_queue_entries = job.get_host_queue_entries()
- else:
- all_queue_entries = [queue_entry]
- all_queue_entry_ids = [queue_entry.id for queue_entry
- in all_queue_entries]
- queue_task = RecoveryQueueTask(
- job=queue_entry.job,
- queue_entries=all_queue_entries,
- run_monitor=run_monitor)
- self.add_agent(Agent(tasks=[queue_task],
- queue_entry_ids=all_queue_entry_ids))
+ @classmethod
+ def find_autoservs(cls, orphans_only=False):
+ """\
+ Returns a dict mapping pids to command lines for root autoserv
+ processes. If orphans_only=True, return only processes that
+ have been orphaned (i.e. parent pid = 1).
+ """
+ if cls.autoserv_procs_cache is not None:
+ return cls.autoserv_procs_cache
+
+ proc = subprocess.Popen(
+ ['/bin/ps', 'x', '-o', 'pid,pgid,ppid,comm,args'],
+ stdout=subprocess.PIPE)
+ # split each line into the four columns output by ps
+ procs = [line.split(None, 4) for line in
+ proc.communicate()[0].splitlines()]
+ autoserv_procs = {}
+ for proc in procs:
+ # check ppid == 1 for orphans
+ if orphans_only and proc[2] != 1:
+ continue
+ # only root autoserv processes have pgid == pid
+ if (proc[3] == 'autoserv' and # comm
+ proc[1] == proc[0]): # pgid == pid
+ # map pid to args
+ autoserv_procs[int(proc[0])] = proc[4]
+ cls.autoserv_procs_cache = autoserv_procs
+ return autoserv_procs
- def _recover_processes(self):
- orphans = self.find_autoservs(orphans_only=True)
-
- # first, recover running queue entries
- rows = _db.execute("""SELECT * FROM host_queue_entries
- WHERE status = 'Running'""")
- queue_entries = [HostQueueEntry(row=i) for i in rows]
- requeue_entries = []
- recovered_entry_ids = set()
- for queue_entry in queue_entries:
- run_monitor = PidfileRunMonitor(
- queue_entry.results_dir())
- run_monitor.run()
- pid, exit_code = run_monitor.get_pidfile_info()
- if pid is None:
- # autoserv apparently never got run, so requeue
- requeue_entries.append(queue_entry)
- continue
- if queue_entry.id in recovered_entry_ids:
- # synchronous job we've already recovered
- continue
- print 'Recovering queue entry %d (pid %d)' % (
- queue_entry.id, pid)
- job = queue_entry.job
- if job.is_synchronous():
- for entry in job.get_host_queue_entries():
- assert entry.active
- recovered_entry_ids.add(entry.id)
- self.recover_queue_entry(queue_entry,
- run_monitor)
- orphans.pop(pid, None)
-
- # and requeue other active queue entries
- rows = _db.execute("""SELECT * FROM host_queue_entries
- WHERE active AND NOT complete
- AND status != 'Running'
- AND status != 'Pending'
- AND status != 'Abort'
- AND status != 'Aborting'""")
- queue_entries = [HostQueueEntry(row=i) for i in rows]
- for queue_entry in queue_entries + requeue_entries:
- print 'Requeuing running QE %d' % queue_entry.id
- queue_entry.clear_results_dir(dont_delete_files=True)
- queue_entry.requeue()
+ def recover_queue_entry(self, queue_entry, run_monitor):
+ job = queue_entry.job
+ if job.is_synchronous():
+ all_queue_entries = job.get_host_queue_entries()
+ else:
+ all_queue_entries = [queue_entry]
+ all_queue_entry_ids = [queue_entry.id for queue_entry
+ in all_queue_entries]
+ queue_task = RecoveryQueueTask(
+ job=queue_entry.job,
+ queue_entries=all_queue_entries,
+ run_monitor=run_monitor)
+ self.add_agent(Agent(tasks=[queue_task],
+ queue_entry_ids=all_queue_entry_ids))
- # now kill any remaining autoserv processes
- for pid in orphans.keys():
- print 'Killing orphan %d (%s)' % (pid, orphans[pid])
- kill_autoserv(pid)
+ def _recover_processes(self):
+ orphans = self.find_autoservs(orphans_only=True)
- # recover aborting tasks
- rebooting_host_ids = set()
- rows = _db.execute("""SELECT * FROM host_queue_entries
- WHERE status='Abort' or status='Aborting'""")
- queue_entries = [HostQueueEntry(row=i) for i in rows]
- for queue_entry in queue_entries:
- print 'Recovering aborting QE %d' % queue_entry.id
- queue_host = queue_entry.get_host()
- reboot_task = RebootTask(queue_host)
- verify_task = VerifyTask(host = queue_host)
- self.add_agent(Agent(tasks=[reboot_task,
- verify_task],
- queue_entry_ids=[queue_entry.id]))
- queue_entry.set_status('Aborted')
- # Secure the host from being picked up
- queue_host.set_status('Rebooting')
- rebooting_host_ids.add(queue_host.id)
+ # first, recover running queue entries
+ rows = _db.execute("""SELECT * FROM host_queue_entries
+ WHERE status = 'Running'""")
+ queue_entries = [HostQueueEntry(row=i) for i in rows]
+ requeue_entries = []
+ recovered_entry_ids = set()
+ for queue_entry in queue_entries:
+ run_monitor = PidfileRunMonitor(
+ queue_entry.results_dir())
+ run_monitor.run()
+ pid, exit_code = run_monitor.get_pidfile_info()
+ if pid is None:
+ # autoserv apparently never got run, so requeue
+ requeue_entries.append(queue_entry)
+ continue
+ if queue_entry.id in recovered_entry_ids:
+ # synchronous job we've already recovered
+ continue
+ print 'Recovering queue entry %d (pid %d)' % (
+ queue_entry.id, pid)
+ job = queue_entry.job
+ if job.is_synchronous():
+ for entry in job.get_host_queue_entries():
+ assert entry.active
+ recovered_entry_ids.add(entry.id)
+ self.recover_queue_entry(queue_entry,
+ run_monitor)
+ orphans.pop(pid, None)
- # reverify hosts that were in the middle of verify, repair or
- # reboot
- self._reverify_hosts_where("""(status = 'Repairing' OR
- status = 'Verifying' OR
- status = 'Rebooting')""",
- exclude_ids=rebooting_host_ids)
-
- # finally, recover "Running" hosts with no active queue entries,
- # although this should never happen
- message = ('Recovering running host %s - this probably '
- 'indicates a scheduler bug')
- self._reverify_hosts_where("""status = 'Running' AND
- id NOT IN (SELECT host_id
- FROM host_queue_entries
- WHERE active)""",
- print_message=message)
+ # and requeue other active queue entries
+ rows = _db.execute("""SELECT * FROM host_queue_entries
+ WHERE active AND NOT complete
+ AND status != 'Running'
+ AND status != 'Pending'
+ AND status != 'Abort'
+ AND status != 'Aborting'""")
+ queue_entries = [HostQueueEntry(row=i) for i in rows]
+ for queue_entry in queue_entries + requeue_entries:
+ print 'Requeuing running QE %d' % queue_entry.id
+ queue_entry.clear_results_dir(dont_delete_files=True)
+ queue_entry.requeue()
- def _reverify_hosts_where(self, where,
- print_message='Reverifying host %s',
- exclude_ids=set()):
- rows = _db.execute('SELECT * FROM hosts WHERE locked = 0 AND '
- 'invalid = 0 AND ' + where)
- hosts = [Host(row=i) for i in rows]
- for host in hosts:
- if host.id in exclude_ids:
- continue
- if print_message is not None:
- print print_message % host.hostname
- verify_task = VerifyTask(host = host)
- self.add_agent(Agent(tasks = [verify_task]))
+ # now kill any remaining autoserv processes
+ for pid in orphans.keys():
+ print 'Killing orphan %d (%s)' % (pid, orphans[pid])
+ kill_autoserv(pid)
+
+ # recover aborting tasks
+ rebooting_host_ids = set()
+ rows = _db.execute("""SELECT * FROM host_queue_entries
+ WHERE status='Abort' or status='Aborting'""")
+ queue_entries = [HostQueueEntry(row=i) for i in rows]
+ for queue_entry in queue_entries:
+ print 'Recovering aborting QE %d' % queue_entry.id
+ queue_host = queue_entry.get_host()
+ reboot_task = RebootTask(queue_host)
+ verify_task = VerifyTask(host = queue_host)
+ self.add_agent(Agent(tasks=[reboot_task,
+ verify_task],
+ queue_entry_ids=[queue_entry.id]))
+ queue_entry.set_status('Aborted')
+ # Secure the host from being picked up
+ queue_host.set_status('Rebooting')
+ rebooting_host_ids.add(queue_host.id)
+
+ # reverify hosts that were in the middle of verify, repair or
+ # reboot
+ self._reverify_hosts_where("""(status = 'Repairing' OR
+ status = 'Verifying' OR
+ status = 'Rebooting')""",
+ exclude_ids=rebooting_host_ids)
+
+ # finally, recover "Running" hosts with no active queue entries,
+ # although this should never happen
+ message = ('Recovering running host %s - this probably '
+ 'indicates a scheduler bug')
+ self._reverify_hosts_where("""status = 'Running' AND
+ id NOT IN (SELECT host_id
+ FROM host_queue_entries
+ WHERE active)""",
+ print_message=message)
- def _recover_hosts(self):
- # recover "Repair Failed" hosts
- message = 'Reverifying dead host %s'
- self._reverify_hosts_where("status = 'Repair Failed'",
- print_message=message)
+ def _reverify_hosts_where(self, where,
+ print_message='Reverifying host %s',
+ exclude_ids=set()):
+ rows = _db.execute('SELECT * FROM hosts WHERE locked = 0 AND '
+ 'invalid = 0 AND ' + where)
+ hosts = [Host(row=i) for i in rows]
+ for host in hosts:
+ if host.id in exclude_ids:
+ continue
+ if print_message is not None:
+ print print_message % host.hostname
+ verify_task = VerifyTask(host = host)
+ self.add_agent(Agent(tasks = [verify_task]))
- def _clear_inactive_blocks(self):
- """
- Clear out blocks for all completed jobs.
- """
- # this would be simpler using NOT IN (subquery), but MySQL
- # treats all IN subqueries as dependent, so this optimizes much
- # better
- _db.execute("""
- DELETE ihq FROM ineligible_host_queues ihq
- LEFT JOIN (SELECT job_id FROM host_queue_entries
- WHERE NOT complete) hqe
- USING (job_id) WHERE hqe.job_id IS NULL""")
+ def _recover_hosts(self):
+ # recover "Repair Failed" hosts
+ message = 'Reverifying dead host %s'
+ self._reverify_hosts_where("status = 'Repair Failed'",
+ print_message=message)
- def _extract_host_and_queue_entry(self, row):
- # each row contains host columns followed by host queue entry
- # columns
- num_host_cols = Host.num_cols()
- assert len(row) == num_host_cols + HostQueueEntry.num_cols()
- host = Host(row=row[:num_host_cols])
- queue_entry = HostQueueEntry(row=row[num_host_cols:])
- return host, queue_entry
+ def _clear_inactive_blocks(self):
+ """
+ Clear out blocks for all completed jobs.
+ """
+ # this would be simpler using NOT IN (subquery), but MySQL
+ # treats all IN subqueries as dependent, so this optimizes much
+ # better
+ _db.execute("""
+ DELETE ihq FROM ineligible_host_queues ihq
+ LEFT JOIN (SELECT job_id FROM host_queue_entries
+ WHERE NOT complete) hqe
+ USING (job_id) WHERE hqe.job_id IS NULL""")
- def _get_runnable_entries(self, extra_join='', extra_where=''):
- query = (
- 'SELECT DISTINCT h.*, queued_hqe.* FROM hosts h '
- # join with running entries
- """
- LEFT JOIN host_queue_entries AS active_hqe
- ON (h.id = active_hqe.host_id AND active_hqe.active)
- """ +
- extra_join +
- # exclude hosts with a running entry
- 'WHERE active_hqe.host_id IS NULL '
- # exclude locked, invalid, and non-Ready hosts
- """
- AND h.locked=false AND h.invalid=false
- AND (h.status IS null OR h.status='Ready')
- """)
- if extra_where:
- query += 'AND ' + extra_where + '\n'
- # respect priority, then sort by ID (most recent first)
- query += 'ORDER BY queued_hqe.priority DESC, queued_hqe.id'
-
- rows = _db.execute(query)
- return [self._extract_host_and_queue_entry(row) for row in rows]
+ def _extract_host_and_queue_entry(self, row):
+ # each row contains host columns followed by host queue entry
+ # columns
+ num_host_cols = Host.num_cols()
+ assert len(row) == num_host_cols + HostQueueEntry.num_cols()
+ host = Host(row=row[:num_host_cols])
+ queue_entry = HostQueueEntry(row=row[num_host_cols:])
+ return host, queue_entry
- def _get_runnable_nonmetahosts(self):
- # find queued HQEs scheduled directly against hosts
- queued_hqe_join = """
- INNER JOIN host_queue_entries AS queued_hqe
- ON (h.id = queued_hqe.host_id
- AND NOT queued_hqe.active AND NOT queued_hqe.complete)
- """
- return self._get_runnable_entries(queued_hqe_join)
+ def _get_runnable_entries(self, extra_join='', extra_where=''):
+ query = (
+ 'SELECT DISTINCT h.*, queued_hqe.* FROM hosts h '
+ # join with running entries
+ """
+ LEFT JOIN host_queue_entries AS active_hqe
+ ON (h.id = active_hqe.host_id AND active_hqe.active)
+ """ +
+ extra_join +
+ # exclude hosts with a running entry
+ 'WHERE active_hqe.host_id IS NULL '
+ # exclude locked, invalid, and non-Ready hosts
+ """
+ AND h.locked=false AND h.invalid=false
+ AND (h.status IS null OR h.status='Ready')
+ """)
+ if extra_where:
+ query += 'AND ' + extra_where + '\n'
+ # respect priority, then sort by ID (most recent first)
+ query += 'ORDER BY queued_hqe.priority DESC, queued_hqe.id'
+
+ rows = _db.execute(query)
+ return [self._extract_host_and_queue_entry(row) for row in rows]
- def _get_runnable_metahosts(self):
- # join with labels for metahost matching
- labels_join = 'INNER JOIN hosts_labels hl ON (hl.host_id=h.id)'
- # find queued HQEs scheduled for metahosts that match idle hosts
- queued_hqe_join = """
- INNER JOIN host_queue_entries AS queued_hqe
- ON (queued_hqe.meta_host = hl.label_id
- AND queued_hqe.host_id IS NULL
- AND NOT queued_hqe.active AND NOT queued_hqe.complete)
- """
- # need to exclude acl-inaccessible hosts
- acl_join = """
- INNER JOIN acl_groups_hosts ON h.id=acl_groups_hosts.host_id
- INNER JOIN acl_groups_users
- ON acl_groups_users.acl_group_id=acl_groups_hosts.acl_group_id
- INNER JOIN users ON acl_groups_users.user_id=users.id
- INNER JOIN jobs
- ON users.login=jobs.owner AND jobs.id=queued_hqe.job_id
- """
- # need to exclude blocked hosts
- block_join = """
- LEFT JOIN ineligible_host_queues AS ihq
- ON (ihq.job_id=queued_hqe.job_id AND ihq.host_id=h.id)
- """
- block_where = 'ihq.id IS NULL'
- extra_join = '\n'.join([labels_join, queued_hqe_join,
- acl_join, block_join])
- return self._get_runnable_entries(extra_join,
- extra_where=block_where)
+ def _get_runnable_nonmetahosts(self):
+ # find queued HQEs scheduled directly against hosts
+ queued_hqe_join = """
+ INNER JOIN host_queue_entries AS queued_hqe
+ ON (h.id = queued_hqe.host_id
+ AND NOT queued_hqe.active AND NOT queued_hqe.complete)
+ """
+ return self._get_runnable_entries(queued_hqe_join)
- def _schedule_new_jobs(self):
- print "finding work"
-
- scheduled_hosts, scheduled_queue_entries = set(), set()
- runnable = (self._get_runnable_nonmetahosts() +
- self._get_runnable_metahosts())
- for host, queue_entry in runnable:
- # we may get duplicate entries for a host or a queue
- # entry. we need to schedule each host and each queue
- # entry only once.
- if (host.id in scheduled_hosts or
- queue_entry.id in scheduled_queue_entries):
- continue
- agent = queue_entry.run(assigned_host=host)
- self.add_agent(agent)
- scheduled_hosts.add(host.id)
- scheduled_queue_entries.add(queue_entry.id)
+ def _get_runnable_metahosts(self):
+ # join with labels for metahost matching
+ labels_join = 'INNER JOIN hosts_labels hl ON (hl.host_id=h.id)'
+ # find queued HQEs scheduled for metahosts that match idle hosts
+ queued_hqe_join = """
+ INNER JOIN host_queue_entries AS queued_hqe
+ ON (queued_hqe.meta_host = hl.label_id
+ AND queued_hqe.host_id IS NULL
+ AND NOT queued_hqe.active AND NOT queued_hqe.complete)
+ """
+ # need to exclude acl-inaccessible hosts
+ acl_join = """
+ INNER JOIN acl_groups_hosts ON h.id=acl_groups_hosts.host_id
+ INNER JOIN acl_groups_users
+ ON acl_groups_users.acl_group_id=acl_groups_hosts.acl_group_id
+ INNER JOIN users ON acl_groups_users.user_id=users.id
+ INNER JOIN jobs
+ ON users.login=jobs.owner AND jobs.id=queued_hqe.job_id
+ """
+ # need to exclude blocked hosts
+ block_join = """
+ LEFT JOIN ineligible_host_queues AS ihq
+ ON (ihq.job_id=queued_hqe.job_id AND ihq.host_id=h.id)
+ """
+ block_where = 'ihq.id IS NULL'
+ extra_join = '\n'.join([labels_join, queued_hqe_join,
+ acl_join, block_join])
+ return self._get_runnable_entries(extra_join,
+ extra_where=block_where)
- def _find_aborting(self):
- num_aborted = 0
- # Find jobs that are aborting
- for entry in queue_entries_to_abort():
- agents_to_abort = self.get_agents(entry)
- entry_host = entry.get_host()
- reboot_task = RebootTask(entry_host)
- verify_task = VerifyTask(host = entry_host)
- tasks = [reboot_task, verify_task]
- if agents_to_abort:
- abort_task = AbortTask(entry, agents_to_abort)
- tasks.insert(0, abort_task)
- else:
- entry.set_status('Aborted')
- # just to make sure this host does not get
- # taken away
- entry_host.set_status('Rebooting')
- self.add_agent(Agent(tasks=tasks,
- queue_entry_ids = [entry.id]))
- num_aborted += 1
- if num_aborted >= 50:
- break
+ def _schedule_new_jobs(self):
+ print "finding work"
+
+ scheduled_hosts, scheduled_queue_entries = set(), set()
+ runnable = (self._get_runnable_nonmetahosts() +
+ self._get_runnable_metahosts())
+ for host, queue_entry in runnable:
+ # we may get duplicate entries for a host or a queue
+ # entry. we need to schedule each host and each queue
+ # entry only once.
+ if (host.id in scheduled_hosts or
+ queue_entry.id in scheduled_queue_entries):
+ continue
+ agent = queue_entry.run(assigned_host=host)
+ self.add_agent(agent)
+ scheduled_hosts.add(host.id)
+ scheduled_queue_entries.add(queue_entry.id)
- def _handle_agents(self):
- still_running = []
- num_started = self.num_started_agents()
- start_new = (num_started < self.max_running_agents)
- num_started_this_cycle = 0
- for agent in self._agents:
- if not agent.is_started():
- if not start_new:
- still_running.append(agent)
- continue
- num_started += 1
- num_started_this_cycle += 1
- if (num_started >= self.max_running_agents or
- num_started_this_cycle >=
- self.max_jobs_started_per_cycle):
- start_new = False
- agent.tick()
- if not agent.is_done():
- still_running.append(agent)
- else:
- print "agent finished"
- self._agents = still_running
- print num_started, 'running agents'
+ def _find_aborting(self):
+ num_aborted = 0
+ # Find jobs that are aborting
+ for entry in queue_entries_to_abort():
+ agents_to_abort = self.get_agents(entry)
+ entry_host = entry.get_host()
+ reboot_task = RebootTask(entry_host)
+ verify_task = VerifyTask(host = entry_host)
+ tasks = [reboot_task, verify_task]
+ if agents_to_abort:
+ abort_task = AbortTask(entry, agents_to_abort)
+ tasks.insert(0, abort_task)
+ else:
+ entry.set_status('Aborted')
+ # just to make sure this host does not get
+ # taken away
+ entry_host.set_status('Rebooting')
+ self.add_agent(Agent(tasks=tasks,
+ queue_entry_ids = [entry.id]))
+ num_aborted += 1
+ if num_aborted >= 50:
+ break
+
+
+ def _handle_agents(self):
+ still_running = []
+ num_started = self.num_started_agents()
+ start_new = (num_started < self.max_running_agents)
+ num_started_this_cycle = 0
+ for agent in self._agents:
+ if not agent.is_started():
+ if not start_new:
+ still_running.append(agent)
+ continue
+ num_started += 1
+ num_started_this_cycle += 1
+ if (num_started >= self.max_running_agents or
+ num_started_this_cycle >=
+ self.max_jobs_started_per_cycle):
+ start_new = False
+ agent.tick()
+ if not agent.is_done():
+ still_running.append(agent)
+ else:
+ print "agent finished"
+ self._agents = still_running
+ print num_started, 'running agents'
class RunMonitor(object):
- def __init__(self, cmd, nice_level = None, log_file = None):
- self.nice_level = nice_level
- self.log_file = log_file
- self.cmd = cmd
+ def __init__(self, cmd, nice_level = None, log_file = None):
+ self.nice_level = nice_level
+ self.log_file = log_file
+ self.cmd = cmd
- def run(self):
- if self.nice_level:
- nice_cmd = ['nice','-n', str(self.nice_level)]
- nice_cmd.extend(self.cmd)
- self.cmd = nice_cmd
+ def run(self):
+ if self.nice_level:
+ nice_cmd = ['nice','-n', str(self.nice_level)]
+ nice_cmd.extend(self.cmd)
+ self.cmd = nice_cmd
- out_file = None
- if self.log_file:
- try:
- os.makedirs(os.path.dirname(self.log_file))
- except OSError, exc:
- if exc.errno != errno.EEXIST:
- log_stacktrace(
- 'Unexpected error creating logfile '
- 'directory for %s' % self.log_file)
- try:
- out_file = open(self.log_file, 'a')
- out_file.write("\n%s\n" % ('*'*80))
- out_file.write("%s> %s\n" %
- (time.strftime("%X %x"),
- self.cmd))
- out_file.write("%s\n" % ('*'*80))
- except (OSError, IOError):
- log_stacktrace('Error opening log file %s' %
- self.log_file)
+ out_file = None
+ if self.log_file:
+ try:
+ os.makedirs(os.path.dirname(self.log_file))
+ except OSError, exc:
+ if exc.errno != errno.EEXIST:
+ log_stacktrace(
+ 'Unexpected error creating logfile '
+ 'directory for %s' % self.log_file)
+ try:
+ out_file = open(self.log_file, 'a')
+ out_file.write("\n%s\n" % ('*'*80))
+ out_file.write("%s> %s\n" %
+ (time.strftime("%X %x"),
+ self.cmd))
+ out_file.write("%s\n" % ('*'*80))
+ except (OSError, IOError):
+ log_stacktrace('Error opening log file %s' %
+ self.log_file)
- if not out_file:
- out_file = open('/dev/null', 'w')
+ if not out_file:
+ out_file = open('/dev/null', 'w')
- in_devnull = open('/dev/null', 'r')
- print "cmd = %s" % self.cmd
- print "path = %s" % os.getcwd()
+ in_devnull = open('/dev/null', 'r')
+ print "cmd = %s" % self.cmd
+ print "path = %s" % os.getcwd()
- self.proc = subprocess.Popen(self.cmd, stdout=out_file,
- stderr=subprocess.STDOUT,
- stdin=in_devnull)
- out_file.close()
- in_devnull.close()
+ self.proc = subprocess.Popen(self.cmd, stdout=out_file,
+ stderr=subprocess.STDOUT,
+ stdin=in_devnull)
+ out_file.close()
+ in_devnull.close()
- def get_pid(self):
- return self.proc.pid
+ def get_pid(self):
+ return self.proc.pid
- def kill(self):
- kill_autoserv(self.get_pid(), self.exit_code)
+ def kill(self):
+ kill_autoserv(self.get_pid(), self.exit_code)
- def exit_code(self):
- return self.proc.poll()
+ def exit_code(self):
+ return self.proc.poll()
class PidfileException(Exception):
- """\
- Raised when there's some unexpected behavior with the pid file.
- """
+ """\
+ Raised when there's some unexpected behavior with the pid file.
+ """
class PidfileRunMonitor(RunMonitor):
- def __init__(self, results_dir, cmd=None, nice_level=None,
- log_file=None):
- self.results_dir = os.path.abspath(results_dir)
- self.pid_file = os.path.join(results_dir, AUTOSERV_PID_FILE)
- self.lost_process = False
- self.start_time = time.time()
- if cmd is None:
- # we're reattaching to an existing pid, so don't call
- # the superconstructor (we don't want to kick off a new
- # process)
- pass
- else:
- super(PidfileRunMonitor, self).__init__(cmd,
- nice_level, log_file)
+ def __init__(self, results_dir, cmd=None, nice_level=None,
+ log_file=None):
+ self.results_dir = os.path.abspath(results_dir)
+ self.pid_file = os.path.join(results_dir, AUTOSERV_PID_FILE)
+ self.lost_process = False
+ self.start_time = time.time()
+ if cmd is None:
+ # we're reattaching to an existing pid, so don't call
+ # the superconstructor (we don't want to kick off a new
+ # process)
+ pass
+ else:
+ super(PidfileRunMonitor, self).__init__(cmd,
+ nice_level, log_file)
- def get_pid(self):
- pid, exit_status = self.get_pidfile_info()
- assert pid is not None
- return pid
+ def get_pid(self):
+ pid, exit_status = self.get_pidfile_info()
+ assert pid is not None
+ return pid
- def _check_command_line(self, command_line, spacer=' ',
- print_error=False):
- results_dir_arg = spacer.join(('', '-r', self.results_dir, ''))
- match = results_dir_arg in command_line
- if print_error and not match:
- print '%s not found in %s' % (repr(results_dir_arg),
- repr(command_line))
- return match
+ def _check_command_line(self, command_line, spacer=' ',
+ print_error=False):
+ results_dir_arg = spacer.join(('', '-r', self.results_dir, ''))
+ match = results_dir_arg in command_line
+ if print_error and not match:
+ print '%s not found in %s' % (repr(results_dir_arg),
+ repr(command_line))
+ return match
- def _check_proc_fs(self, pid):
- cmdline_path = os.path.join('/proc', str(pid), 'cmdline')
- try:
- cmdline_file = open(cmdline_path, 'r')
- cmdline = cmdline_file.read().strip()
- cmdline_file.close()
- except IOError:
- return False
- # /proc/.../cmdline has \x00 separating args
- return self._check_command_line(cmdline, spacer='\x00',
- print_error=True)
+ def _check_proc_fs(self, pid):
+ cmdline_path = os.path.join('/proc', str(pid), 'cmdline')
+ try:
+ cmdline_file = open(cmdline_path, 'r')
+ cmdline = cmdline_file.read().strip()
+ cmdline_file.close()
+ except IOError:
+ return False
+ # /proc/.../cmdline has \x00 separating args
+ return self._check_command_line(cmdline, spacer='\x00',
+ print_error=True)
- def read_pidfile(self):
- if not os.path.exists(self.pid_file):
- return None, None
- file_obj = open(self.pid_file, 'r')
- lines = file_obj.readlines()
- file_obj.close()
- assert 1 <= len(lines) <= 2
- try:
- pid = int(lines[0])
- exit_status = None
- if len(lines) == 2:
- exit_status = int(lines[1])
- except ValueError, exc:
- raise PidfileException('Corrupt pid file: ' +
- str(exc.args))
+ def read_pidfile(self):
+ if not os.path.exists(self.pid_file):
+ return None, None
+ file_obj = open(self.pid_file, 'r')
+ lines = file_obj.readlines()
+ file_obj.close()
+ assert 1 <= len(lines) <= 2
+ try:
+ pid = int(lines[0])
+ exit_status = None
+ if len(lines) == 2:
+ exit_status = int(lines[1])
+ except ValueError, exc:
+ raise PidfileException('Corrupt pid file: ' +
+ str(exc.args))
- return pid, exit_status
+ return pid, exit_status
- def _find_autoserv_proc(self):
- autoserv_procs = Dispatcher.find_autoservs()
- for pid, args in autoserv_procs.iteritems():
- if self._check_command_line(args):
- return pid, args
- return None, None
+ def _find_autoserv_proc(self):
+ autoserv_procs = Dispatcher.find_autoservs()
+ for pid, args in autoserv_procs.iteritems():
+ if self._check_command_line(args):
+ return pid, args
+ return None, None
- def get_pidfile_info(self):
- """\
- Returns:
- None, None if autoserv has not yet run
- pid, None if autoserv is running
- pid, exit_status if autoserv has completed
- """
- if self.lost_process:
- return self.pid, self.exit_status
+ def get_pidfile_info(self):
+ """\
+ Returns:
+ None, None if autoserv has not yet run
+ pid, None if autoserv is running
+ pid, exit_status if autoserv has completed
+ """
+ if self.lost_process:
+ return self.pid, self.exit_status
- pid, exit_status = self.read_pidfile()
+ pid, exit_status = self.read_pidfile()
- if pid is None:
- return self._handle_no_pid()
+ if pid is None:
+ return self._handle_no_pid()
- if exit_status is None:
- # double check whether or not autoserv is running
- proc_running = self._check_proc_fs(pid)
- if proc_running:
- return pid, exit_status
+ if exit_status is None:
+ # double check whether or not autoserv is running
+ proc_running = self._check_proc_fs(pid)
+ if proc_running:
+ return pid, exit_status
- # pid but no process - maybe process *just* exited
- pid, exit_status = self.read_pidfile()
- if exit_status is None:
- # autoserv exited without writing an exit code
- # to the pidfile
- error = ('autoserv died without writing exit '
- 'code')
- message = error + '\nPid: %s\nPidfile: %s' % (
- pid, self.pid_file)
- print message
- email_manager.enqueue_notify_email(error,
- message)
- self.on_lost_process(pid)
- return self.pid, self.exit_status
+ # pid but no process - maybe process *just* exited
+ pid, exit_status = self.read_pidfile()
+ if exit_status is None:
+ # autoserv exited without writing an exit code
+ # to the pidfile
+ error = ('autoserv died without writing exit '
+ 'code')
+ message = error + '\nPid: %s\nPidfile: %s' % (
+ pid, self.pid_file)
+ print message
+ email_manager.enqueue_notify_email(error,
+ message)
+ self.on_lost_process(pid)
+ return self.pid, self.exit_status
- return pid, exit_status
+ return pid, exit_status
- def _handle_no_pid(self):
- """\
- Called when no pidfile is found or no pid is in the pidfile.
- """
- # is autoserv running?
- pid, args = self._find_autoserv_proc()
- if pid is None:
- # no autoserv process running
- message = 'No pid found at ' + self.pid_file
- else:
- message = ("Process %d (%s) hasn't written pidfile %s" %
- (pid, args, self.pid_file))
+ def _handle_no_pid(self):
+ """\
+ Called when no pidfile is found or no pid is in the pidfile.
+ """
+ # is autoserv running?
+ pid, args = self._find_autoserv_proc()
+ if pid is None:
+ # no autoserv process running
+ message = 'No pid found at ' + self.pid_file
+ else:
+ message = ("Process %d (%s) hasn't written pidfile %s" %
+ (pid, args, self.pid_file))
- print message
- if time.time() - self.start_time > PIDFILE_TIMEOUT:
- email_manager.enqueue_notify_email(
- 'Process has failed to write pidfile', message)
- if pid is not None:
- kill_autoserv(pid)
- else:
- pid = 0
- self.on_lost_process(pid)
- return self.pid, self.exit_status
+ print message
+ if time.time() - self.start_time > PIDFILE_TIMEOUT:
+ email_manager.enqueue_notify_email(
+ 'Process has failed to write pidfile', message)
+ if pid is not None:
+ kill_autoserv(pid)
+ else:
+ pid = 0
+ self.on_lost_process(pid)
+ return self.pid, self.exit_status
- return None, None
+ return None, None
- def on_lost_process(self, pid):
- """\
- Called when autoserv has exited without writing an exit status,
- or we've timed out waiting for autoserv to write a pid to the
- pidfile. In either case, we just return failure and the caller
- should signal some kind of warning.
+ def on_lost_process(self, pid):
+ """\
+ Called when autoserv has exited without writing an exit status,
+ or we've timed out waiting for autoserv to write a pid to the
+ pidfile. In either case, we just return failure and the caller
+ should signal some kind of warning.
- pid is unimportant here, as it shouldn't be used by anyone.
- """
- self.lost_process = True
- self.pid = pid
- self.exit_status = 1
+ pid is unimportant here, as it shouldn't be used by anyone.
+ """
+ self.lost_process = True
+ self.pid = pid
+ self.exit_status = 1
- def exit_code(self):
- pid, exit_code = self.get_pidfile_info()
- return exit_code
+ def exit_code(self):
+ pid, exit_code = self.get_pidfile_info()
+ return exit_code
class Agent(object):
- def __init__(self, tasks, queue_entry_ids=[]):
- self.active_task = None
- self.queue = Queue.Queue(0)
- self.dispatcher = None
- self.queue_entry_ids = queue_entry_ids
-
- for task in tasks:
- self.add_task(task)
+ def __init__(self, tasks, queue_entry_ids=[]):
+ self.active_task = None
+ self.queue = Queue.Queue(0)
+ self.dispatcher = None
+ self.queue_entry_ids = queue_entry_ids
+
+ for task in tasks:
+ self.add_task(task)
- def add_task(self, task):
- self.queue.put_nowait(task)
- task.agent = self
+ def add_task(self, task):
+ self.queue.put_nowait(task)
+ task.agent = self
- def tick(self):
- print "agent tick"
- if self.active_task and not self.active_task.is_done():
- self.active_task.poll()
- else:
- self._next_task();
+ def tick(self):
+ print "agent tick"
+ if self.active_task and not self.active_task.is_done():
+ self.active_task.poll()
+ else:
+ self._next_task();
- def _next_task(self):
- print "agent picking task"
- if self.active_task:
- assert self.active_task.is_done()
+ def _next_task(self):
+ print "agent picking task"
+ if self.active_task:
+ assert self.active_task.is_done()
- if not self.active_task.success:
- self.on_task_failure()
+ if not self.active_task.success:
+ self.on_task_failure()
- self.active_task = None
- if not self.is_done():
- self.active_task = self.queue.get_nowait()
- if self.active_task:
- self.active_task.start()
+ self.active_task = None
+ if not self.is_done():
+ self.active_task = self.queue.get_nowait()
+ if self.active_task:
+ self.active_task.start()
- def on_task_failure(self):
- self.queue = Queue.Queue(0)
- for task in self.active_task.failure_tasks:
- self.add_task(task)
+ def on_task_failure(self):
+ self.queue = Queue.Queue(0)
+ for task in self.active_task.failure_tasks:
+ self.add_task(task)
- def is_started(self):
- return self.active_task is not None
+ def is_started(self):
+ return self.active_task is not None
- def is_done(self):
- return self.active_task == None and self.queue.empty()
+ def is_done(self):
+ return self.active_task == None and self.queue.empty()
- def start(self):
- assert self.dispatcher
+ def start(self):
+ assert self.dispatcher
- self._next_task()
+ self._next_task()
-
+
class AgentTask(object):
- def __init__(self, cmd, failure_tasks = []):
- self.done = False
- self.failure_tasks = failure_tasks
- self.started = False
- self.cmd = cmd
- self.task = None
- self.agent = None
- self.monitor = None
- self.success = None
+ def __init__(self, cmd, failure_tasks = []):
+ self.done = False
+ self.failure_tasks = failure_tasks
+ self.started = False
+ self.cmd = cmd
+ self.task = None
+ self.agent = None
+ self.monitor = None
+ self.success = None
- def poll(self):
- print "poll"
- if self.monitor:
- self.tick(self.monitor.exit_code())
- else:
- self.finished(False)
+ def poll(self):
+ print "poll"
+ if self.monitor:
+ self.tick(self.monitor.exit_code())
+ else:
+ self.finished(False)
- def tick(self, exit_code):
- if exit_code==None:
- return
-# print "exit_code was %d" % exit_code
- if exit_code == 0:
- success = True
- else:
- success = False
+ def tick(self, exit_code):
+ if exit_code==None:
+ return
+# print "exit_code was %d" % exit_code
+ if exit_code == 0:
+ success = True
+ else:
+ success = False
- self.finished(success)
+ self.finished(success)
- def is_done(self):
- return self.done
+ def is_done(self):
+ return self.done
- def finished(self, success):
- self.done = True
- self.success = success
- self.epilog()
+ def finished(self, success):
+ self.done = True
+ self.success = success
+ self.epilog()
- def prolog(self):
- pass
-
-
- def create_temp_resultsdir(self, suffix=''):
- self.temp_results_dir = tempfile.mkdtemp(suffix=suffix)
+ def prolog(self):
+ pass
- def cleanup(self):
- if (hasattr(self, 'temp_results_dir') and
- os.path.exists(self.temp_results_dir)):
- shutil.rmtree(self.temp_results_dir)
+ def create_temp_resultsdir(self, suffix=''):
+ self.temp_results_dir = tempfile.mkdtemp(suffix=suffix)
- def epilog(self):
- self.cleanup()
+ def cleanup(self):
+ if (hasattr(self, 'temp_results_dir') and
+ os.path.exists(self.temp_results_dir)):
+ shutil.rmtree(self.temp_results_dir)
- def start(self):
- assert self.agent
-
- if not self.started:
- self.prolog()
- self.run()
-
- self.started = True
-
-
- def abort(self):
- if self.monitor:
- self.monitor.kill()
- self.done = True
- self.cleanup()
+ def epilog(self):
+ self.cleanup()
- def run(self):
- if self.cmd:
- print "agent starting monitor"
- log_file = None
- if hasattr(self, 'host'):
- log_file = os.path.join(RESULTS_DIR, 'hosts',
- self.host.hostname)
- self.monitor = RunMonitor(
- self.cmd, nice_level = AUTOSERV_NICE_LEVEL,
- log_file = log_file)
- self.monitor.run()
+ def start(self):
+ assert self.agent
+
+ if not self.started:
+ self.prolog()
+ self.run()
+
+ self.started = True
+
+
+ def abort(self):
+ if self.monitor:
+ self.monitor.kill()
+ self.done = True
+ self.cleanup()
+
+
+ def run(self):
+ if self.cmd:
+ print "agent starting monitor"
+ log_file = None
+ if hasattr(self, 'host'):
+ log_file = os.path.join(RESULTS_DIR, 'hosts',
+ self.host.hostname)
+ self.monitor = RunMonitor(
+ self.cmd, nice_level = AUTOSERV_NICE_LEVEL,
+ log_file = log_file)
+ self.monitor.run()
class RepairTask(AgentTask):
- def __init__(self, host, fail_queue_entry=None):
- """\
- fail_queue_entry: queue entry to mark failed if this repair
- fails.
- """
- self.create_temp_resultsdir('.repair')
- cmd = [_autoserv_path , '-R', '-m', host.hostname,
- '-r', self.temp_results_dir]
- self.host = host
- self.fail_queue_entry = fail_queue_entry
- super(RepairTask, self).__init__(cmd)
+ def __init__(self, host, fail_queue_entry=None):
+ """\
+ fail_queue_entry: queue entry to mark failed if this repair
+ fails.
+ """
+ self.create_temp_resultsdir('.repair')
+ cmd = [_autoserv_path , '-R', '-m', host.hostname,
+ '-r', self.temp_results_dir]
+ self.host = host
+ self.fail_queue_entry = fail_queue_entry
+ super(RepairTask, self).__init__(cmd)
- def prolog(self):
- print "repair_task starting"
- self.host.set_status('Repairing')
+ def prolog(self):
+ print "repair_task starting"
+ self.host.set_status('Repairing')
- def epilog(self):
- super(RepairTask, self).epilog()
- if self.success:
- self.host.set_status('Ready')
- else:
- self.host.set_status('Repair Failed')
- if self.fail_queue_entry:
- self.fail_queue_entry.handle_host_failure()
+ def epilog(self):
+ super(RepairTask, self).epilog()
+ if self.success:
+ self.host.set_status('Ready')
+ else:
+ self.host.set_status('Repair Failed')
+ if self.fail_queue_entry:
+ self.fail_queue_entry.handle_host_failure()
class VerifyTask(AgentTask):
- def __init__(self, queue_entry=None, host=None):
- assert bool(queue_entry) != bool(host)
+ def __init__(self, queue_entry=None, host=None):
+ assert bool(queue_entry) != bool(host)
- self.host = host or queue_entry.host
- self.queue_entry = queue_entry
+ self.host = host or queue_entry.host
+ self.queue_entry = queue_entry
- self.create_temp_resultsdir('.verify')
- cmd = [_autoserv_path,'-v','-m',self.host.hostname,
- '-r', self.temp_results_dir]
+ self.create_temp_resultsdir('.verify')
+ cmd = [_autoserv_path,'-v','-m',self.host.hostname,
+ '-r', self.temp_results_dir]
- fail_queue_entry = None
- if queue_entry and not queue_entry.meta_host:
- fail_queue_entry = queue_entry
- failure_tasks = [RepairTask(self.host, fail_queue_entry)]
+ fail_queue_entry = None
+ if queue_entry and not queue_entry.meta_host:
+ fail_queue_entry = queue_entry
+ failure_tasks = [RepairTask(self.host, fail_queue_entry)]
- super(VerifyTask, self).__init__(cmd,
- failure_tasks=failure_tasks)
+ super(VerifyTask, self).__init__(cmd,
+ failure_tasks=failure_tasks)
- def prolog(self):
- print "starting verify on %s" % (self.host.hostname)
- if self.queue_entry:
- self.queue_entry.set_status('Verifying')
- self.queue_entry.clear_results_dir(
- self.queue_entry.verify_results_dir())
- self.host.set_status('Verifying')
+ def prolog(self):
+ print "starting verify on %s" % (self.host.hostname)
+ if self.queue_entry:
+ self.queue_entry.set_status('Verifying')
+ self.queue_entry.clear_results_dir(
+ self.queue_entry.verify_results_dir())
+ self.host.set_status('Verifying')
- def cleanup(self):
- if not os.path.exists(self.temp_results_dir):
- return
- if self.queue_entry and (self.success or
- not self.queue_entry.meta_host):
- self.move_results()
- super(VerifyTask, self).cleanup()
+ def cleanup(self):
+ if not os.path.exists(self.temp_results_dir):
+ return
+ if self.queue_entry and (self.success or
+ not self.queue_entry.meta_host):
+ self.move_results()
+ super(VerifyTask, self).cleanup()
- def epilog(self):
- super(VerifyTask, self).epilog()
+ def epilog(self):
+ super(VerifyTask, self).epilog()
- if self.success:
- self.host.set_status('Ready')
- elif self.queue_entry:
- self.queue_entry.requeue()
+ if self.success:
+ self.host.set_status('Ready')
+ elif self.queue_entry:
+ self.queue_entry.requeue()
- def move_results(self):
- assert self.queue_entry is not None
- target_dir = self.queue_entry.verify_results_dir()
- if not os.path.exists(target_dir):
- os.makedirs(target_dir)
- files = os.listdir(self.temp_results_dir)
- for filename in files:
- if filename == AUTOSERV_PID_FILE:
- continue
- self.force_move(os.path.join(self.temp_results_dir,
- filename),
- os.path.join(target_dir, filename))
+ def move_results(self):
+ assert self.queue_entry is not None
+ target_dir = self.queue_entry.verify_results_dir()
+ if not os.path.exists(target_dir):
+ os.makedirs(target_dir)
+ files = os.listdir(self.temp_results_dir)
+ for filename in files:
+ if filename == AUTOSERV_PID_FILE:
+ continue
+ self.force_move(os.path.join(self.temp_results_dir,
+ filename),
+ os.path.join(target_dir, filename))
- @staticmethod
- def force_move(source, dest):
- """\
- Replacement for shutil.move() that will delete the destination
- if it exists, even if it's a directory.
- """
- if os.path.exists(dest):
- print ('Warning: removing existing destination file ' +
- dest)
- remove_file_or_dir(dest)
- shutil.move(source, dest)
+ @staticmethod
+ def force_move(source, dest):
+ """\
+ Replacement for shutil.move() that will delete the destination
+ if it exists, even if it's a directory.
+ """
+ if os.path.exists(dest):
+ print ('Warning: removing existing destination file ' +
+ dest)
+ remove_file_or_dir(dest)
+ shutil.move(source, dest)
class VerifySynchronousTask(VerifyTask):
- def __init__(self, queue_entry):
- super(VerifySynchronousTask, self).__init__(
- queue_entry = queue_entry)
+ def __init__(self, queue_entry):
+ super(VerifySynchronousTask, self).__init__(
+ queue_entry = queue_entry)
- def epilog(self):
- super(VerifySynchronousTask, self).epilog()
- if self.success:
- if self.queue_entry.job.num_complete() > 0:
- # some other entry failed verify, and we've
- # already been marked as stopped
- return
+ def epilog(self):
+ super(VerifySynchronousTask, self).epilog()
+ if self.success:
+ if self.queue_entry.job.num_complete() > 0:
+ # some other entry failed verify, and we've
+ # already been marked as stopped
+ return
- self.queue_entry.set_status('Pending')
- job = self.queue_entry.job
- if job.is_ready():
- agent = job.run(self.queue_entry)
- self.agent.dispatcher.add_agent(agent)
+ self.queue_entry.set_status('Pending')
+ job = self.queue_entry.job
+ if job.is_ready():
+ agent = job.run(self.queue_entry)
+ self.agent.dispatcher.add_agent(agent)
class QueueTask(AgentTask):
- def __init__(self, job, queue_entries, cmd):
- super(QueueTask, self).__init__(cmd)
- self.job = job
- self.queue_entries = queue_entries
+ def __init__(self, job, queue_entries, cmd):
+ super(QueueTask, self).__init__(cmd)
+ self.job = job
+ self.queue_entries = queue_entries
- @staticmethod
- def _write_keyval(results_dir, field, value):
- key_path = os.path.join(results_dir, 'keyval')
- keyval_file = open(key_path, 'a')
- print >> keyval_file, '%s=%d' % (field, value)
- keyval_file.close()
+ @staticmethod
+ def _write_keyval(results_dir, field, value):
+ key_path = os.path.join(results_dir, 'keyval')
+ keyval_file = open(key_path, 'a')
+ print >> keyval_file, '%s=%d' % (field, value)
+ keyval_file.close()
- def results_dir(self):
- return self.queue_entries[0].results_dir()
+ def results_dir(self):
+ return self.queue_entries[0].results_dir()
- def run(self):
- """\
- Override AgentTask.run() so we can use a PidfileRunMonitor.
- """
- self.monitor = PidfileRunMonitor(self.results_dir(),
- cmd=self.cmd,
- nice_level=AUTOSERV_NICE_LEVEL)
- self.monitor.run()
+ def run(self):
+ """\
+ Override AgentTask.run() so we can use a PidfileRunMonitor.
+ """
+ self.monitor = PidfileRunMonitor(self.results_dir(),
+ cmd=self.cmd,
+ nice_level=AUTOSERV_NICE_LEVEL)
+ self.monitor.run()
- def prolog(self):
- # write some job timestamps into the job keyval file
- queued = time.mktime(self.job.created_on.timetuple())
- started = time.time()
- self._write_keyval(self.results_dir(), "job_queued", queued)
- self._write_keyval(self.results_dir(), "job_started", started)
- for queue_entry in self.queue_entries:
- print "starting queue_task on %s/%s" % (queue_entry.host.hostname, queue_entry.id)
- queue_entry.set_status('Running')
- queue_entry.host.set_status('Running')
- if (not self.job.is_synchronous() and
- self.job.num_machines() > 1):
- assert len(self.queue_entries) == 1
- self.job.write_to_machines_file(self.queue_entries[0])
+ def prolog(self):
+ # write some job timestamps into the job keyval file
+ queued = time.mktime(self.job.created_on.timetuple())
+ started = time.time()
+ self._write_keyval(self.results_dir(), "job_queued", queued)
+ self._write_keyval(self.results_dir(), "job_started", started)
+ for queue_entry in self.queue_entries:
+ print "starting queue_task on %s/%s" % (queue_entry.host.hostname, queue_entry.id)
+ queue_entry.set_status('Running')
+ queue_entry.host.set_status('Running')
+ if (not self.job.is_synchronous() and
+ self.job.num_machines() > 1):
+ assert len(self.queue_entries) == 1
+ self.job.write_to_machines_file(self.queue_entries[0])
- def _finish_task(self):
- # write out the finished time into the results keyval
- finished = time.time()
- self._write_keyval(self.results_dir(), "job_finished",
- finished)
+ def _finish_task(self):
+ # write out the finished time into the results keyval
+ finished = time.time()
+ self._write_keyval(self.results_dir(), "job_finished",
+ finished)
- # parse the results of the job
- if self.job.is_synchronous() or self.job.num_machines() == 1:
- parse_results(self.job.results_dir())
- else:
- for queue_entry in self.queue_entries:
- parse_results(queue_entry.results_dir(),
- flags="-l 2")
+ # parse the results of the job
+ if self.job.is_synchronous() or self.job.num_machines() == 1:
+ parse_results(self.job.results_dir())
+ else:
+ for queue_entry in self.queue_entries:
+ parse_results(queue_entry.results_dir(),
+ flags="-l 2")
- def abort(self):
- super(QueueTask, self).abort()
- self._finish_task()
+ def abort(self):
+ super(QueueTask, self).abort()
+ self._finish_task()
- def epilog(self):
- super(QueueTask, self).epilog()
- if self.success:
- status = 'Completed'
- else:
- status = 'Failed'
+ def epilog(self):
+ super(QueueTask, self).epilog()
+ if self.success:
+ status = 'Completed'
+ else:
+ status = 'Failed'
- for queue_entry in self.queue_entries:
- queue_entry.set_status(status)
- queue_entry.host.set_status('Ready')
+ for queue_entry in self.queue_entries:
+ queue_entry.set_status(status)
+ queue_entry.host.set_status('Ready')
- self._finish_task()
+ self._finish_task()
- print "queue_task finished with %s/%s" % (status, self.success)
+ print "queue_task finished with %s/%s" % (status, self.success)
class RecoveryQueueTask(QueueTask):
- def __init__(self, job, queue_entries, run_monitor):
- super(RecoveryQueueTask, self).__init__(job,
- queue_entries, cmd=None)
- self.run_monitor = run_monitor
+ def __init__(self, job, queue_entries, run_monitor):
+ super(RecoveryQueueTask, self).__init__(job,
+ queue_entries, cmd=None)
+ self.run_monitor = run_monitor
- def run(self):
- self.monitor = self.run_monitor
+ def run(self):
+ self.monitor = self.run_monitor
- def prolog(self):
- # recovering an existing process - don't do prolog
- pass
+ def prolog(self):
+ # recovering an existing process - don't do prolog
+ pass
class RebootTask(AgentTask):
- def __init__(self, host):
- global _autoserv_path
-
- # Current implementation of autoserv requires control file
- # to be passed on reboot action request. TODO: remove when no
- # longer appropriate.
- self.create_temp_resultsdir('.reboot')
- self.cmd = [_autoserv_path, '-b', '-m', host.hostname,
- '-r', self.temp_results_dir, '/dev/null']
- self.host = host
- super(RebootTask, self).__init__(self.cmd,
- failure_tasks=[RepairTask(host)])
+ def __init__(self, host):
+ global _autoserv_path
+
+ # Current implementation of autoserv requires control file
+ # to be passed on reboot action request. TODO: remove when no
+ # longer appropriate.
+ self.create_temp_resultsdir('.reboot')
+ self.cmd = [_autoserv_path, '-b', '-m', host.hostname,
+ '-r', self.temp_results_dir, '/dev/null']
+ self.host = host
+ super(RebootTask, self).__init__(self.cmd,
+ failure_tasks=[RepairTask(host)])
- def prolog(self):
- print "starting reboot task for host: %s" % self.host.hostname
- self.host.set_status("Rebooting")
+ def prolog(self):
+ print "starting reboot task for host: %s" % self.host.hostname
+ self.host.set_status("Rebooting")
class AbortTask(AgentTask):
- def __init__(self, queue_entry, agents_to_abort):
- self.queue_entry = queue_entry
- self.agents_to_abort = agents_to_abort
- for agent in agents_to_abort:
- agent.dispatcher.remove_agent(agent)
- super(AbortTask, self).__init__('')
+ def __init__(self, queue_entry, agents_to_abort):
+ self.queue_entry = queue_entry
+ self.agents_to_abort = agents_to_abort
+ for agent in agents_to_abort:
+ agent.dispatcher.remove_agent(agent)
+ super(AbortTask, self).__init__('')
- def prolog(self):
- print "starting abort on host %s, job %s" % (
- self.queue_entry.host_id, self.queue_entry.job_id)
- self.queue_entry.set_status('Aborting')
-
-
- def epilog(self):
- super(AbortTask, self).epilog()
- self.queue_entry.set_status('Aborted')
- self.success = True
+ def prolog(self):
+ print "starting abort on host %s, job %s" % (
+ self.queue_entry.host_id, self.queue_entry.job_id)
+ self.queue_entry.set_status('Aborting')
- def run(self):
- for agent in self.agents_to_abort:
- if (agent.active_task):
- agent.active_task.abort()
+ def epilog(self):
+ super(AbortTask, self).epilog()
+ self.queue_entry.set_status('Aborted')
+ self.success = True
+
+
+ def run(self):
+ for agent in self.agents_to_abort:
+ if (agent.active_task):
+ agent.active_task.abort()
class DBObject(object):
- def __init__(self, id=None, row=None, new_record=False):
- assert (bool(id) != bool(row))
+ def __init__(self, id=None, row=None, new_record=False):
+ assert (bool(id) != bool(row))
- self.__table = self._get_table()
- fields = self._fields()
+ self.__table = self._get_table()
+ fields = self._fields()
- self.__new_record = new_record
+ self.__new_record = new_record
- if row is None:
- sql = 'SELECT * FROM %s WHERE ID=%%s' % self.__table
- rows = _db.execute(sql, (id,))
- if len(rows) == 0:
- raise "row not found (table=%s, id=%s)" % \
- (self.__table, id)
- row = rows[0]
+ if row is None:
+ sql = 'SELECT * FROM %s WHERE ID=%%s' % self.__table
+ rows = _db.execute(sql, (id,))
+ if len(rows) == 0:
+ raise "row not found (table=%s, id=%s)" % \
+ (self.__table, id)
+ row = rows[0]
- assert len(row) == self.num_cols(), (
- "table = %s, row = %s/%d, fields = %s/%d" % (
- self.__table, row, len(row), fields, self.num_cols()))
+ assert len(row) == self.num_cols(), (
+ "table = %s, row = %s/%d, fields = %s/%d" % (
+ self.__table, row, len(row), fields, self.num_cols()))
- self.__valid_fields = {}
- for i,value in enumerate(row):
- self.__dict__[fields[i]] = value
- self.__valid_fields[fields[i]] = True
+ self.__valid_fields = {}
+ for i,value in enumerate(row):
+ self.__dict__[fields[i]] = value
+ self.__valid_fields[fields[i]] = True
- del self.__valid_fields['id']
+ del self.__valid_fields['id']
- @classmethod
- def _get_table(cls):
- raise NotImplementedError('Subclasses must override this')
+ @classmethod
+ def _get_table(cls):
+ raise NotImplementedError('Subclasses must override this')
- @classmethod
- def _fields(cls):
- raise NotImplementedError('Subclasses must override this')
+ @classmethod
+ def _fields(cls):
+ raise NotImplementedError('Subclasses must override this')
- @classmethod
- def num_cols(cls):
- return len(cls._fields())
+ @classmethod
+ def num_cols(cls):
+ return len(cls._fields())
- def count(self, where, table = None):
- if not table:
- table = self.__table
-
- rows = _db.execute("""
- SELECT count(*) FROM %s
- WHERE %s
- """ % (table, where))
+ def count(self, where, table = None):
+ if not table:
+ table = self.__table
- assert len(rows) == 1
+ rows = _db.execute("""
+ SELECT count(*) FROM %s
+ WHERE %s
+ """ % (table, where))
- return int(rows[0][0])
+ assert len(rows) == 1
+
+ return int(rows[0][0])
- def update_field(self, field, value):
- assert self.__valid_fields[field]
-
- if self.__dict__[field] == value:
- return
+ def update_field(self, field, value):
+ assert self.__valid_fields[field]
- query = "UPDATE %s SET %s = %%s WHERE id = %%s" % \
- (self.__table, field)
- _db.execute(query, (value, self.id))
+ if self.__dict__[field] == value:
+ return
- self.__dict__[field] = value
+ query = "UPDATE %s SET %s = %%s WHERE id = %%s" % \
+ (self.__table, field)
+ _db.execute(query, (value, self.id))
+
+ self.__dict__[field] = value
- def save(self):
- if self.__new_record:
- keys = self._fields()[1:] # avoid id
- columns = ','.join([str(key) for key in keys])
- values = ['"%s"' % self.__dict__[key] for key in keys]
- values = ','.join(values)
- query = """INSERT INTO %s (%s) VALUES (%s)""" % \
- (self.__table, columns, values)
- _db.execute(query)
+ def save(self):
+ if self.__new_record:
+ keys = self._fields()[1:] # avoid id
+ columns = ','.join([str(key) for key in keys])
+ values = ['"%s"' % self.__dict__[key] for key in keys]
+ values = ','.join(values)
+ query = """INSERT INTO %s (%s) VALUES (%s)""" % \
+ (self.__table, columns, values)
+ _db.execute(query)
- def delete(self):
- query = 'DELETE FROM %s WHERE id=%%s' % self.__table
- _db.execute(query, (self.id,))
+ def delete(self):
+ query = 'DELETE FROM %s WHERE id=%%s' % self.__table
+ _db.execute(query, (self.id,))
- @classmethod
- def fetch(cls, where, params=()):
- rows = _db.execute(
- 'SELECT * FROM %s WHERE %s' % (cls._get_table(), where),
- params)
- for row in rows:
- yield cls(row=row)
+ @classmethod
+ def fetch(cls, where, params=()):
+ rows = _db.execute(
+ 'SELECT * FROM %s WHERE %s' % (cls._get_table(), where),
+ params)
+ for row in rows:
+ yield cls(row=row)
class IneligibleHostQueue(DBObject):
- def __init__(self, id=None, row=None, new_record=None):
- super(IneligibleHostQueue, self).__init__(id=id, row=row,
- new_record=new_record)
+ def __init__(self, id=None, row=None, new_record=None):
+ super(IneligibleHostQueue, self).__init__(id=id, row=row,
+ new_record=new_record)
- @classmethod
- def _get_table(cls):
- return 'ineligible_host_queues'
+ @classmethod
+ def _get_table(cls):
+ return 'ineligible_host_queues'
- @classmethod
- def _fields(cls):
- return ['id', 'job_id', 'host_id']
+ @classmethod
+ def _fields(cls):
+ return ['id', 'job_id', 'host_id']
class Host(DBObject):
- def __init__(self, id=None, row=None):
- super(Host, self).__init__(id=id, row=row)
+ def __init__(self, id=None, row=None):
+ super(Host, self).__init__(id=id, row=row)
- @classmethod
- def _get_table(cls):
- return 'hosts'
+ @classmethod
+ def _get_table(cls):
+ return 'hosts'
- @classmethod
- def _fields(cls):
- return ['id', 'hostname', 'locked', 'synch_id','status',
- 'invalid']
+ @classmethod
+ def _fields(cls):
+ return ['id', 'hostname', 'locked', 'synch_id','status',
+ 'invalid']
- def current_task(self):
- rows = _db.execute("""
- SELECT * FROM host_queue_entries WHERE host_id=%s AND NOT complete AND active
- """, (self.id,))
-
- if len(rows) == 0:
- return None
- else:
- assert len(rows) == 1
- results = rows[0];
-# print "current = %s" % results
- return HostQueueEntry(row=results)
+ def current_task(self):
+ rows = _db.execute("""
+ SELECT * FROM host_queue_entries WHERE host_id=%s AND NOT complete AND active
+ """, (self.id,))
+
+ if len(rows) == 0:
+ return None
+ else:
+ assert len(rows) == 1
+ results = rows[0];
+# print "current = %s" % results
+ return HostQueueEntry(row=results)
- def yield_work(self):
- print "%s yielding work" % self.hostname
- if self.current_task():
- self.current_task().requeue()
-
- def set_status(self,status):
- print '%s -> %s' % (self.hostname, status)
- self.update_field('status',status)
+ def yield_work(self):
+ print "%s yielding work" % self.hostname
+ if self.current_task():
+ self.current_task().requeue()
+
+ def set_status(self,status):
+ print '%s -> %s' % (self.hostname, status)
+ self.update_field('status',status)
class HostQueueEntry(DBObject):
- def __init__(self, id=None, row=None):
- assert id or row
- super(HostQueueEntry, self).__init__(id=id, row=row)
- self.job = Job(self.job_id)
+ def __init__(self, id=None, row=None):
+ assert id or row
+ super(HostQueueEntry, self).__init__(id=id, row=row)
+ self.job = Job(self.job_id)
- if self.host_id:
- self.host = Host(self.host_id)
- else:
- self.host = None
+ if self.host_id:
+ self.host = Host(self.host_id)
+ else:
+ self.host = None
- self.queue_log_path = os.path.join(self.job.results_dir(),
- 'queue.log.' + str(self.id))
+ self.queue_log_path = os.path.join(self.job.results_dir(),
+ 'queue.log.' + str(self.id))
- @classmethod
- def _get_table(cls):
- return 'host_queue_entries'
+ @classmethod
+ def _get_table(cls):
+ return 'host_queue_entries'
- @classmethod
- def _fields(cls):
- return ['id', 'job_id', 'host_id', 'priority', 'status',
- 'meta_host', 'active', 'complete']
+ @classmethod
+ def _fields(cls):
+ return ['id', 'job_id', 'host_id', 'priority', 'status',
+ 'meta_host', 'active', 'complete']
- def set_host(self, host):
- if host:
- self.queue_log_record('Assigning host ' + host.hostname)
- self.update_field('host_id', host.id)
- self.update_field('active', True)
- self.block_host(host.id)
- else:
- self.queue_log_record('Releasing host')
- self.unblock_host(self.host.id)
- self.update_field('host_id', None)
+ def set_host(self, host):
+ if host:
+ self.queue_log_record('Assigning host ' + host.hostname)
+ self.update_field('host_id', host.id)
+ self.update_field('active', True)
+ self.block_host(host.id)
+ else:
+ self.queue_log_record('Releasing host')
+ self.unblock_host(self.host.id)
+ self.update_field('host_id', None)
- self.host = host
+ self.host = host
- def get_host(self):
- return self.host
+ def get_host(self):
+ return self.host
- def queue_log_record(self, log_line):
- now = str(datetime.datetime.now())
- queue_log = open(self.queue_log_path, 'a', 0)
- queue_log.write(now + ' ' + log_line + '\n')
- queue_log.close()
+ def queue_log_record(self, log_line):
+ now = str(datetime.datetime.now())
+ queue_log = open(self.queue_log_path, 'a', 0)
+ queue_log.write(now + ' ' + log_line + '\n')
+ queue_log.close()
- def block_host(self, host_id):
- print "creating block %s/%s" % (self.job.id, host_id)
- row = [0, self.job.id, host_id]
- block = IneligibleHostQueue(row=row, new_record=True)
- block.save()
+ def block_host(self, host_id):
+ print "creating block %s/%s" % (self.job.id, host_id)
+ row = [0, self.job.id, host_id]
+ block = IneligibleHostQueue(row=row, new_record=True)
+ block.save()
- def unblock_host(self, host_id):
- print "removing block %s/%s" % (self.job.id, host_id)
- blocks = IneligibleHostQueue.fetch(
- 'job_id=%d and host_id=%d' % (self.job.id, host_id))
- for block in blocks:
- block.delete()
+ def unblock_host(self, host_id):
+ print "removing block %s/%s" % (self.job.id, host_id)
+ blocks = IneligibleHostQueue.fetch(
+ 'job_id=%d and host_id=%d' % (self.job.id, host_id))
+ for block in blocks:
+ block.delete()
- def results_dir(self):
- if self.job.is_synchronous() or self.job.num_machines() == 1:
- return self.job.job_dir
- else:
- assert self.host
- return os.path.join(self.job.job_dir,
- self.host.hostname)
+ def results_dir(self):
+ if self.job.is_synchronous() or self.job.num_machines() == 1:
+ return self.job.job_dir
+ else:
+ assert self.host
+ return os.path.join(self.job.job_dir,
+ self.host.hostname)
- def verify_results_dir(self):
- if self.job.is_synchronous() or self.job.num_machines() > 1:
- assert self.host
- return os.path.join(self.job.job_dir,
- self.host.hostname)
- else:
- return self.job.job_dir
+ def verify_results_dir(self):
+ if self.job.is_synchronous() or self.job.num_machines() > 1:
+ assert self.host
+ return os.path.join(self.job.job_dir,
+ self.host.hostname)
+ else:
+ return self.job.job_dir
- def set_status(self, status):
- self.update_field('status', status)
- if self.host:
- hostname = self.host.hostname
- else:
- hostname = 'no host'
- print "%s/%d status -> %s" % (hostname, self.id, self.status)
- if status in ['Queued']:
- self.update_field('complete', False)
- self.update_field('active', False)
+ def set_status(self, status):
+ self.update_field('status', status)
+ if self.host:
+ hostname = self.host.hostname
+ else:
+ hostname = 'no host'
+ print "%s/%d status -> %s" % (hostname, self.id, self.status)
+ if status in ['Queued']:
+ self.update_field('complete', False)
+ self.update_field('active', False)
- if status in ['Pending', 'Running', 'Verifying', 'Starting',
- 'Abort', 'Aborting']:
- self.update_field('complete', False)
- self.update_field('active', True)
+ if status in ['Pending', 'Running', 'Verifying', 'Starting',
+ 'Abort', 'Aborting']:
+ self.update_field('complete', False)
+ self.update_field('active', True)
- if status in ['Failed', 'Completed', 'Stopped', 'Aborted']:
- self.update_field('complete', True)
- self.update_field('active', False)
+ if status in ['Failed', 'Completed', 'Stopped', 'Aborted']:
+ self.update_field('complete', True)
+ self.update_field('active', False)
- def run(self,assigned_host=None):
- if self.meta_host:
- assert assigned_host
- # ensure results dir exists for the queue log
- self.job.create_results_dir()
- self.set_host(assigned_host)
+ def run(self,assigned_host=None):
+ if self.meta_host:
+ assert assigned_host
+ # ensure results dir exists for the queue log
+ self.job.create_results_dir()
+ self.set_host(assigned_host)
- print "%s/%s scheduled on %s, status=%s" % (self.job.name,
- self.meta_host, self.host.hostname, self.status)
+ print "%s/%s scheduled on %s, status=%s" % (self.job.name,
+ self.meta_host, self.host.hostname, self.status)
- return self.job.run(queue_entry=self)
+ return self.job.run(queue_entry=self)
- def requeue(self):
- self.set_status('Queued')
+ def requeue(self):
+ self.set_status('Queued')
- if self.meta_host:
- self.set_host(None)
+ if self.meta_host:
+ self.set_host(None)
- def handle_host_failure(self):
- """\
- Called when this queue entry's host has failed verification and
- repair.
- """
- assert not self.meta_host
- self.set_status('Failed')
- if self.job.is_synchronous():
- self.job.stop_all_entries()
+ def handle_host_failure(self):
+ """\
+ Called when this queue entry's host has failed verification and
+ repair.
+ """
+ assert not self.meta_host
+ self.set_status('Failed')
+ if self.job.is_synchronous():
+ self.job.stop_all_entries()
- def clear_results_dir(self, results_dir=None, dont_delete_files=False):
- results_dir = results_dir or self.results_dir()
- if not os.path.exists(results_dir):
- return
- if dont_delete_files:
- temp_dir = tempfile.mkdtemp(suffix='.clear_results')
- print 'Moving results from %s to %s' % (results_dir,
- temp_dir)
- for filename in os.listdir(results_dir):
- path = os.path.join(results_dir, filename)
- if dont_delete_files:
- shutil.move(path,
- os.path.join(temp_dir, filename))
- else:
- remove_file_or_dir(path)
+ def clear_results_dir(self, results_dir=None, dont_delete_files=False):
+ results_dir = results_dir or self.results_dir()
+ if not os.path.exists(results_dir):
+ return
+ if dont_delete_files:
+ temp_dir = tempfile.mkdtemp(suffix='.clear_results')
+ print 'Moving results from %s to %s' % (results_dir,
+ temp_dir)
+ for filename in os.listdir(results_dir):
+ path = os.path.join(results_dir, filename)
+ if dont_delete_files:
+ shutil.move(path,
+ os.path.join(temp_dir, filename))
+ else:
+ remove_file_or_dir(path)
class Job(DBObject):
- def __init__(self, id=None, row=None):
- assert id or row
- super(Job, self).__init__(id=id, row=row)
+ def __init__(self, id=None, row=None):
+ assert id or row
+ super(Job, self).__init__(id=id, row=row)
- self.job_dir = os.path.join(RESULTS_DIR, "%s-%s" % (self.id,
- self.owner))
+ self.job_dir = os.path.join(RESULTS_DIR, "%s-%s" % (self.id,
+ self.owner))
- @classmethod
- def _get_table(cls):
- return 'jobs'
+ @classmethod
+ def _get_table(cls):
+ return 'jobs'
- @classmethod
- def _fields(cls):
- return ['id', 'owner', 'name', 'priority', 'control_file',
- 'control_type', 'created_on', 'synch_type',
- 'synch_count', 'synchronizing']
+ @classmethod
+ def _fields(cls):
+ return ['id', 'owner', 'name', 'priority', 'control_file',
+ 'control_type', 'created_on', 'synch_type',
+ 'synch_count', 'synchronizing']
- def is_server_job(self):
- return self.control_type != 2
+ def is_server_job(self):
+ return self.control_type != 2
- def get_host_queue_entries(self):
- rows = _db.execute("""
- SELECT * FROM host_queue_entries
- WHERE job_id= %s
- """, (self.id,))
- entries = [HostQueueEntry(row=i) for i in rows]
+ def get_host_queue_entries(self):
+ rows = _db.execute("""
+ SELECT * FROM host_queue_entries
+ WHERE job_id= %s
+ """, (self.id,))
+ entries = [HostQueueEntry(row=i) for i in rows]
- assert len(entries)>0
+ assert len(entries)>0
- return entries
+ return entries
- def set_status(self, status, update_queues=False):
- self.update_field('status',status)
-
- if update_queues:
- for queue_entry in self.get_host_queue_entries():
- queue_entry.set_status(status)
+ def set_status(self, status, update_queues=False):
+ self.update_field('status',status)
+
+ if update_queues:
+ for queue_entry in self.get_host_queue_entries():
+ queue_entry.set_status(status)
- def is_synchronous(self):
- return self.synch_type == 2
+ def is_synchronous(self):
+ return self.synch_type == 2
- def is_ready(self):
- if not self.is_synchronous():
- return True
- sql = "job_id=%s AND status='Pending'" % self.id
- count = self.count(sql, table='host_queue_entries')
- return (count == self.synch_count)
+ def is_ready(self):
+ if not self.is_synchronous():
+ return True
+ sql = "job_id=%s AND status='Pending'" % self.id
+ count = self.count(sql, table='host_queue_entries')
+ return (count == self.synch_count)
- def ready_to_synchronize(self):
- # heuristic
- queue_entries = self.get_host_queue_entries()
- count = 0
- for queue_entry in queue_entries:
- if queue_entry.status == 'Pending':
- count += 1
+ def ready_to_synchronize(self):
+ # heuristic
+ queue_entries = self.get_host_queue_entries()
+ count = 0
+ for queue_entry in queue_entries:
+ if queue_entry.status == 'Pending':
+ count += 1
- return (count/self.synch_count >= 0.5)
+ return (count/self.synch_count >= 0.5)
- def start_synchronizing(self):
- self.update_field('synchronizing', True)
+ def start_synchronizing(self):
+ self.update_field('synchronizing', True)
- def results_dir(self):
- return self.job_dir
+ def results_dir(self):
+ return self.job_dir
- def num_machines(self, clause = None):
- sql = "job_id=%s" % self.id
- if clause:
- sql += " AND (%s)" % clause
- return self.count(sql, table='host_queue_entries')
+ def num_machines(self, clause = None):
+ sql = "job_id=%s" % self.id
+ if clause:
+ sql += " AND (%s)" % clause
+ return self.count(sql, table='host_queue_entries')
- def num_queued(self):
- return self.num_machines('not complete')
+ def num_queued(self):
+ return self.num_machines('not complete')
- def num_active(self):
- return self.num_machines('active')
+ def num_active(self):
+ return self.num_machines('active')
- def num_complete(self):
- return self.num_machines('complete')
+ def num_complete(self):
+ return self.num_machines('complete')
- def is_finished(self):
- left = self.num_queued()
- print "%s: %s machines left" % (self.name, left)
- return left==0
+ def is_finished(self):
+ left = self.num_queued()
+ print "%s: %s machines left" % (self.name, left)
+ return left==0
- def stop_synchronizing(self):
- self.update_field('synchronizing', False)
- self.set_status('Queued', update_queues = False)
+ def stop_synchronizing(self):
+ self.update_field('synchronizing', False)
+ self.set_status('Queued', update_queues = False)
- def stop_all_entries(self):
- for child_entry in self.get_host_queue_entries():
- if not child_entry.complete:
- child_entry.set_status('Stopped')
+ def stop_all_entries(self):
+ for child_entry in self.get_host_queue_entries():
+ if not child_entry.complete:
+ child_entry.set_status('Stopped')
- def write_to_machines_file(self, queue_entry):
- hostname = queue_entry.get_host().hostname
- print "writing %s to job %s machines file" % (hostname, self.id)
- file_path = os.path.join(self.job_dir, '.machines')
- mf = open(file_path, 'a')
- mf.write("%s\n" % queue_entry.get_host().hostname)
- mf.close()
+ def write_to_machines_file(self, queue_entry):
+ hostname = queue_entry.get_host().hostname
+ print "writing %s to job %s machines file" % (hostname, self.id)
+ file_path = os.path.join(self.job_dir, '.machines')
+ mf = open(file_path, 'a')
+ mf.write("%s\n" % queue_entry.get_host().hostname)
+ mf.close()
- def create_results_dir(self, queue_entry=None):
- print "create: active: %s complete %s" % (self.num_active(),
- self.num_complete())
+ def create_results_dir(self, queue_entry=None):
+ print "create: active: %s complete %s" % (self.num_active(),
+ self.num_complete())
- if not os.path.exists(self.job_dir):
- os.makedirs(self.job_dir)
+ if not os.path.exists(self.job_dir):
+ os.makedirs(self.job_dir)
- if queue_entry:
- return queue_entry.results_dir()
- return self.job_dir
+ if queue_entry:
+ return queue_entry.results_dir()
+ return self.job_dir
- def run(self, queue_entry):
- results_dir = self.create_results_dir(queue_entry)
+ def run(self, queue_entry):
+ results_dir = self.create_results_dir(queue_entry)
- if self.is_synchronous():
- if not self.is_ready():
- return Agent([VerifySynchronousTask(
- queue_entry = queue_entry)],
- [queue_entry.id])
+ if self.is_synchronous():
+ if not self.is_ready():
+ return Agent([VerifySynchronousTask(
+ queue_entry = queue_entry)],
+ [queue_entry.id])
- queue_entry.set_status('Starting')
+ queue_entry.set_status('Starting')
- ctrl = open(os.tmpnam(), 'w')
- if self.control_file:
- ctrl.write(self.control_file)
- else:
- ctrl.write("")
- ctrl.flush()
+ ctrl = open(os.tmpnam(), 'w')
+ if self.control_file:
+ ctrl.write(self.control_file)
+ else:
+ ctrl.write("")
+ ctrl.flush()
- if self.is_synchronous():
- queue_entries = self.get_host_queue_entries()
- else:
- assert queue_entry
- queue_entries = [queue_entry]
- hostnames = ','.join([entry.get_host().hostname
- for entry in queue_entries])
+ if self.is_synchronous():
+ queue_entries = self.get_host_queue_entries()
+ else:
+ assert queue_entry
+ queue_entries = [queue_entry]
+ hostnames = ','.join([entry.get_host().hostname
+ for entry in queue_entries])
- # determine the job tag
- if self.is_synchronous() or self.num_machines() == 1:
- job_name = "%s-%s" % (self.id, self.owner)
- else:
- job_name = "%s-%s/%s" % (self.id, self.owner,
- hostnames)
+ # determine the job tag
+ if self.is_synchronous() or self.num_machines() == 1:
+ job_name = "%s-%s" % (self.id, self.owner)
+ else:
+ job_name = "%s-%s/%s" % (self.id, self.owner,
+ hostnames)
- params = [_autoserv_path, '-P', job_name, '-p', '-n',
- '-r', os.path.abspath(results_dir),
- '-b', '-u', self.owner, '-l', self.name,
- '-m', hostnames, ctrl.name]
+ params = [_autoserv_path, '-P', job_name, '-p', '-n',
+ '-r', os.path.abspath(results_dir),
+ '-b', '-u', self.owner, '-l', self.name,
+ '-m', hostnames, ctrl.name]
- if not self.is_server_job():
- params.append('-c')
+ if not self.is_server_job():
+ params.append('-c')
- tasks = []
- if not self.is_synchronous():
- tasks.append(VerifyTask(queue_entry))
+ tasks = []
+ if not self.is_synchronous():
+ tasks.append(VerifyTask(queue_entry))
- tasks.append(QueueTask(job = self,
- queue_entries = queue_entries,
- cmd = params))
+ tasks.append(QueueTask(job = self,
+ queue_entries = queue_entries,
+ cmd = params))
- ids = []
- for entry in queue_entries:
- ids.append(entry.id)
-
- agent = Agent(tasks, ids)
+ ids = []
+ for entry in queue_entries:
+ ids.append(entry.id)
- return agent
+ agent = Agent(tasks, ids)
+
+ return agent
if __name__ == '__main__':
- main()
+ main()
diff --git a/scheduler/monitor_db_unittest.py b/scheduler/monitor_db_unittest.py
index 68d71d2..c312636 100644
--- a/scheduler/monitor_db_unittest.py
+++ b/scheduler/monitor_db_unittest.py
@@ -31,610 +31,610 @@
"""
class Dummy(object):
- 'Dummy object that can have attribute assigned to it'
+ 'Dummy object that can have attribute assigned to it'
class DispatcherTest(unittest.TestCase):
- _jobs_scheduled = []
- _job_counter = 0
+ _jobs_scheduled = []
+ _job_counter = 0
- def _read_db_info(self):
- config = global_config.global_config
- section = 'AUTOTEST_WEB'
- self._host = config.get_config_value(section, "host")
- self._db_name = config.get_config_value(section, "database")
- self._user = config.get_config_value(section, "user")
- self._password = config.get_config_value(section, "password")
+ def _read_db_info(self):
+ config = global_config.global_config
+ section = 'AUTOTEST_WEB'
+ self._host = config.get_config_value(section, "host")
+ self._db_name = config.get_config_value(section, "database")
+ self._user = config.get_config_value(section, "user")
+ self._password = config.get_config_value(section, "password")
- def _connect_to_db(self, db_name=''):
- self._con = MySQLdb.connect(host=self._host, user=self._user,
- passwd=self._password, db=db_name)
- self._con.autocommit(True)
- self._cur = self._con.cursor()
+ def _connect_to_db(self, db_name=''):
+ self._con = MySQLdb.connect(host=self._host, user=self._user,
+ passwd=self._password, db=db_name)
+ self._con.autocommit(True)
+ self._cur = self._con.cursor()
- def _disconnect_from_db(self):
- self._con.close()
+ def _disconnect_from_db(self):
+ self._con.close()
- def _do_query(self, sql):
- if _DEBUG:
- print 'SQL:', sql
- self._cur.execute(sql)
+ def _do_query(self, sql):
+ if _DEBUG:
+ print 'SQL:', sql
+ self._cur.execute(sql)
- def _do_queries(self, sql_queries):
- for query in sql_queries.split(';'):
- query = query.strip()
- if query:
- self._do_query(query)
+ def _do_queries(self, sql_queries):
+ for query in sql_queries.split(';'):
+ query = query.strip()
+ if query:
+ self._do_query(query)
- def _get_db_schema(self):
- command = 'mysqldump --no-data -u %s -p%s -h %s %s' % (
- self._user, self._password, self._host, self._db_name)
- proc = subprocess.Popen(command, stdout=subprocess.PIPE,
- shell=True)
- return proc.communicate()[0]
+ def _get_db_schema(self):
+ command = 'mysqldump --no-data -u %s -p%s -h %s %s' % (
+ self._user, self._password, self._host, self._db_name)
+ proc = subprocess.Popen(command, stdout=subprocess.PIPE,
+ shell=True)
+ return proc.communicate()[0]
- def _open_test_db(self, schema):
- self._db_name = 'test_' + self._db_name
- self._connect_to_db()
- self._do_query('CREATE DATABASE ' + self._db_name)
- self._disconnect_from_db()
- self._connect_to_db(self._db_name)
- self._do_queries(schema)
+ def _open_test_db(self, schema):
+ self._db_name = 'test_' + self._db_name
+ self._connect_to_db()
+ self._do_query('CREATE DATABASE ' + self._db_name)
+ self._disconnect_from_db()
+ self._connect_to_db(self._db_name)
+ self._do_queries(schema)
- def _close_test_db(self):
- self._do_query('DROP DATABASE ' + self._db_name)
- self._disconnect_from_db()
+ def _close_test_db(self):
+ self._do_query('DROP DATABASE ' + self._db_name)
+ self._disconnect_from_db()
- def _fill_in_test_data(self):
- self._do_queries(_TEST_DATA)
+ def _fill_in_test_data(self):
+ self._do_queries(_TEST_DATA)
- def _set_monitor_stubs(self):
- monitor_db._db = monitor_db.DatabaseConn()
- monitor_db._db.connect(db_name=self._db_name)
- def run_stub(hqe_self, assigned_host=None):
- if hqe_self.meta_host:
- host = assigned_host
- else:
- host = hqe_self.host
- self._record_job_scheduled(hqe_self.job.id, host.id)
- return Dummy()
- monitor_db.HostQueueEntry.run = run_stub
+ def _set_monitor_stubs(self):
+ monitor_db._db = monitor_db.DatabaseConn()
+ monitor_db._db.connect(db_name=self._db_name)
+ def run_stub(hqe_self, assigned_host=None):
+ if hqe_self.meta_host:
+ host = assigned_host
+ else:
+ host = hqe_self.host
+ self._record_job_scheduled(hqe_self.job.id, host.id)
+ return Dummy()
+ monitor_db.HostQueueEntry.run = run_stub
- def _record_job_scheduled(self, job_id, host_id):
- record = (job_id, host_id)
- self.assert_(record not in self._jobs_scheduled,
- 'Job %d scheduled on host %d twice' %
- (job_id, host_id))
- self._jobs_scheduled.append(record)
+ def _record_job_scheduled(self, job_id, host_id):
+ record = (job_id, host_id)
+ self.assert_(record not in self._jobs_scheduled,
+ 'Job %d scheduled on host %d twice' %
+ (job_id, host_id))
+ self._jobs_scheduled.append(record)
- def _assert_job_scheduled_on(self, job_id, host_id):
- record = (job_id, host_id)
- self.assert_(record in self._jobs_scheduled,
- 'Job %d not scheduled on host %d as expected\n'
- 'Jobs scheduled: %s' %
- (job_id, host_id, self._jobs_scheduled))
- self._jobs_scheduled.remove(record)
+ def _assert_job_scheduled_on(self, job_id, host_id):
+ record = (job_id, host_id)
+ self.assert_(record in self._jobs_scheduled,
+ 'Job %d not scheduled on host %d as expected\n'
+ 'Jobs scheduled: %s' %
+ (job_id, host_id, self._jobs_scheduled))
+ self._jobs_scheduled.remove(record)
- def _check_for_extra_schedulings(self):
- if len(self._jobs_scheduled) != 0:
- self.fail('Extra jobs scheduled: ' +
- str(self._jobs_scheduled))
+ def _check_for_extra_schedulings(self):
+ if len(self._jobs_scheduled) != 0:
+ self.fail('Extra jobs scheduled: ' +
+ str(self._jobs_scheduled))
- def _create_job(self, hosts=[], metahosts=[], priority=0, active=0):
- self._do_query('INSERT INTO jobs (name, owner, priority) '
- 'VALUES ("test", "my_user", %d)' % priority)
- self._job_counter += 1
- job_id = self._job_counter
- queue_entry_sql = (
- 'INSERT INTO host_queue_entries '
- '(job_id, priority, host_id, meta_host, active) '
- 'VALUES (%d, %d, %%s, %%s, %d)' %
- (job_id, priority, active))
- for host_id in hosts:
- self._do_query(queue_entry_sql % (host_id, 'NULL'))
- self._do_query('INSERT INTO ineligible_host_queues '
- '(job_id, host_id) VALUES (%d, %d)' %
- (job_id, host_id))
- for label_id in metahosts:
- self._do_query(queue_entry_sql % ('NULL', label_id))
+ def _create_job(self, hosts=[], metahosts=[], priority=0, active=0):
+ self._do_query('INSERT INTO jobs (name, owner, priority) '
+ 'VALUES ("test", "my_user", %d)' % priority)
+ self._job_counter += 1
+ job_id = self._job_counter
+ queue_entry_sql = (
+ 'INSERT INTO host_queue_entries '
+ '(job_id, priority, host_id, meta_host, active) '
+ 'VALUES (%d, %d, %%s, %%s, %d)' %
+ (job_id, priority, active))
+ for host_id in hosts:
+ self._do_query(queue_entry_sql % (host_id, 'NULL'))
+ self._do_query('INSERT INTO ineligible_host_queues '
+ '(job_id, host_id) VALUES (%d, %d)' %
+ (job_id, host_id))
+ for label_id in metahosts:
+ self._do_query(queue_entry_sql % ('NULL', label_id))
- def _create_job_simple(self, hosts, use_metahost=False,
- priority=0, active=0):
- 'An alternative interface to _create_job'
- args = {'hosts' : [], 'metahosts' : []}
- if use_metahost:
- args['metahosts'] = hosts
- else:
- args['hosts'] = hosts
- self._create_job(priority=priority, active=active, **args)
+ def _create_job_simple(self, hosts, use_metahost=False,
+ priority=0, active=0):
+ 'An alternative interface to _create_job'
+ args = {'hosts' : [], 'metahosts' : []}
+ if use_metahost:
+ args['metahosts'] = hosts
+ else:
+ args['hosts'] = hosts
+ self._create_job(priority=priority, active=active, **args)
- def _convert_jobs_to_metahosts(self, *job_ids):
- sql_tuple = '(' + ','.join(str(i) for i in job_ids) + ')'
- self._do_query('UPDATE host_queue_entries SET '
- 'meta_host=host_id, host_id=NULL '
- 'WHERE job_id IN ' + sql_tuple)
+ def _convert_jobs_to_metahosts(self, *job_ids):
+ sql_tuple = '(' + ','.join(str(i) for i in job_ids) + ')'
+ self._do_query('UPDATE host_queue_entries SET '
+ 'meta_host=host_id, host_id=NULL '
+ 'WHERE job_id IN ' + sql_tuple)
- def _lock_host(self, host_id):
- self._do_query('UPDATE hosts SET locked=1 WHERE id=' +
- str(host_id))
+ def _lock_host(self, host_id):
+ self._do_query('UPDATE hosts SET locked=1 WHERE id=' +
+ str(host_id))
- def setUp(self):
- self._read_db_info()
- schema = self._get_db_schema()
- self._open_test_db(schema)
- self._fill_in_test_data()
- self._set_monitor_stubs()
- self._dispatcher = monitor_db.Dispatcher()
- self._jobs_scheduled = []
- self._job_counter = 0
+ def setUp(self):
+ self._read_db_info()
+ schema = self._get_db_schema()
+ self._open_test_db(schema)
+ self._fill_in_test_data()
+ self._set_monitor_stubs()
+ self._dispatcher = monitor_db.Dispatcher()
+ self._jobs_scheduled = []
+ self._job_counter = 0
- def tearDown(self):
- self._close_test_db()
+ def tearDown(self):
+ self._close_test_db()
- def _test_basic_scheduling_helper(self, use_metahosts):
- 'Basic nonmetahost scheduling'
- self._create_job_simple([1], use_metahosts)
- self._create_job_simple([2], use_metahosts)
- self._dispatcher._schedule_new_jobs()
- self._assert_job_scheduled_on(1, 1)
- self._assert_job_scheduled_on(2, 2)
- self._check_for_extra_schedulings()
+ def _test_basic_scheduling_helper(self, use_metahosts):
+ 'Basic nonmetahost scheduling'
+ self._create_job_simple([1], use_metahosts)
+ self._create_job_simple([2], use_metahosts)
+ self._dispatcher._schedule_new_jobs()
+ self._assert_job_scheduled_on(1, 1)
+ self._assert_job_scheduled_on(2, 2)
+ self._check_for_extra_schedulings()
- def _test_priorities_helper(self, use_metahosts):
- 'Test prioritization ordering'
- self._create_job_simple([1], use_metahosts)
- self._create_job_simple([2], use_metahosts)
- self._create_job_simple([1,2], use_metahosts)
- self._create_job_simple([1], use_metahosts, priority=1)
- self._dispatcher._schedule_new_jobs()
- self._assert_job_scheduled_on(4, 1) # higher priority
- self._assert_job_scheduled_on(2, 2) # earlier job over later
- self._check_for_extra_schedulings()
+ def _test_priorities_helper(self, use_metahosts):
+ 'Test prioritization ordering'
+ self._create_job_simple([1], use_metahosts)
+ self._create_job_simple([2], use_metahosts)
+ self._create_job_simple([1,2], use_metahosts)
+ self._create_job_simple([1], use_metahosts, priority=1)
+ self._dispatcher._schedule_new_jobs()
+ self._assert_job_scheduled_on(4, 1) # higher priority
+ self._assert_job_scheduled_on(2, 2) # earlier job over later
+ self._check_for_extra_schedulings()
- def _test_hosts_ready_helper(self, use_metahosts):
- """
- Only hosts that are status=Ready, unlocked and not invalid get
- scheduled.
- """
- self._create_job_simple([1], use_metahosts)
- self._do_query('UPDATE hosts SET status="Running" WHERE id=1')
- self._dispatcher._schedule_new_jobs()
- self._check_for_extra_schedulings()
+ def _test_hosts_ready_helper(self, use_metahosts):
+ """
+ Only hosts that are status=Ready, unlocked and not invalid get
+ scheduled.
+ """
+ self._create_job_simple([1], use_metahosts)
+ self._do_query('UPDATE hosts SET status="Running" WHERE id=1')
+ self._dispatcher._schedule_new_jobs()
+ self._check_for_extra_schedulings()
- self._do_query('UPDATE hosts SET status="Ready", locked=1 '
- 'WHERE id=1')
- self._dispatcher._schedule_new_jobs()
- self._check_for_extra_schedulings()
+ self._do_query('UPDATE hosts SET status="Ready", locked=1 '
+ 'WHERE id=1')
+ self._dispatcher._schedule_new_jobs()
+ self._check_for_extra_schedulings()
- self._do_query('UPDATE hosts SET locked=0, invalid=1 '
- 'WHERE id=1')
- self._dispatcher._schedule_new_jobs()
- self._check_for_extra_schedulings()
+ self._do_query('UPDATE hosts SET locked=0, invalid=1 '
+ 'WHERE id=1')
+ self._dispatcher._schedule_new_jobs()
+ self._check_for_extra_schedulings()
- def _test_hosts_idle_helper(self, use_metahosts):
- 'Only idle hosts get scheduled'
- self._create_job(hosts=[1], active=1)
- self._create_job_simple([1], use_metahosts)
- self._dispatcher._schedule_new_jobs()
- self._check_for_extra_schedulings()
+ def _test_hosts_idle_helper(self, use_metahosts):
+ 'Only idle hosts get scheduled'
+ self._create_job(hosts=[1], active=1)
+ self._create_job_simple([1], use_metahosts)
+ self._dispatcher._schedule_new_jobs()
+ self._check_for_extra_schedulings()
- def test_basic_scheduling(self):
- self._test_basic_scheduling_helper(False)
+ def test_basic_scheduling(self):
+ self._test_basic_scheduling_helper(False)
- def test_priorities(self):
- self._test_priorities_helper(False)
+ def test_priorities(self):
+ self._test_priorities_helper(False)
- def test_hosts_ready(self):
- self._test_hosts_ready_helper(False)
+ def test_hosts_ready(self):
+ self._test_hosts_ready_helper(False)
- def test_hosts_idle(self):
- self._test_hosts_idle_helper(False)
+ def test_hosts_idle(self):
+ self._test_hosts_idle_helper(False)
- def test_metahost_scheduling(self):
- 'Basic metahost scheduling'
- self._test_basic_scheduling_helper(True)
+ def test_metahost_scheduling(self):
+ 'Basic metahost scheduling'
+ self._test_basic_scheduling_helper(True)
- def test_metahost_priorities(self):
- self._test_priorities_helper(True)
+ def test_metahost_priorities(self):
+ self._test_priorities_helper(True)
- def test_metahost_hosts_ready(self):
- self._test_hosts_ready_helper(True)
+ def test_metahost_hosts_ready(self):
+ self._test_hosts_ready_helper(True)
- def test_metahost_hosts_idle(self):
- self._test_hosts_idle_helper(True)
+ def test_metahost_hosts_idle(self):
+ self._test_hosts_idle_helper(True)
- def test_nonmetahost_over_metahost(self):
- """
- Non-metahost entries should take priority over metahost entries
- for the same host
- """
- self._create_job(metahosts=[1])
- self._create_job(hosts=[1])
- self._dispatcher._schedule_new_jobs()
- self._assert_job_scheduled_on(2, 1)
- self._check_for_extra_schedulings()
+ def test_nonmetahost_over_metahost(self):
+ """
+ Non-metahost entries should take priority over metahost entries
+ for the same host
+ """
+ self._create_job(metahosts=[1])
+ self._create_job(hosts=[1])
+ self._dispatcher._schedule_new_jobs()
+ self._assert_job_scheduled_on(2, 1)
+ self._check_for_extra_schedulings()
- def test_metahosts_obey_blocks(self):
- """
- Metahosts can't get scheduled on hosts already scheduled for
- that job.
- """
- self._create_job(metahosts=[1], hosts=[1])
- # make the nonmetahost entry complete, so the metahost can try
- # to get scheduled
- self._do_query('UPDATE host_queue_entries SET complete = 1 '
- 'WHERE host_id=1')
- self._dispatcher._schedule_new_jobs()
- self._check_for_extra_schedulings()
+ def test_metahosts_obey_blocks(self):
+ """
+ Metahosts can't get scheduled on hosts already scheduled for
+ that job.
+ """
+ self._create_job(metahosts=[1], hosts=[1])
+ # make the nonmetahost entry complete, so the metahost can try
+ # to get scheduled
+ self._do_query('UPDATE host_queue_entries SET complete = 1 '
+ 'WHERE host_id=1')
+ self._dispatcher._schedule_new_jobs()
+ self._check_for_extra_schedulings()
- def test_metahosts_obey_ACLs(self):
- "ACL-inaccessible hosts can't get scheduled for metahosts"
- self._do_query('DELETE FROM acl_groups_hosts WHERE host_id=1')
- self._create_job(metahosts=[1])
- self._dispatcher._schedule_new_jobs()
- self._check_for_extra_schedulings()
+ def test_metahosts_obey_ACLs(self):
+ "ACL-inaccessible hosts can't get scheduled for metahosts"
+ self._do_query('DELETE FROM acl_groups_hosts WHERE host_id=1')
+ self._create_job(metahosts=[1])
+ self._dispatcher._schedule_new_jobs()
+ self._check_for_extra_schedulings()
class PidfileRunMonitorTest(unittest.TestCase):
- results_dir = '/test/path'
- pidfile_path = os.path.join(results_dir, monitor_db.AUTOSERV_PID_FILE)
- pid = 12345
- args = ('nice -n 10 autoserv -P 123-myuser/myhost -p -n '
- '-r ' + results_dir + ' -b -u myuser -l my-job-name '
- '-m myhost /tmp/filejx43Zi -c')
- bad_args = args.replace(results_dir, '/random/results/dir')
+ results_dir = '/test/path'
+ pidfile_path = os.path.join(results_dir, monitor_db.AUTOSERV_PID_FILE)
+ pid = 12345
+ args = ('nice -n 10 autoserv -P 123-myuser/myhost -p -n '
+ '-r ' + results_dir + ' -b -u myuser -l my-job-name '
+ '-m myhost /tmp/filejx43Zi -c')
+ bad_args = args.replace(results_dir, '/random/results/dir')
- def setUp(self):
- self.god = mock.mock_god()
- self.god.stub_function(monitor_db, 'open')
- self.god.stub_function(os.path, 'exists')
- self.god.stub_function(monitor_db.email_manager,
- 'enqueue_notify_email')
- self.monitor = monitor_db.PidfileRunMonitor(self.results_dir)
+ def setUp(self):
+ self.god = mock.mock_god()
+ self.god.stub_function(monitor_db, 'open')
+ self.god.stub_function(os.path, 'exists')
+ self.god.stub_function(monitor_db.email_manager,
+ 'enqueue_notify_email')
+ self.monitor = monitor_db.PidfileRunMonitor(self.results_dir)
- def tearDown(self):
- self.god.unstub_all()
+ def tearDown(self):
+ self.god.unstub_all()
- def set_not_yet_run(self):
- os.path.exists.expect_call(self.pidfile_path).and_return(False)
+ def set_not_yet_run(self):
+ os.path.exists.expect_call(self.pidfile_path).and_return(False)
- def setup_pidfile(self, pidfile_contents):
- os.path.exists.expect_call(self.pidfile_path).and_return(True)
- pidfile = StringIO.StringIO(pidfile_contents)
- monitor_db.open.expect_call(
- self.pidfile_path, 'r').and_return(pidfile)
+ def setup_pidfile(self, pidfile_contents):
+ os.path.exists.expect_call(self.pidfile_path).and_return(True)
+ pidfile = StringIO.StringIO(pidfile_contents)
+ monitor_db.open.expect_call(
+ self.pidfile_path, 'r').and_return(pidfile)
- def set_running(self):
- self.setup_pidfile(str(self.pid) + '\n')
+ def set_running(self):
+ self.setup_pidfile(str(self.pid) + '\n')
- def set_complete(self, error_code):
- self.setup_pidfile(str(self.pid) + '\n' +
- str(error_code) + '\n')
+ def set_complete(self, error_code):
+ self.setup_pidfile(str(self.pid) + '\n' +
+ str(error_code) + '\n')
- def _test_read_pidfile_helper(self, expected_pid, expected_exit_status):
- pid, exit_status = self.monitor.read_pidfile()
- self.assertEquals(pid, expected_pid)
- self.assertEquals(exit_status, expected_exit_status)
- self.god.check_playback()
+ def _test_read_pidfile_helper(self, expected_pid, expected_exit_status):
+ pid, exit_status = self.monitor.read_pidfile()
+ self.assertEquals(pid, expected_pid)
+ self.assertEquals(exit_status, expected_exit_status)
+ self.god.check_playback()
- def test_read_pidfile(self):
- self.set_not_yet_run()
- self._test_read_pidfile_helper(None, None)
+ def test_read_pidfile(self):
+ self.set_not_yet_run()
+ self._test_read_pidfile_helper(None, None)
- self.set_running()
- self._test_read_pidfile_helper(self.pid, None)
+ self.set_running()
+ self._test_read_pidfile_helper(self.pid, None)
- self.set_complete(123)
- self._test_read_pidfile_helper(self.pid, 123)
+ self.set_complete(123)
+ self._test_read_pidfile_helper(self.pid, 123)
- def test_read_pidfile_error(self):
- self.setup_pidfile('asdf')
- self.assertRaises(monitor_db.PidfileException,
- self.monitor.read_pidfile)
- self.god.check_playback()
+ def test_read_pidfile_error(self):
+ self.setup_pidfile('asdf')
+ self.assertRaises(monitor_db.PidfileException,
+ self.monitor.read_pidfile)
+ self.god.check_playback()
- def setup_proc_cmdline(self, args):
- proc_cmdline = args.replace(' ', '\x00')
- proc_file = StringIO.StringIO(proc_cmdline)
- monitor_db.open.expect_call(
- '/proc/%d/cmdline' % self.pid, 'r').and_return(proc_file)
+ def setup_proc_cmdline(self, args):
+ proc_cmdline = args.replace(' ', '\x00')
+ proc_file = StringIO.StringIO(proc_cmdline)
+ monitor_db.open.expect_call(
+ '/proc/%d/cmdline' % self.pid, 'r').and_return(proc_file)
- def setup_find_autoservs(self, process_dict):
- self.god.stub_class_method(monitor_db.Dispatcher,
- 'find_autoservs')
- monitor_db.Dispatcher.find_autoservs.expect_call().and_return(
- process_dict)
+ def setup_find_autoservs(self, process_dict):
+ self.god.stub_class_method(monitor_db.Dispatcher,
+ 'find_autoservs')
+ monitor_db.Dispatcher.find_autoservs.expect_call().and_return(
+ process_dict)
- def _test_get_pidfile_info_helper(self, expected_pid,
- expected_exit_status):
- pid, exit_status = self.monitor.get_pidfile_info()
- self.assertEquals(pid, expected_pid)
- self.assertEquals(exit_status, expected_exit_status)
- self.god.check_playback()
+ def _test_get_pidfile_info_helper(self, expected_pid,
+ expected_exit_status):
+ pid, exit_status = self.monitor.get_pidfile_info()
+ self.assertEquals(pid, expected_pid)
+ self.assertEquals(exit_status, expected_exit_status)
+ self.god.check_playback()
- def test_get_pidfile_info(self):
- 'normal cases for get_pidfile_info'
- # running
- self.set_running()
- self.setup_proc_cmdline(self.args)
- self._test_get_pidfile_info_helper(self.pid, None)
+ def test_get_pidfile_info(self):
+ 'normal cases for get_pidfile_info'
+ # running
+ self.set_running()
+ self.setup_proc_cmdline(self.args)
+ self._test_get_pidfile_info_helper(self.pid, None)
- # exited during check
- self.set_running()
- monitor_db.open.expect_call(
- '/proc/%d/cmdline' % self.pid, 'r').and_raises(IOError)
- self.set_complete(123) # pidfile gets read again
- self._test_get_pidfile_info_helper(self.pid, 123)
+ # exited during check
+ self.set_running()
+ monitor_db.open.expect_call(
+ '/proc/%d/cmdline' % self.pid, 'r').and_raises(IOError)
+ self.set_complete(123) # pidfile gets read again
+ self._test_get_pidfile_info_helper(self.pid, 123)
- # completed
- self.set_complete(123)
- self._test_get_pidfile_info_helper(self.pid, 123)
+ # completed
+ self.set_complete(123)
+ self._test_get_pidfile_info_helper(self.pid, 123)
- def test_get_pidfile_info_running_no_proc(self):
- 'pidfile shows process running, but no proc exists'
- # running but no proc
- self.set_running()
- monitor_db.open.expect_call(
- '/proc/%d/cmdline' % self.pid, 'r').and_raises(IOError)
- self.set_running()
- monitor_db.email_manager.enqueue_notify_email.expect_call(
- mock.is_string_comparator(), mock.is_string_comparator())
- self._test_get_pidfile_info_helper(self.pid, 1)
- self.assertTrue(self.monitor.lost_process)
+ def test_get_pidfile_info_running_no_proc(self):
+ 'pidfile shows process running, but no proc exists'
+ # running but no proc
+ self.set_running()
+ monitor_db.open.expect_call(
+ '/proc/%d/cmdline' % self.pid, 'r').and_raises(IOError)
+ self.set_running()
+ monitor_db.email_manager.enqueue_notify_email.expect_call(
+ mock.is_string_comparator(), mock.is_string_comparator())
+ self._test_get_pidfile_info_helper(self.pid, 1)
+ self.assertTrue(self.monitor.lost_process)
- def test_get_pidfile_info_not_yet_run(self):
- "pidfile hasn't been written yet"
- # process not running
- self.set_not_yet_run()
- self.setup_find_autoservs({})
- self._test_get_pidfile_info_helper(None, None)
+ def test_get_pidfile_info_not_yet_run(self):
+ "pidfile hasn't been written yet"
+ # process not running
+ self.set_not_yet_run()
+ self.setup_find_autoservs({})
+ self._test_get_pidfile_info_helper(None, None)
- # process running
- self.set_not_yet_run()
- self.setup_find_autoservs({self.pid : self.args})
- self._test_get_pidfile_info_helper(None, None)
+ # process running
+ self.set_not_yet_run()
+ self.setup_find_autoservs({self.pid : self.args})
+ self._test_get_pidfile_info_helper(None, None)
- # another process running under same pid
- self.set_not_yet_run()
- self.setup_find_autoservs({self.pid : self.bad_args})
- self._test_get_pidfile_info_helper(None, None)
+ # another process running under same pid
+ self.set_not_yet_run()
+ self.setup_find_autoservs({self.pid : self.bad_args})
+ self._test_get_pidfile_info_helper(None, None)
class AgentTest(unittest.TestCase):
- def setUp(self):
- self.god = mock.mock_god()
+ def setUp(self):
+ self.god = mock.mock_god()
- def tearDown(self):
- self.god.unstub_all()
+ def tearDown(self):
+ self.god.unstub_all()
- def test_agent(self):
- task1 = self.god.create_mock_class(monitor_db.AgentTask,
- 'task1')
- task2 = self.god.create_mock_class(monitor_db.AgentTask,
- 'task2')
- task3 = self.god.create_mock_class(monitor_db.AgentTask,
- 'task3')
+ def test_agent(self):
+ task1 = self.god.create_mock_class(monitor_db.AgentTask,
+ 'task1')
+ task2 = self.god.create_mock_class(monitor_db.AgentTask,
+ 'task2')
+ task3 = self.god.create_mock_class(monitor_db.AgentTask,
+ 'task3')
- task1.start.expect_call()
- task1.is_done.expect_call().and_return(False)
- task1.poll.expect_call()
- task1.is_done.expect_call().and_return(True)
- task1.is_done.expect_call().and_return(True)
- task1.success = True
+ task1.start.expect_call()
+ task1.is_done.expect_call().and_return(False)
+ task1.poll.expect_call()
+ task1.is_done.expect_call().and_return(True)
+ task1.is_done.expect_call().and_return(True)
+ task1.success = True
- task2.start.expect_call()
- task2.is_done.expect_call().and_return(True)
- task2.is_done.expect_call().and_return(True)
- task2.success = False
- task2.failure_tasks = [task3]
+ task2.start.expect_call()
+ task2.is_done.expect_call().and_return(True)
+ task2.is_done.expect_call().and_return(True)
+ task2.success = False
+ task2.failure_tasks = [task3]
- task3.start.expect_call()
- task3.is_done.expect_call().and_return(True)
- task3.is_done.expect_call().and_return(True)
- task3.success = True
+ task3.start.expect_call()
+ task3.is_done.expect_call().and_return(True)
+ task3.is_done.expect_call().and_return(True)
+ task3.success = True
- agent = monitor_db.Agent([task1, task2])
- agent.dispatcher = object()
- agent.start()
- while not agent.is_done():
- agent.tick()
- self.god.check_playback()
+ agent = monitor_db.Agent([task1, task2])
+ agent.dispatcher = object()
+ agent.start()
+ while not agent.is_done():
+ agent.tick()
+ self.god.check_playback()
class AgentTasksTest(unittest.TestCase):
- TEMP_DIR = '/temp/dir'
- HOSTNAME = 'myhost'
+ TEMP_DIR = '/temp/dir'
+ HOSTNAME = 'myhost'
- def setUp(self):
- self.god = mock.mock_god()
- self.god.stub_with(tempfile, 'mkdtemp',
- mock.mock_function('mkdtemp', self.TEMP_DIR))
- self.god.stub_class_method(monitor_db.RunMonitor, 'run')
- self.god.stub_class_method(monitor_db.RunMonitor, 'exit_code')
- self.host = self.god.create_mock_class(monitor_db.Host, 'host')
- self.host.hostname = self.HOSTNAME
- self.queue_entry = self.god.create_mock_class(
- monitor_db.HostQueueEntry, 'queue_entry')
- self.queue_entry.host = self.host
- self.queue_entry.meta_host = None
+ def setUp(self):
+ self.god = mock.mock_god()
+ self.god.stub_with(tempfile, 'mkdtemp',
+ mock.mock_function('mkdtemp', self.TEMP_DIR))
+ self.god.stub_class_method(monitor_db.RunMonitor, 'run')
+ self.god.stub_class_method(monitor_db.RunMonitor, 'exit_code')
+ self.host = self.god.create_mock_class(monitor_db.Host, 'host')
+ self.host.hostname = self.HOSTNAME
+ self.queue_entry = self.god.create_mock_class(
+ monitor_db.HostQueueEntry, 'queue_entry')
+ self.queue_entry.host = self.host
+ self.queue_entry.meta_host = None
- def tearDown(self):
- self.god.unstub_all()
+ def tearDown(self):
+ self.god.unstub_all()
- def run_task(self, task, success):
- """
- Do essentially what an Agent would do, but protect againt
- infinite looping from test errors.
- """
- if not getattr(task, 'agent', None):
- task.agent = object()
- task.start()
- count = 0
- while not task.is_done():
- count += 1
- if count > 10:
- print 'Task failed to finish'
- # in case the playback has clues to why it
- # failed
- self.god.check_playback()
- self.fail()
- task.poll()
- self.assertEquals(task.success, success)
+ def run_task(self, task, success):
+ """
+ Do essentially what an Agent would do, but protect againt
+ infinite looping from test errors.
+ """
+ if not getattr(task, 'agent', None):
+ task.agent = object()
+ task.start()
+ count = 0
+ while not task.is_done():
+ count += 1
+ if count > 10:
+ print 'Task failed to finish'
+ # in case the playback has clues to why it
+ # failed
+ self.god.check_playback()
+ self.fail()
+ task.poll()
+ self.assertEquals(task.success, success)
- def setup_run_monitor(self, exit_status):
- monitor_db.RunMonitor.run.expect_call()
- monitor_db.RunMonitor.exit_code.expect_call()
- monitor_db.RunMonitor.exit_code.expect_call().and_return(
- exit_status)
+ def setup_run_monitor(self, exit_status):
+ monitor_db.RunMonitor.run.expect_call()
+ monitor_db.RunMonitor.exit_code.expect_call()
+ monitor_db.RunMonitor.exit_code.expect_call().and_return(
+ exit_status)
- def _test_repair_task_helper(self, success):
- self.host.set_status.expect_call('Repairing')
- if success:
- self.setup_run_monitor(0)
- self.host.set_status.expect_call('Ready')
- else:
- self.setup_run_monitor(1)
- self.host.set_status.expect_call('Repair Failed')
+ def _test_repair_task_helper(self, success):
+ self.host.set_status.expect_call('Repairing')
+ if success:
+ self.setup_run_monitor(0)
+ self.host.set_status.expect_call('Ready')
+ else:
+ self.setup_run_monitor(1)
+ self.host.set_status.expect_call('Repair Failed')
- task = monitor_db.RepairTask(self.host)
- self.run_task(task, success)
- self.assertEquals(task.monitor.cmd,
- ['autoserv', '-R', '-m', self.HOSTNAME, '-r',
- self.TEMP_DIR])
- self.god.check_playback()
+ task = monitor_db.RepairTask(self.host)
+ self.run_task(task, success)
+ self.assertEquals(task.monitor.cmd,
+ ['autoserv', '-R', '-m', self.HOSTNAME, '-r',
+ self.TEMP_DIR])
+ self.god.check_playback()
- def test_repair_task(self):
- self._test_repair_task_helper(True)
- self._test_repair_task_helper(False)
+ def test_repair_task(self):
+ self._test_repair_task_helper(True)
+ self._test_repair_task_helper(False)
- def test_repair_task_with_queue_entry(self):
- queue_entry = self.god.create_mock_class(
- monitor_db.HostQueueEntry, 'queue_entry')
- self.host.set_status.expect_call('Repairing')
- self.setup_run_monitor(1)
- self.host.set_status.expect_call('Repair Failed')
- queue_entry.handle_host_failure.expect_call()
+ def test_repair_task_with_queue_entry(self):
+ queue_entry = self.god.create_mock_class(
+ monitor_db.HostQueueEntry, 'queue_entry')
+ self.host.set_status.expect_call('Repairing')
+ self.setup_run_monitor(1)
+ self.host.set_status.expect_call('Repair Failed')
+ queue_entry.handle_host_failure.expect_call()
- task = monitor_db.RepairTask(self.host, queue_entry)
- self.run_task(task, False)
- self.god.check_playback()
+ task = monitor_db.RepairTask(self.host, queue_entry)
+ self.run_task(task, False)
+ self.god.check_playback()
- def setup_verify_expects(self, success, use_queue_entry):
- if use_queue_entry:
- self.queue_entry.set_status.expect_call('Verifying')
- self.queue_entry.verify_results_dir.expect_call(
- ).and_return('/verify/results/dir')
- self.queue_entry.clear_results_dir.expect_call(
- '/verify/results/dir')
- self.host.set_status.expect_call('Verifying')
- if success:
- self.setup_run_monitor(0)
- self.host.set_status.expect_call('Ready')
- else:
- self.setup_run_monitor(1)
- if use_queue_entry:
- self.queue_entry.requeue.expect_call()
+ def setup_verify_expects(self, success, use_queue_entry):
+ if use_queue_entry:
+ self.queue_entry.set_status.expect_call('Verifying')
+ self.queue_entry.verify_results_dir.expect_call(
+ ).and_return('/verify/results/dir')
+ self.queue_entry.clear_results_dir.expect_call(
+ '/verify/results/dir')
+ self.host.set_status.expect_call('Verifying')
+ if success:
+ self.setup_run_monitor(0)
+ self.host.set_status.expect_call('Ready')
+ else:
+ self.setup_run_monitor(1)
+ if use_queue_entry:
+ self.queue_entry.requeue.expect_call()
- def _test_verify_task_with_host_helper(self, success, use_queue_entry):
- self.setup_verify_expects(success, use_queue_entry)
+ def _test_verify_task_with_host_helper(self, success, use_queue_entry):
+ self.setup_verify_expects(success, use_queue_entry)
- if use_queue_entry:
- task = monitor_db.VerifyTask(
- queue_entry=self.queue_entry)
- else:
- task = monitor_db.VerifyTask(host=self.host)
- self.run_task(task, success)
- self.assertEquals(task.monitor.cmd,
- ['autoserv', '-v', '-m', self.HOSTNAME, '-r',
- self.TEMP_DIR])
- self.god.check_playback()
+ if use_queue_entry:
+ task = monitor_db.VerifyTask(
+ queue_entry=self.queue_entry)
+ else:
+ task = monitor_db.VerifyTask(host=self.host)
+ self.run_task(task, success)
+ self.assertEquals(task.monitor.cmd,
+ ['autoserv', '-v', '-m', self.HOSTNAME, '-r',
+ self.TEMP_DIR])
+ self.god.check_playback()
- def test_verify_task_with_host(self):
- self._test_verify_task_with_host_helper(True, False)
- self._test_verify_task_with_host_helper(False, False)
+ def test_verify_task_with_host(self):
+ self._test_verify_task_with_host_helper(True, False)
+ self._test_verify_task_with_host_helper(False, False)
- def test_verify_task_with_queue_entry(self):
- self._test_verify_task_with_host_helper(True, True)
- self._test_verify_task_with_host_helper(False, True)
+ def test_verify_task_with_queue_entry(self):
+ self._test_verify_task_with_host_helper(True, True)
+ self._test_verify_task_with_host_helper(False, True)
- def test_verify_synchronous_task(self):
- job = self.god.create_mock_class(monitor_db.Job, 'job')
+ def test_verify_synchronous_task(self):
+ job = self.god.create_mock_class(monitor_db.Job, 'job')
- self.setup_verify_expects(True, True)
- job.num_complete.expect_call().and_return(0)
- self.queue_entry.set_status.expect_call('Pending')
- job.is_ready.expect_call().and_return(True)
- job.run.expect_call(self.queue_entry)
- self.queue_entry.job = job
+ self.setup_verify_expects(True, True)
+ job.num_complete.expect_call().and_return(0)
+ self.queue_entry.set_status.expect_call('Pending')
+ job.is_ready.expect_call().and_return(True)
+ job.run.expect_call(self.queue_entry)
+ self.queue_entry.job = job
- task = monitor_db.VerifySynchronousTask(self.queue_entry)
- task.agent = Dummy()
- task.agent.dispatcher = Dummy()
- self.god.stub_with(task.agent.dispatcher, 'add_agent',
- mock.mock_function('add_agent'))
- self.run_task(task, True)
- self.god.check_playback()
+ task = monitor_db.VerifySynchronousTask(self.queue_entry)
+ task.agent = Dummy()
+ task.agent.dispatcher = Dummy()
+ self.god.stub_with(task.agent.dispatcher, 'add_agent',
+ mock.mock_function('add_agent'))
+ self.run_task(task, True)
+ self.god.check_playback()
if __name__ == '__main__':
- unittest.main()
+ unittest.main()
diff --git a/scheduler/monitor_queue b/scheduler/monitor_queue
index df058b4..74eaca1 100755
--- a/scheduler/monitor_queue
+++ b/scheduler/monitor_queue
@@ -5,35 +5,35 @@
import tempfile
if (len(sys.argv) < 3):
- print "Usage: monitor_queue <spool_directory> <resultsdir> [<conmux_server>]"
- sys.exit(1)
+ print "Usage: monitor_queue <spool_directory> <resultsdir> [<conmux_server>]"
+ sys.exit(1)
(spooldir, resultsdir) = [os.path.abspath(p) for p in sys.argv[1:3]]
queue_name = os.path.basename(spooldir)
dotmachines = os.path.join(spooldir, '.machines')
if os.path.exists(dotmachines):
- machines = [l.strip() for l in open(dotmachines).readlines() if len(l.strip())]
+ machines = [l.strip() for l in open(dotmachines).readlines() if len(l.strip())]
else:
- print "No .machines file in %s, assuming queue name is a machine"\
- % queue_name
- machines = [queue_name]
+ print "No .machines file in %s, assuming queue name is a machine"\
+ % queue_name
+ machines = [queue_name]
if len(sys.argv) == 5:
- console = sys.argv[4]
+ console = sys.argv[4]
else:
- console = None
+ console = None
if not os.path.exists(spooldir):
- print "spooldir %s does not exist" % spooldir
- sys.exit(1)
+ print "spooldir %s does not exist" % spooldir
+ sys.exit(1)
if not os.path.exists(resultsdir):
- print "resultsdir %s does not exist" % resultsdir
- sys.exit(1)
+ print "resultsdir %s does not exist" % resultsdir
+ sys.exit(1)
##### Control file templates #####
SERV_MULTI = """# monitor_queue generated autoserv file (SERV_MULTI template)
hosts = [hosts.SSHHost(hostname, server=%s)
- for hostname in machines]
+ for hostname in machines]
at = autotest.Autotest()
@@ -41,16 +41,16 @@
results = %s
def install_run(host):
- at.install(host)
- host_results = os.path.join(results, host.hostname)
- at.run(control_path, host_results, host)
+ at.install(host)
+ host_results = os.path.join(results, host.hostname)
+ at.run(control_path, host_results, host)
parallel([subcommand(install_run, [host]) for host in hosts])"""
SERV_SINGLE = """# monitor_queue generated autoserv file (SERV_SINGLE template)
host = hosts.SSHHost(machines[0], server=%s)
-
+
at = autotest.Autotest()
control_path = %s
@@ -62,99 +62,99 @@
##### End control file templates #####
def pick_job(jobs):
- """Pick the next job to run. Currently we just pick the oldest job
- However, this would be the place to put prioritizations."""
- if not jobs:
- return None
- return sorted(jobs, key=lambda x:os.stat(x).st_mtime, reverse=True)[0]
+ """Pick the next job to run. Currently we just pick the oldest job
+ However, this would be the place to put prioritizations."""
+ if not jobs:
+ return None
+ return sorted(jobs, key=lambda x:os.stat(x).st_mtime, reverse=True)[0]
def __create_autoserv_wrapper(template, control_path, results):
- """Create an autoserv file that runs an autotest file at
- control_path on clients and outputs the results in results."""
- # Create an autoserv control file to run this autotest control file
- tmpfd, tmpname = tempfile.mkstemp()
- tmp = os.fdopen(tmpfd, 'w')
+ """Create an autoserv file that runs an autotest file at
+ control_path on clients and outputs the results in results."""
+ # Create an autoserv control file to run this autotest control file
+ tmpfd, tmpname = tempfile.mkstemp()
+ tmp = os.fdopen(tmpfd, 'w')
- print >> tmp, template % tuple([repr(s) for s in (console,
- control_path,
- results)])
- return tmpname
-
+ print >> tmp, template % tuple([repr(s) for s in (console,
+ control_path,
+ results)])
+ return tmpname
+
def run_job(control, queuename, scheduler_dir):
- """Runs a control file from the spooldir.
- Args:
- control: A path to a control file. It is assumed to be an
- Autotest control file in which case it will automatically
- be wrapped with autoserv control commands and run with
- autoserv. If the file name ends with .srv the wrapping
- procedure will be skipped and the autoserv file will be
- run directly.
+ """Runs a control file from the spooldir.
+ Args:
+ control: A path to a control file. It is assumed to be an
+ Autotest control file in which case it will automatically
+ be wrapped with autoserv control commands and run with
+ autoserv. If the file name ends with .srv the wrapping
+ procedure will be skipped and the autoserv file will be
+ run directly.
- Return:
- The return code from the autoserv process.
- """
- # Make sure all the output directories are all setup
- results = os.path.join(resultsdir, queuename + '-' + control)
- if os.path.exists(results):
- print "Resultsdir %s already present, " % results,
- results = "%s.%d" % (results, int(time.time()))
- print "changing to %s" % results
- os.mkdir(results)
- debug = os.path.join(results, 'debug')
- os.mkdir(debug)
+ Return:
+ The return code from the autoserv process.
+ """
+ # Make sure all the output directories are all setup
+ results = os.path.join(resultsdir, queuename + '-' + control)
+ if os.path.exists(results):
+ print "Resultsdir %s already present, " % results,
+ results = "%s.%d" % (results, int(time.time()))
+ print "changing to %s" % results
+ os.mkdir(results)
+ debug = os.path.join(results, 'debug')
+ os.mkdir(debug)
- # If this is an autoserv file then don't create the wrapper control
- is_autoserv_ctl = control.endswith('.srv')
- control_path = os.path.abspath(os.path.join(spooldir, control))
- # Otherwise create a tmp autoserv file just to launch the AT ctl file
- if not is_autoserv_ctl:
- if len(machines) > 1:
- # Run autotest file on *all* machines in *parallel*
- template = SERV_MULTI
- else:
- # Run autotest file on *one* machine
- template = SERV_SINGLE
- control_path = __create_autoserv_wrapper(template,
- control_path,
- results)
+ # If this is an autoserv file then don't create the wrapper control
+ is_autoserv_ctl = control.endswith('.srv')
+ control_path = os.path.abspath(os.path.join(spooldir, control))
+ # Otherwise create a tmp autoserv file just to launch the AT ctl file
+ if not is_autoserv_ctl:
+ if len(machines) > 1:
+ # Run autotest file on *all* machines in *parallel*
+ template = SERV_MULTI
+ else:
+ # Run autotest file on *one* machine
+ template = SERV_SINGLE
+ control_path = __create_autoserv_wrapper(template,
+ control_path,
+ results)
- # Now run the job
- autoserv_exe = os.path.join(scheduler_dir, '..', 'server', 'autoserv')
- autoserv_exe = os.path.abspath(autoserv_exe)
+ # Now run the job
+ autoserv_exe = os.path.join(scheduler_dir, '..', 'server', 'autoserv')
+ autoserv_exe = os.path.abspath(autoserv_exe)
- autoserv_cmd = ' '.join([autoserv_exe, '-m', ','.join(machines),
- control_path])
+ autoserv_cmd = ' '.join([autoserv_exe, '-m', ','.join(machines),
+ control_path])
- print "Starting job: %s" % control
- print autoserv_cmd
-
- open(os.path.join(debug, 'autoserv.cmd'), 'w', 0).write(autoserv_cmd + '\n')
- autoserv_stdout = open(os.path.join(debug, 'autoserv.stdout'), 'w', 0)
- autoserv_stderr = open(os.path.join(debug, 'autoserv.stderr'), 'w', 0)
- p = Popen(autoserv_cmd, shell=True, stdout=autoserv_stdout,
- stderr=autoserv_stderr, cwd=results)
- (pid, ret) = os.waitpid(p.pid, 0)
- autoserv_stdout.close()
- autoserv_stderr.close()
-
- # If this was a tempfile then clean it up
- if not is_autoserv_ctl:
- os.unlink(control_path)
- print "Completed job: %s (%d) " % (control, ret)
-
- return ret
-
+ print "Starting job: %s" % control
+ print autoserv_cmd
+
+ open(os.path.join(debug, 'autoserv.cmd'), 'w', 0).write(autoserv_cmd + '\n')
+ autoserv_stdout = open(os.path.join(debug, 'autoserv.stdout'), 'w', 0)
+ autoserv_stderr = open(os.path.join(debug, 'autoserv.stderr'), 'w', 0)
+ p = Popen(autoserv_cmd, shell=True, stdout=autoserv_stdout,
+ stderr=autoserv_stderr, cwd=results)
+ (pid, ret) = os.waitpid(p.pid, 0)
+ autoserv_stdout.close()
+ autoserv_stderr.close()
+
+ # If this was a tempfile then clean it up
+ if not is_autoserv_ctl:
+ os.unlink(control_path)
+ print "Completed job: %s (%d) " % (control, ret)
+
+ return ret
+
scheduler_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
os.chdir(spooldir)
print "monitoring spool directory: " + spooldir
while True:
- jobs = [j for j in os.listdir(spooldir) if not j.startswith('.')]
- next_job = pick_job(jobs)
- if not next_job:
- time.sleep(10)
- continue
- ret = run_job(next_job, os.path.basename(spooldir), scheduler_dir)
- os.remove(next_job)
+ jobs = [j for j in os.listdir(spooldir) if not j.startswith('.')]
+ next_job = pick_job(jobs)
+ if not next_job:
+ time.sleep(10)
+ continue
+ ret = run_job(next_job, os.path.basename(spooldir), scheduler_dir)
+ os.remove(next_job)
diff --git a/scheduler/start_all_queues b/scheduler/start_all_queues
index ba1b219..d3bbf34 100755
--- a/scheduler/start_all_queues
+++ b/scheduler/start_all_queues
@@ -6,8 +6,8 @@
monitor_queue = os.path.join(dir, 'monitor_queue')
if len(sys.argv) < 2:
- print "Usage: start_all_queues <top_level_dir>"
- sys.exit(1)
+ print "Usage: start_all_queues <top_level_dir>"
+ sys.exit(1)
os.system("killall monitor_queue")
@@ -16,20 +16,20 @@
results_dir = os.path.join(top_dir, 'results')
for machine in os.listdir(queue_dir):
- queue = os.path.join(queue_dir, machine)
- if not os.path.isdir(queue):
- continue
+ queue = os.path.join(queue_dir, machine)
+ if not os.path.isdir(queue):
+ continue
- if not os.path.exists(results_dir):
- print "No results directory: %s" % results_dir
- sys.exit(1)
+ if not os.path.exists(results_dir):
+ print "No results directory: %s" % results_dir
+ sys.exit(1)
- console_path = "/usr/local/conmux/etc/%s.cf" % machine
- if os.path.exists(console_path):
- console = machine
- else:
- console = ''
+ console_path = "/usr/local/conmux/etc/%s.cf" % machine
+ if os.path.exists(console_path):
+ console = machine
+ else:
+ console = ''
- cmd = ' '.join([monitor_queue, queue, results_dir, console])
- print cmd
- os.system("nohup %s > %s/.log 2>&1 &" % (cmd,queue))
+ cmd = ' '.join([monitor_queue, queue, results_dir, console])
+ print cmd
+ os.system("nohup %s > %s/.log 2>&1 &" % (cmd,queue))
diff --git a/server/autoserv b/server/autoserv
index bf7dfa1..bab1a58 100755
--- a/server/autoserv
+++ b/server/autoserv
@@ -18,155 +18,155 @@
class PidFileManager(object):
- pid_file = None
+ pid_file = None
- def open_pid_file(self, results_dir):
- pid_file_path = os.path.join(results_dir, '.autoserv_execute')
- assert not os.path.exists(pid_file_path)
- self.pid_file = open(pid_file_path, 'w')
- self.pid_file.write(str(os.getpid()) + '\n')
- self.pid_file.flush()
+ def open_pid_file(self, results_dir):
+ pid_file_path = os.path.join(results_dir, '.autoserv_execute')
+ assert not os.path.exists(pid_file_path)
+ self.pid_file = open(pid_file_path, 'w')
+ self.pid_file.write(str(os.getpid()) + '\n')
+ self.pid_file.flush()
- def close_pid_file(self, exit_code, signal_code=0):
- if not self.pid_file:
- return
- real_exit_code = (exit_code << 8) | (signal_code & 0xFF)
- self.pid_file.write(str(real_exit_code) + '\n')
- self.pid_file.close()
- self.pid_file = None
+ def close_pid_file(self, exit_code, signal_code=0):
+ if not self.pid_file:
+ return
+ real_exit_code = (exit_code << 8) | (signal_code & 0xFF)
+ self.pid_file.write(str(real_exit_code) + '\n')
+ self.pid_file.close()
+ self.pid_file = None
def run_autoserv(pid_file_manager, results, parser):
- # send stdin to /dev/null
- dev_null = os.open(os.devnull, os.O_RDONLY)
- os.dup2(dev_null, sys.stdin.fileno())
- os.close(dev_null)
+ # send stdin to /dev/null
+ dev_null = os.open(os.devnull, os.O_RDONLY)
+ os.dup2(dev_null, sys.stdin.fileno())
+ os.close(dev_null)
- # Create separate process group
- os.setpgrp()
+ # Create separate process group
+ os.setpgrp()
- # Implement SIGTERM handler
- def handle_sigint(signum, frame):
- pid_file_manager.close_pid_file(1, signal.SIGTERM)
- os.killpg(os.getpgrp(), signal.SIGKILL)
+ # Implement SIGTERM handler
+ def handle_sigint(signum, frame):
+ pid_file_manager.close_pid_file(1, signal.SIGTERM)
+ os.killpg(os.getpgrp(), signal.SIGKILL)
- # Set signal handler
- signal.signal(signal.SIGTERM, handle_sigint)
+ # Set signal handler
+ signal.signal(signal.SIGTERM, handle_sigint)
- # Get a useful value for running 'USER'
- realuser = os.environ.get('USER')
- if not realuser:
- realuser = 'anonymous'
+ # Get a useful value for running 'USER'
+ realuser = os.environ.get('USER')
+ if not realuser:
+ realuser = 'anonymous'
- machines = parser.options.machines.split(',')
- machines_file = parser.options.machines_file
- label = parser.options.label
- user = parser.options.user
- client = parser.options.client
- reboot = parser.options.reboot
- install_before = parser.options.install_before
- install_after = parser.options.install_after
- verify = parser.options.verify
- repair = parser.options.repair
- no_tee = parser.options.no_tee
- parse_job = parser.options.parse_job
- ssh_user = parser.options.ssh_user
- ssh_port = parser.options.ssh_port
- ssh_pass = parser.options.ssh_pass
+ machines = parser.options.machines.split(',')
+ machines_file = parser.options.machines_file
+ label = parser.options.label
+ user = parser.options.user
+ client = parser.options.client
+ reboot = parser.options.reboot
+ install_before = parser.options.install_before
+ install_after = parser.options.install_after
+ verify = parser.options.verify
+ repair = parser.options.repair
+ no_tee = parser.options.no_tee
+ parse_job = parser.options.parse_job
+ ssh_user = parser.options.ssh_user
+ ssh_port = parser.options.ssh_port
+ ssh_pass = parser.options.ssh_pass
- if len(parser.args) < 1 and not verify and not repair:
- print parser.parser.print_help()
- sys.exit(1)
+ if len(parser.args) < 1 and not verify and not repair:
+ print parser.parser.print_help()
+ sys.exit(1)
- # We have a control file unless it's just a verify/repair job
- if len(parser.args) > 0:
- control = parser.args[0]
- else:
- control = None
+ # We have a control file unless it's just a verify/repair job
+ if len(parser.args) > 0:
+ control = parser.args[0]
+ else:
+ control = None
- if machines_file:
- machines = []
- for m in open(machines_file, 'r').readlines():
- # remove comments, spaces
- m = re.sub('#.*', '', m).strip()
- if m:
- machines.append(m)
- print "Read list of machines from file: %s" % machines_file
- print ','.join(machines)
+ if machines_file:
+ machines = []
+ for m in open(machines_file, 'r').readlines():
+ # remove comments, spaces
+ m = re.sub('#.*', '', m).strip()
+ if m:
+ machines.append(m)
+ print "Read list of machines from file: %s" % machines_file
+ print ','.join(machines)
- if machines:
- for machine in machines:
- if not machine or re.search('\s', machine):
- print "Invalid machine %s" % str(machine)
- sys.exit(1)
- machines = list(set(machines))
- machines.sort()
+ if machines:
+ for machine in machines:
+ if not machine or re.search('\s', machine):
+ print "Invalid machine %s" % str(machine)
+ sys.exit(1)
+ machines = list(set(machines))
+ machines.sort()
- job = server_job.server_job(control, parser.args[1:], results, label,
- user, machines, client, parse_job,
- ssh_user, ssh_port, ssh_pass)
- debug_dir = os.path.join(results, 'debug')
- stdout = os.path.join(debug_dir, 'autoserv.stdout')
- stderr = os.path.join(debug_dir, 'autoserv.stderr')
- if no_tee:
- job.stdout.redirect(stdout)
- job.stderr.redirect(stderr)
- else:
- job.stdout.tee_redirect(stdout)
- job.stderr.tee_redirect(stderr)
+ job = server_job.server_job(control, parser.args[1:], results, label,
+ user, machines, client, parse_job,
+ ssh_user, ssh_port, ssh_pass)
+ debug_dir = os.path.join(results, 'debug')
+ stdout = os.path.join(debug_dir, 'autoserv.stdout')
+ stderr = os.path.join(debug_dir, 'autoserv.stderr')
+ if no_tee:
+ job.stdout.redirect(stdout)
+ job.stderr.redirect(stderr)
+ else:
+ job.stdout.tee_redirect(stdout)
+ job.stderr.tee_redirect(stderr)
- # run the job
- exit_code = 0
- try:
- if repair:
- job.repair()
- elif verify:
- job.verify()
- else:
- try:
- job.run(reboot, install_before, install_after)
- finally:
- job.cleanup_parser()
- except:
- job.aborted = True
- traceback.print_exc()
+ # run the job
+ exit_code = 0
+ try:
+ if repair:
+ job.repair()
+ elif verify:
+ job.verify()
+ else:
+ try:
+ job.run(reboot, install_before, install_after)
+ finally:
+ job.cleanup_parser()
+ except:
+ job.aborted = True
+ traceback.print_exc()
- if getattr(job, 'aborted', False):
- sys.exit(1)
+ if getattr(job, 'aborted', False):
+ sys.exit(1)
def main():
- pid_file_manager = PidFileManager()
+ pid_file_manager = PidFileManager()
- # grab the parser
- parser = autoserv_parser.autoserv_parser
+ # grab the parser
+ parser = autoserv_parser.autoserv_parser
- if len(sys.argv) == 1:
- parser.parser.print_help()
- sys.exit(1)
+ if len(sys.argv) == 1:
+ parser.parser.print_help()
+ sys.exit(1)
- results = parser.options.results
- results = os.path.abspath(results)
- write_pidfile = parser.options.write_pidfile
- if write_pidfile:
- pid_file_manager.open_pid_file(results)
+ results = parser.options.results
+ results = os.path.abspath(results)
+ write_pidfile = parser.options.write_pidfile
+ if write_pidfile:
+ pid_file_manager.open_pid_file(results)
- exit_code = 0
- try:
- try:
- run_autoserv(pid_file_manager, results, parser)
- except SystemExit, e:
- exit_code = e.code
- except:
- traceback.print_exc()
- # If we don't know what happened, we'll classify it as
- # an 'abort' and return 1.
- exit_code = 1
- finally:
- pid_file_manager.close_pid_file(exit_code)
- sys.exit(exit_code)
+ exit_code = 0
+ try:
+ try:
+ run_autoserv(pid_file_manager, results, parser)
+ except SystemExit, e:
+ exit_code = e.code
+ except:
+ traceback.print_exc()
+ # If we don't know what happened, we'll classify it as
+ # an 'abort' and return 1.
+ exit_code = 1
+ finally:
+ pid_file_manager.close_pid_file(exit_code)
+ sys.exit(exit_code)
if __name__ == '__main__':
- main()
+ main()
diff --git a/server/autoserv_parser.py b/server/autoserv_parser.py
index 43e859f..b32e889 100644
--- a/server/autoserv_parser.py
+++ b/server/autoserv_parser.py
@@ -5,97 +5,97 @@
class base_autoserv_parser(object):
- """Custom command-line options parser for autoserv.
+ """Custom command-line options parser for autoserv.
- We can't use the general getopt methods here, as there will be unknown
- extra arguments that we pass down into the control file instead.
- Thus we process the arguments by hand, for which we are duly repentant.
- Making a single function here just makes it harder to read. Suck it up.
- """
- def __init__(self):
- self.args = sys.argv[1:]
- self.parser = optparse.OptionParser()
- self.setup_options()
- self.parse_args()
-
-
- def setup_options(self):
- self.parser.add_option("-m", action="store", type="string",
- dest="machines",
- help="list of machines")
- self.parser.add_option("-M", action="store", type="string",
- dest="machines_file",
- help="list of machines from file")
- self.parser.add_option("-c", action="store_true",
- dest="client", default=False,
- help="control file is client side")
- self.parser.add_option("-r", action="store", type="string",
- dest="results", default='.',
- help="specify results directory")
- self.parser.add_option("-l", action="store", type="string",
- dest="label", default='',
- help="label for the job")
- self.parser.add_option("-u", action="store", type="string",
- dest="user",
- default=os.environ.get('USER'),
- help="username for the job")
- self.parser.add_option("-P", action="store", type="string",
- dest="parse_job",
- default='',
- help="parse the results of the job")
- self.parser.add_option("-i", action="store_true",
- dest="install_before", default=False,
- help="reinstall machines before running the job")
- self.parser.add_option("-I", action="store_true",
- dest="install_after", default=False,
- help="reinstall machines after running the job")
- self.parser.add_option("-b", action="store_true",
- dest="reboot", default=False,
- help="reboot all machines after job")
- self.parser.add_option("-v", action="store_true",
- dest="verify", default=False,
- help="verify the machines only")
- self.parser.add_option("-R", action="store_true",
- dest="repair", default=False,
- help="repair the machines")
- self.parser.add_option("-n", action="store_true",
- dest="no_tee", default=False,
- help="no teeing the status to stdout/err")
- self.parser.add_option("-p", action="store_true",
- dest="write_pidfile", default=False,
- help="write pidfile (.autoserv_execute)")
- self.parser.add_option("--ssh-user", action="store",
- type="string", dest="ssh_user",
- default="root",
- help=("specify the user for ssh"
- "connections"))
- self.parser.add_option("--ssh-port", action="store",
- type="int", dest="ssh_port",
- default=22,
- help=("specify the port to use for "
- "ssh connections"))
- self.parser.add_option("--ssh-pass", action="store",
- type="string", dest="ssh_pass",
- default="",
- help=("specify the password to use "
- "for ssh connections"))
+ We can't use the general getopt methods here, as there will be unknown
+ extra arguments that we pass down into the control file instead.
+ Thus we process the arguments by hand, for which we are duly repentant.
+ Making a single function here just makes it harder to read. Suck it up.
+ """
+ def __init__(self):
+ self.args = sys.argv[1:]
+ self.parser = optparse.OptionParser()
+ self.setup_options()
+ self.parse_args()
- def parse_args(self):
- (self.options, self.args) = self.parser.parse_args()
+ def setup_options(self):
+ self.parser.add_option("-m", action="store", type="string",
+ dest="machines",
+ help="list of machines")
+ self.parser.add_option("-M", action="store", type="string",
+ dest="machines_file",
+ help="list of machines from file")
+ self.parser.add_option("-c", action="store_true",
+ dest="client", default=False,
+ help="control file is client side")
+ self.parser.add_option("-r", action="store", type="string",
+ dest="results", default='.',
+ help="specify results directory")
+ self.parser.add_option("-l", action="store", type="string",
+ dest="label", default='',
+ help="label for the job")
+ self.parser.add_option("-u", action="store", type="string",
+ dest="user",
+ default=os.environ.get('USER'),
+ help="username for the job")
+ self.parser.add_option("-P", action="store", type="string",
+ dest="parse_job",
+ default='',
+ help="parse the results of the job")
+ self.parser.add_option("-i", action="store_true",
+ dest="install_before", default=False,
+ help="reinstall machines before running the job")
+ self.parser.add_option("-I", action="store_true",
+ dest="install_after", default=False,
+ help="reinstall machines after running the job")
+ self.parser.add_option("-b", action="store_true",
+ dest="reboot", default=False,
+ help="reboot all machines after job")
+ self.parser.add_option("-v", action="store_true",
+ dest="verify", default=False,
+ help="verify the machines only")
+ self.parser.add_option("-R", action="store_true",
+ dest="repair", default=False,
+ help="repair the machines")
+ self.parser.add_option("-n", action="store_true",
+ dest="no_tee", default=False,
+ help="no teeing the status to stdout/err")
+ self.parser.add_option("-p", action="store_true",
+ dest="write_pidfile", default=False,
+ help="write pidfile (.autoserv_execute)")
+ self.parser.add_option("--ssh-user", action="store",
+ type="string", dest="ssh_user",
+ default="root",
+ help=("specify the user for ssh"
+ "connections"))
+ self.parser.add_option("--ssh-port", action="store",
+ type="int", dest="ssh_port",
+ default=22,
+ help=("specify the port to use for "
+ "ssh connections"))
+ self.parser.add_option("--ssh-pass", action="store",
+ type="string", dest="ssh_pass",
+ default="",
+ help=("specify the password to use "
+ "for ssh connections"))
+
+
+ def parse_args(self):
+ (self.options, self.args) = self.parser.parse_args()
try:
- from autotest_lib.server.site_autoserv_parser \
- import site_autoserv_parser
+ from autotest_lib.server.site_autoserv_parser \
+ import site_autoserv_parser
except ImportError:
- class site_autoserv_parser(base_autoserv_parser):
- pass
+ class site_autoserv_parser(base_autoserv_parser):
+ pass
class autoserv_parser(site_autoserv_parser):
- pass
+ pass
# create the one and only one instance of autoserv_parser
diff --git a/server/autotest.py b/server/autotest.py
index 57a80a1..21e519b 100644
--- a/server/autotest.py
+++ b/server/autotest.py
@@ -5,7 +5,7 @@
"""
This module defines the Autotest class
- Autotest: software to run tests automatically
+ Autotest: software to run tests automatically
"""
__author__ = """
@@ -31,462 +31,462 @@
class BaseAutotest(installable_object.InstallableObject):
- """
- This class represents the Autotest program.
+ """
+ This class represents the Autotest program.
- Autotest is used to run tests automatically and collect the results.
- It also supports profilers.
+ Autotest is used to run tests automatically and collect the results.
+ It also supports profilers.
- Implementation details:
- This is a leaf class in an abstract class hierarchy, it must
- implement the unimplemented methods in parent classes.
- """
- job = None
+ Implementation details:
+ This is a leaf class in an abstract class hierarchy, it must
+ implement the unimplemented methods in parent classes.
+ """
+ job = None
- def __init__(self, host = None):
- self.host = host
- self.got = False
- self.installed = False
- self.serverdir = utils.get_server_dir()
- super(BaseAutotest, self).__init__()
+ def __init__(self, host = None):
+ self.host = host
+ self.got = False
+ self.installed = False
+ self.serverdir = utils.get_server_dir()
+ super(BaseAutotest, self).__init__()
- @logging.record
- def install(self, host = None):
- """
- Install autotest. If get() was not called previously, an
- attempt will be made to install from the autotest svn
- repository.
+ @logging.record
+ def install(self, host = None):
+ """
+ Install autotest. If get() was not called previously, an
+ attempt will be made to install from the autotest svn
+ repository.
- Args:
- host: a Host instance on which autotest will be
- installed
+ Args:
+ host: a Host instance on which autotest will be
+ installed
- Raises:
- AutoservError: if a tarball was not specified and
- the target host does not have svn installed in its path
+ Raises:
+ AutoservError: if a tarball was not specified and
+ the target host does not have svn installed in its path
- TODO(poirier): check dependencies
- autotest needs:
- bzcat
- liboptdev (oprofile)
- binutils-dev (oprofile)
- make
- psutils (netperf)
- """
- if not host:
- host = self.host
- if not self.got:
- self.get()
- host.wait_up(timeout=30)
- host.setup()
- print "Installing autotest on %s" % host.hostname
+ TODO(poirier): check dependencies
+ autotest needs:
+ bzcat
+ liboptdev (oprofile)
+ binutils-dev (oprofile)
+ make
+ psutils (netperf)
+ """
+ if not host:
+ host = self.host
+ if not self.got:
+ self.get()
+ host.wait_up(timeout=30)
+ host.setup()
+ print "Installing autotest on %s" % host.hostname
- # Let's try to figure out where autotest is installed. If we can't,
- # (autotest not installed) just assume '/usr/local/autotest' and
- # proceed.
- try:
- autodir = _get_autodir(host)
- except error.AutotestRunError:
- autodir = '/usr/local/autotest'
+ # Let's try to figure out where autotest is installed. If we can't,
+ # (autotest not installed) just assume '/usr/local/autotest' and
+ # proceed.
+ try:
+ autodir = _get_autodir(host)
+ except error.AutotestRunError:
+ autodir = '/usr/local/autotest'
- host.run('mkdir -p "%s"' % utils.sh_escape(autodir))
+ host.run('mkdir -p "%s"' % utils.sh_escape(autodir))
- if getattr(host, 'site_install_autotest', None):
- if host.site_install_autotest():
- self.installed = True
- return
+ if getattr(host, 'site_install_autotest', None):
+ if host.site_install_autotest():
+ self.installed = True
+ return
- # try to install from file or directory
- if self.source_material:
- if os.path.isdir(self.source_material):
- # Copy autotest recursively
- host.send_file(self.source_material, autodir)
- else:
- # Copy autotest via tarball
- e_msg = 'Installation method not yet implemented!'
- raise NotImplementedError(e_msg)
- print "Installation of autotest completed"
- self.installed = True
- return
+ # try to install from file or directory
+ if self.source_material:
+ if os.path.isdir(self.source_material):
+ # Copy autotest recursively
+ host.send_file(self.source_material, autodir)
+ else:
+ # Copy autotest via tarball
+ e_msg = 'Installation method not yet implemented!'
+ raise NotImplementedError(e_msg)
+ print "Installation of autotest completed"
+ self.installed = True
+ return
- # if that fails try to install using svn
- if utils.run('which svn').exit_status:
- raise error.AutoservError('svn not found in path on \
- target machine: %s' % host.name)
- try:
- host.run('svn checkout %s %s' %
- (AUTOTEST_SVN, autodir))
- except error.AutoservRunError, e:
- host.run('svn checkout %s %s' %
- (AUTOTEST_HTTP, autodir))
- print "Installation of autotest completed"
- self.installed = True
+ # if that fails try to install using svn
+ if utils.run('which svn').exit_status:
+ raise error.AutoservError('svn not found in path on \
+ target machine: %s' % host.name)
+ try:
+ host.run('svn checkout %s %s' %
+ (AUTOTEST_SVN, autodir))
+ except error.AutoservRunError, e:
+ host.run('svn checkout %s %s' %
+ (AUTOTEST_HTTP, autodir))
+ print "Installation of autotest completed"
+ self.installed = True
- def get(self, location = None):
- if not location:
- location = os.path.join(self.serverdir, '../client')
- location = os.path.abspath(location)
- # If there's stuff run on our client directory already, it
- # can cause problems. Try giving it a quick clean first.
- cwd = os.getcwd()
- os.chdir(location)
- os.system('tools/make_clean')
- os.chdir(cwd)
- super(BaseAutotest, self).get(location)
- self.got = True
+ def get(self, location = None):
+ if not location:
+ location = os.path.join(self.serverdir, '../client')
+ location = os.path.abspath(location)
+ # If there's stuff run on our client directory already, it
+ # can cause problems. Try giving it a quick clean first.
+ cwd = os.getcwd()
+ os.chdir(location)
+ os.system('tools/make_clean')
+ os.chdir(cwd)
+ super(BaseAutotest, self).get(location)
+ self.got = True
- def run(self, control_file, results_dir = '.', host = None,
- timeout=None, tag=None, parallel_flag=False):
- """
- Run an autotest job on the remote machine.
+ def run(self, control_file, results_dir = '.', host = None,
+ timeout=None, tag=None, parallel_flag=False):
+ """
+ Run an autotest job on the remote machine.
- Args:
- control_file: an open file-like-obj of the control file
- results_dir: a str path where the results should be stored
- on the local filesystem
- host: a Host instance on which the control file should
- be run
- tag: tag name for the client side instance of autotest
- parallel_flag: flag set when multiple jobs are run at the
- same time
- Raises:
- AutotestRunError: if there is a problem executing
- the control file
- """
- host = self._get_host_and_setup(host)
- results_dir = os.path.abspath(results_dir)
+ Args:
+ control_file: an open file-like-obj of the control file
+ results_dir: a str path where the results should be stored
+ on the local filesystem
+ host: a Host instance on which the control file should
+ be run
+ tag: tag name for the client side instance of autotest
+ parallel_flag: flag set when multiple jobs are run at the
+ same time
+ Raises:
+ AutotestRunError: if there is a problem executing
+ the control file
+ """
+ host = self._get_host_and_setup(host)
+ results_dir = os.path.abspath(results_dir)
- if tag:
- results_dir = os.path.join(results_dir, tag)
+ if tag:
+ results_dir = os.path.join(results_dir, tag)
- atrun = _Run(host, results_dir, tag, parallel_flag)
- self._do_run(control_file, results_dir, host, atrun, timeout)
+ atrun = _Run(host, results_dir, tag, parallel_flag)
+ self._do_run(control_file, results_dir, host, atrun, timeout)
- def _get_host_and_setup(self, host):
- if not host:
- host = self.host
- if not self.installed:
- self.install(host)
+ def _get_host_and_setup(self, host):
+ if not host:
+ host = self.host
+ if not self.installed:
+ self.install(host)
- host.wait_up(timeout=30)
- return host
+ host.wait_up(timeout=30)
+ return host
- def prepare_for_copying_logs(self, src, dest, host):
- keyval_path = ''
- if not os.path.exists(os.path.join(dest, 'keyval')):
- # Client-side keyval file can be copied directly
- return keyval_path
- # Copy client-side keyval to temporary location
- try:
- try:
- # Create temp file
- fd, keyval_path = tempfile.mkstemp(
- '.keyval_%s' % host.hostname)
- host.get_file(os.path.join(src, 'keyval'),
- keyval_path)
- finally:
- # We will squirrel away the client side keyval
- # away and move it back when we are done
- self.temp_keyval_path = tempfile.mktemp()
- host.run('mv %s %s' %
- (os.path.join(src, 'keyval'),
- self.temp_keyval_path))
- except (error.AutoservRunError, error.AutoservSSHTimeout):
- print "Prepare for copying logs failed"
- return keyval_path
+ def prepare_for_copying_logs(self, src, dest, host):
+ keyval_path = ''
+ if not os.path.exists(os.path.join(dest, 'keyval')):
+ # Client-side keyval file can be copied directly
+ return keyval_path
+ # Copy client-side keyval to temporary location
+ try:
+ try:
+ # Create temp file
+ fd, keyval_path = tempfile.mkstemp(
+ '.keyval_%s' % host.hostname)
+ host.get_file(os.path.join(src, 'keyval'),
+ keyval_path)
+ finally:
+ # We will squirrel away the client side keyval
+ # away and move it back when we are done
+ self.temp_keyval_path = tempfile.mktemp()
+ host.run('mv %s %s' %
+ (os.path.join(src, 'keyval'),
+ self.temp_keyval_path))
+ except (error.AutoservRunError, error.AutoservSSHTimeout):
+ print "Prepare for copying logs failed"
+ return keyval_path
- def process_copied_logs(self, dest, host, keyval_path):
- if not os.path.exists(os.path.join(dest, 'keyval')):
- # Client-side keyval file was copied directly
- return
- # Append contents of keyval_<host> file to keyval file
- try:
- # Read in new and old keyval files
- new_keyval = utils.read_keyval(keyval_path)
- old_keyval = utils.read_keyval(dest)
- # 'Delete' from new keyval entries that are in both
- tmp_keyval = {}
- for key, val in new_keyval.iteritems():
- if key not in old_keyval:
- tmp_keyval[key] = val
- # Append new info to keyval file
- utils.write_keyval(dest, tmp_keyval)
- # Delete keyval_<host> file
- os.remove(keyval_path)
- except IOError:
- print "Process copied logs failed"
+ def process_copied_logs(self, dest, host, keyval_path):
+ if not os.path.exists(os.path.join(dest, 'keyval')):
+ # Client-side keyval file was copied directly
+ return
+ # Append contents of keyval_<host> file to keyval file
+ try:
+ # Read in new and old keyval files
+ new_keyval = utils.read_keyval(keyval_path)
+ old_keyval = utils.read_keyval(dest)
+ # 'Delete' from new keyval entries that are in both
+ tmp_keyval = {}
+ for key, val in new_keyval.iteritems():
+ if key not in old_keyval:
+ tmp_keyval[key] = val
+ # Append new info to keyval file
+ utils.write_keyval(dest, tmp_keyval)
+ # Delete keyval_<host> file
+ os.remove(keyval_path)
+ except IOError:
+ print "Process copied logs failed"
- def postprocess_copied_logs(self, src, host):
- # we can now put our keyval file back
- try:
- host.run('mv %s %s' % (self.temp_keyval_path,
- os.path.join(src, 'keyval')))
- except:
- pass
+ def postprocess_copied_logs(self, src, host):
+ # we can now put our keyval file back
+ try:
+ host.run('mv %s %s' % (self.temp_keyval_path,
+ os.path.join(src, 'keyval')))
+ except:
+ pass
- def _do_run(self, control_file, results_dir, host, atrun, timeout):
- try:
- atrun.verify_machine()
- except:
- print "Verify machine failed on %s. Reinstalling" % \
- host.hostname
- self.install(host)
- atrun.verify_machine()
- debug = os.path.join(results_dir, 'debug')
- try:
- os.makedirs(debug)
- except:
- pass
+ def _do_run(self, control_file, results_dir, host, atrun, timeout):
+ try:
+ atrun.verify_machine()
+ except:
+ print "Verify machine failed on %s. Reinstalling" % \
+ host.hostname
+ self.install(host)
+ atrun.verify_machine()
+ debug = os.path.join(results_dir, 'debug')
+ try:
+ os.makedirs(debug)
+ except:
+ pass
- # Ready .... Aim ....
- for control in [atrun.remote_control_file,
- atrun.remote_control_file + '.state',
- atrun.manual_control_file,
- atrun.manual_control_file + '.state']:
- host.run('rm -f ' + control)
+ # Ready .... Aim ....
+ for control in [atrun.remote_control_file,
+ atrun.remote_control_file + '.state',
+ atrun.manual_control_file,
+ atrun.manual_control_file + '.state']:
+ host.run('rm -f ' + control)
- # Copy control_file to remote_control_file on the host
- tmppath = utils.get(control_file)
- host.send_file(tmppath, atrun.remote_control_file)
- if os.path.abspath(tmppath) != os.path.abspath(control_file):
- os.remove(tmppath)
+ # Copy control_file to remote_control_file on the host
+ tmppath = utils.get(control_file)
+ host.send_file(tmppath, atrun.remote_control_file)
+ if os.path.abspath(tmppath) != os.path.abspath(control_file):
+ os.remove(tmppath)
- try:
- atrun.execute_control(timeout=timeout)
- finally:
- # make an effort to wait for the machine to come up
- try:
- host.wait_up(timeout=30)
- except error.AutoservError:
- # don't worry about any errors, we'll try and
- # get the results anyway
- pass
+ try:
+ atrun.execute_control(timeout=timeout)
+ finally:
+ # make an effort to wait for the machine to come up
+ try:
+ host.wait_up(timeout=30)
+ except error.AutoservError:
+ # don't worry about any errors, we'll try and
+ # get the results anyway
+ pass
- # get the results
- if not atrun.tag:
- results = os.path.join(atrun.autodir,
- 'results', 'default')
- else:
- results = os.path.join(atrun.autodir,
- 'results', atrun.tag)
+ # get the results
+ if not atrun.tag:
+ results = os.path.join(atrun.autodir,
+ 'results', 'default')
+ else:
+ results = os.path.join(atrun.autodir,
+ 'results', atrun.tag)
- # Copy all dirs in default to results_dir
- keyval_path = self.prepare_for_copying_logs(results,
- results_dir, host)
- host.get_file(results + '/', results_dir)
- self.process_copied_logs(results_dir, host, keyval_path)
- self.postprocess_copied_logs(results, host)
+ # Copy all dirs in default to results_dir
+ keyval_path = self.prepare_for_copying_logs(results,
+ results_dir, host)
+ host.get_file(results + '/', results_dir)
+ self.process_copied_logs(results_dir, host, keyval_path)
+ self.postprocess_copied_logs(results, host)
- def run_timed_test(self, test_name, results_dir='.', host=None,
- timeout=None, tag=None, *args, **dargs):
- """
- Assemble a tiny little control file to just run one test,
- and run it as an autotest client-side test
- """
- if not host:
- host = self.host
- if not self.installed:
- self.install(host)
- opts = ["%s=%s" % (o[0], repr(o[1])) for o in dargs.items()]
- cmd = ", ".join([repr(test_name)] + map(repr, args) + opts)
- control = "job.run_test(%s)\n" % cmd
- self.run(control, results_dir, host, timeout=timeout, tag=tag)
+ def run_timed_test(self, test_name, results_dir='.', host=None,
+ timeout=None, tag=None, *args, **dargs):
+ """
+ Assemble a tiny little control file to just run one test,
+ and run it as an autotest client-side test
+ """
+ if not host:
+ host = self.host
+ if not self.installed:
+ self.install(host)
+ opts = ["%s=%s" % (o[0], repr(o[1])) for o in dargs.items()]
+ cmd = ", ".join([repr(test_name)] + map(repr, args) + opts)
+ control = "job.run_test(%s)\n" % cmd
+ self.run(control, results_dir, host, timeout=timeout, tag=tag)
- def run_test(self, test_name, results_dir='.', host=None,
- tag=None, *args, **dargs):
- self.run_timed_test(test_name, results_dir, host, timeout=None,
- tag=tag, *args, **dargs)
+ def run_test(self, test_name, results_dir='.', host=None,
+ tag=None, *args, **dargs):
+ self.run_timed_test(test_name, results_dir, host, timeout=None,
+ tag=tag, *args, **dargs)
class _Run(object):
- """
- Represents a run of autotest control file. This class maintains
- all the state necessary as an autotest control file is executed.
+ """
+ Represents a run of autotest control file. This class maintains
+ all the state necessary as an autotest control file is executed.
- It is not intended to be used directly, rather control files
- should be run using the run method in Autotest.
- """
- def __init__(self, host, results_dir, tag, parallel_flag):
- self.host = host
- self.results_dir = results_dir
- self.env = host.env
- self.tag = tag
- self.parallel_flag = parallel_flag
- self.autodir = _get_autodir(self.host)
- if tag:
- self.manual_control_file = os.path.join(self.autodir,
- 'control.%s' % tag)
- self.remote_control_file = os.path.join(self.autodir,
- 'control.%s.autoserv' % tag)
- else:
- self.manual_control_file = os.path.join(self.autodir,
- 'control')
- self.remote_control_file = os.path.join(self.autodir,
- 'control.autoserv')
+ It is not intended to be used directly, rather control files
+ should be run using the run method in Autotest.
+ """
+ def __init__(self, host, results_dir, tag, parallel_flag):
+ self.host = host
+ self.results_dir = results_dir
+ self.env = host.env
+ self.tag = tag
+ self.parallel_flag = parallel_flag
+ self.autodir = _get_autodir(self.host)
+ if tag:
+ self.manual_control_file = os.path.join(self.autodir,
+ 'control.%s' % tag)
+ self.remote_control_file = os.path.join(self.autodir,
+ 'control.%s.autoserv' % tag)
+ else:
+ self.manual_control_file = os.path.join(self.autodir,
+ 'control')
+ self.remote_control_file = os.path.join(self.autodir,
+ 'control.autoserv')
- def verify_machine(self):
- binary = os.path.join(self.autodir, 'bin/autotest')
- try:
- self.host.run('ls %s > /dev/null 2>&1' % binary)
- except:
- raise "Autotest does not appear to be installed"
+ def verify_machine(self):
+ binary = os.path.join(self.autodir, 'bin/autotest')
+ try:
+ self.host.run('ls %s > /dev/null 2>&1' % binary)
+ except:
+ raise "Autotest does not appear to be installed"
- if not self.parallel_flag:
- tmpdir = os.path.join(self.autodir, 'tmp')
- download = os.path.join(self.autodir, 'tests/download')
- self.host.run('umount %s' % tmpdir, ignore_status=True)
- self.host.run('umount %s' % download, ignore_status=True)
+ if not self.parallel_flag:
+ tmpdir = os.path.join(self.autodir, 'tmp')
+ download = os.path.join(self.autodir, 'tests/download')
+ self.host.run('umount %s' % tmpdir, ignore_status=True)
+ self.host.run('umount %s' % download, ignore_status=True)
- def get_full_cmd(self, section):
- # build up the full command we want to run over the host
- cmd = [os.path.join(self.autodir, 'bin/autotest_client')]
- if section > 0:
- cmd.append('-c')
- if self.tag:
- cmd.append('-t %s' % self.tag)
- if self.host.job.use_external_logging():
- cmd.append('-l')
- cmd.append(self.remote_control_file)
- return ' '.join(cmd)
+ def get_full_cmd(self, section):
+ # build up the full command we want to run over the host
+ cmd = [os.path.join(self.autodir, 'bin/autotest_client')]
+ if section > 0:
+ cmd.append('-c')
+ if self.tag:
+ cmd.append('-t %s' % self.tag)
+ if self.host.job.use_external_logging():
+ cmd.append('-l')
+ cmd.append(self.remote_control_file)
+ return ' '.join(cmd)
- def get_client_log(self, section):
- # open up the files we need for our logging
- client_log_file = os.path.join(self.results_dir, 'debug',
- 'client.log.%d' % section)
- return open(client_log_file, 'w', 0)
+ def get_client_log(self, section):
+ # open up the files we need for our logging
+ client_log_file = os.path.join(self.results_dir, 'debug',
+ 'client.log.%d' % section)
+ return open(client_log_file, 'w', 0)
- def execute_section(self, section, timeout):
- print "Executing %s/bin/autotest %s/control phase %d" % \
- (self.autodir, self.autodir,
- section)
+ def execute_section(self, section, timeout):
+ print "Executing %s/bin/autotest %s/control phase %d" % \
+ (self.autodir, self.autodir,
+ section)
- full_cmd = self.get_full_cmd(section)
- client_log = self.get_client_log(section)
- redirector = server_job.client_logger(self.host.job)
+ full_cmd = self.get_full_cmd(section)
+ client_log = self.get_client_log(section)
+ redirector = server_job.client_logger(self.host.job)
- try:
- old_resultdir = self.host.job.resultdir
- self.host.job.resultdir = self.results_dir
- result = self.host.run(full_cmd, ignore_status=True,
- timeout=timeout,
- stdout_tee=client_log,
- stderr_tee=redirector)
- finally:
- redirector.close()
- self.host.job.resultdir = old_resultdir
+ try:
+ old_resultdir = self.host.job.resultdir
+ self.host.job.resultdir = self.results_dir
+ result = self.host.run(full_cmd, ignore_status=True,
+ timeout=timeout,
+ stdout_tee=client_log,
+ stderr_tee=redirector)
+ finally:
+ redirector.close()
+ self.host.job.resultdir = old_resultdir
- if result.exit_status == 1:
- self.host.job.aborted = True
- if not result.stderr:
- raise error.AutotestRunError(
- "execute_section: %s failed to return anything\n"
- "stdout:%s\n" % (full_cmd, result.stdout))
+ if result.exit_status == 1:
+ self.host.job.aborted = True
+ if not result.stderr:
+ raise error.AutotestRunError(
+ "execute_section: %s failed to return anything\n"
+ "stdout:%s\n" % (full_cmd, result.stdout))
- return redirector.last_line
+ return redirector.last_line
- def execute_control(self, timeout=None):
- section = 0
- time_left = None
- if timeout:
- end_time = time.time() + timeout
- time_left = end_time - time.time()
- while not timeout or time_left > 0:
- last = self.execute_section(section, time_left)
- if timeout:
- time_left = end_time - time.time()
- if time_left <= 0:
- break
- section += 1
- if re.match(r'^END .*\t----\t----\t.*$', last):
- print "Client complete"
- return
- elif re.match('^\t*GOOD\t----\treboot\.start.*$', last):
- print "Client is rebooting"
- print "Waiting for client to halt"
- if not self.host.wait_down(HALT_TIME):
- raise error.AutotestRunError("%s \
- failed to shutdown after %ds" %
- (self.host.hostname,
- HALT_TIME))
- print "Client down, waiting for restart"
- if not self.host.wait_up(BOOT_TIME):
- # since reboot failed
- # hardreset the machine once if possible
- # before failing this control file
- print "Hardresetting %s" % (
- self.host.hostname,)
- try:
- self.host.hardreset(wait=False)
- except error.AutoservUnsupportedError:
- print "Hardreset unsupported on %s" % (
- self.host.hostname,)
- raise error.AutotestRunError("%s failed"
- " to boot after %ds" % (
- self.host.hostname,
- BOOT_TIME,))
- self.host.reboot_followup()
- continue
- self.host.job.record("ABORT", None, None,
- "Autotest client terminated " +
- "unexpectedly")
- # give the client machine a chance to recover from
- # possible crash
- self.host.wait_up(CRASH_RECOVERY_TIME)
- raise error.AutotestRunError("Aborting - unexpected "
- "final status message "
- "from client: %s\n"
- % last)
+ def execute_control(self, timeout=None):
+ section = 0
+ time_left = None
+ if timeout:
+ end_time = time.time() + timeout
+ time_left = end_time - time.time()
+ while not timeout or time_left > 0:
+ last = self.execute_section(section, time_left)
+ if timeout:
+ time_left = end_time - time.time()
+ if time_left <= 0:
+ break
+ section += 1
+ if re.match(r'^END .*\t----\t----\t.*$', last):
+ print "Client complete"
+ return
+ elif re.match('^\t*GOOD\t----\treboot\.start.*$', last):
+ print "Client is rebooting"
+ print "Waiting for client to halt"
+ if not self.host.wait_down(HALT_TIME):
+ raise error.AutotestRunError("%s \
+ failed to shutdown after %ds" %
+ (self.host.hostname,
+ HALT_TIME))
+ print "Client down, waiting for restart"
+ if not self.host.wait_up(BOOT_TIME):
+ # since reboot failed
+ # hardreset the machine once if possible
+ # before failing this control file
+ print "Hardresetting %s" % (
+ self.host.hostname,)
+ try:
+ self.host.hardreset(wait=False)
+ except error.AutoservUnsupportedError:
+ print "Hardreset unsupported on %s" % (
+ self.host.hostname,)
+ raise error.AutotestRunError("%s failed"
+ " to boot after %ds" % (
+ self.host.hostname,
+ BOOT_TIME,))
+ self.host.reboot_followup()
+ continue
+ self.host.job.record("ABORT", None, None,
+ "Autotest client terminated " +
+ "unexpectedly")
+ # give the client machine a chance to recover from
+ # possible crash
+ self.host.wait_up(CRASH_RECOVERY_TIME)
+ raise error.AutotestRunError("Aborting - unexpected "
+ "final status message "
+ "from client: %s\n"
+ % last)
- # should only get here if we timed out
- assert timeout
- raise error.AutotestTimeoutError()
+ # should only get here if we timed out
+ assert timeout
+ raise error.AutotestTimeoutError()
def _get_autodir(host):
- dir = host.get_autodir()
- if dir:
- return dir
- try:
- # There's no clean way to do this. readlink may not exist
- cmd = "python -c 'import os,sys; print os.readlink(sys.argv[1])' /etc/autotest.conf 2> /dev/null"
- dir = os.path.dirname(host.run(cmd).stdout)
- if dir:
- return dir
- except error.AutoservRunError:
- pass
- for path in ['/usr/local/autotest', '/home/autotest']:
- try:
- host.run('ls %s > /dev/null 2>&1' % \
- os.path.join(path, 'bin/autotest'))
- return path
- except error.AutoservRunError:
- pass
- raise error.AutotestRunError("Cannot figure out autotest directory")
+ dir = host.get_autodir()
+ if dir:
+ return dir
+ try:
+ # There's no clean way to do this. readlink may not exist
+ cmd = "python -c 'import os,sys; print os.readlink(sys.argv[1])' /etc/autotest.conf 2> /dev/null"
+ dir = os.path.dirname(host.run(cmd).stdout)
+ if dir:
+ return dir
+ except error.AutoservRunError:
+ pass
+ for path in ['/usr/local/autotest', '/home/autotest']:
+ try:
+ host.run('ls %s > /dev/null 2>&1' % \
+ os.path.join(path, 'bin/autotest'))
+ return path
+ except error.AutoservRunError:
+ pass
+ raise error.AutotestRunError("Cannot figure out autotest directory")
# site_autotest.py may be non-existant or empty, make sure that an appropriate
# SiteAutotest class is created nevertheless
try:
- from site_autotest import SiteAutotest
+ from site_autotest import SiteAutotest
except ImportError:
- class SiteAutotest(BaseAutotest):
- pass
+ class SiteAutotest(BaseAutotest):
+ pass
class Autotest(SiteAutotest):
- pass
+ pass
diff --git a/server/autotest_unittest.py b/server/autotest_unittest.py
index f14ea21..3df353c 100644
--- a/server/autotest_unittest.py
+++ b/server/autotest_unittest.py
@@ -11,287 +11,287 @@
class TestBaseAutotest(unittest.TestCase):
- def setUp(self):
- # create god
- self.god = mock.mock_god()
+ def setUp(self):
+ # create god
+ self.god = mock.mock_god()
- # stub out utils
- self.utils_obj = self.god.create_mock_class(utils, "utils")
- self.old_utils = autotest.utils
- autotest.utils = self.utils_obj
+ # stub out utils
+ self.utils_obj = self.god.create_mock_class(utils, "utils")
+ self.old_utils = autotest.utils
+ autotest.utils = self.utils_obj
- # stub out os
- self.old_os = autotest.os
- self.os_obj = self.god.create_mock_class(os, "os")
- autotest.os = self.os_obj
+ # stub out os
+ self.old_os = autotest.os
+ self.os_obj = self.god.create_mock_class(os, "os")
+ autotest.os = self.os_obj
- # stub out os.path
- self.path_obj = self.god.create_mock_class(os.path, "os.path")
- autotest.os.path = self.path_obj
+ # stub out os.path
+ self.path_obj = self.god.create_mock_class(os.path, "os.path")
+ autotest.os.path = self.path_obj
- # need to set return of one function in utils called in constr.
- self.server_dir = "autotest_lib.server"
- func_call = self.utils_obj.get_server_dir.expect_call()
- func_call.and_return(self.server_dir)
+ # need to set return of one function in utils called in constr.
+ self.server_dir = "autotest_lib.server"
+ func_call = self.utils_obj.get_server_dir.expect_call()
+ func_call.and_return(self.server_dir)
- # create our host mock (and give it a hostname)
- self.host = self.god.create_mock_class(ssh_host.SSHHost,
- "SSHHost")
- self.host.hostname = "foo"
+ # create our host mock (and give it a hostname)
+ self.host = self.god.create_mock_class(ssh_host.SSHHost,
+ "SSHHost")
+ self.host.hostname = "foo"
- # create the autotest object
- self.base_autotest = autotest.BaseAutotest(self.host)
+ # create the autotest object
+ self.base_autotest = autotest.BaseAutotest(self.host)
- def tearDown(self):
- # put things back
- autotest.utils = self.old_utils
- autotest.os = self.old_os
+ def tearDown(self):
+ # put things back
+ autotest.utils = self.old_utils
+ autotest.os = self.old_os
- def test_constructor(self):
- # we should check the calls
- self.god.check_playback()
+ def test_constructor(self):
+ # we should check the calls
+ self.god.check_playback()
- def common_install_test_setup(self, autodir, is_site_install_autotest):
- # mock other methods
- old_get_autodir = autotest._get_autodir
- get_autodir_obj = self.god.create_mock_function("_get_autodir")
- autotest._get_autodir = get_autodir_obj
+ def common_install_test_setup(self, autodir, is_site_install_autotest):
+ # mock other methods
+ old_get_autodir = autotest._get_autodir
+ get_autodir_obj = self.god.create_mock_function("_get_autodir")
+ autotest._get_autodir = get_autodir_obj
- self.base_autotest.got = True
- self.source_material = None
+ self.base_autotest.got = True
+ self.source_material = None
- # record calls
- self.host.wait_up.expect_call(timeout=30)
- self.host.setup.expect_call()
- get_autodir_obj.expect_call(self.host).and_return(autodir)
- rt = self.utils_obj.sh_escape.expect_call(autodir)
- rt.and_return(autodir)
- self.host.run.expect_call('mkdir -p "%s"' % (autodir))
- rt = self.host.site_install_autotest.expect_call()
- rt.and_return(is_site_install_autotest)
+ # record calls
+ self.host.wait_up.expect_call(timeout=30)
+ self.host.setup.expect_call()
+ get_autodir_obj.expect_call(self.host).and_return(autodir)
+ rt = self.utils_obj.sh_escape.expect_call(autodir)
+ rt.and_return(autodir)
+ self.host.run.expect_call('mkdir -p "%s"' % (autodir))
+ rt = self.host.site_install_autotest.expect_call()
+ rt.and_return(is_site_install_autotest)
- return old_get_autodir
+ return old_get_autodir
- def common_install_test_teardown(self, old_get_autodir):
- # put things back
- autotest._get_autodir = old_get_autodir
+ def common_install_test_teardown(self, old_get_autodir):
+ # put things back
+ autotest._get_autodir = old_get_autodir
- def test_install1(self):
- # setup
- autodir = "autodir"
- old_get_autodir = self.common_install_test_setup(autodir, True)
+ def test_install1(self):
+ # setup
+ autodir = "autodir"
+ old_get_autodir = self.common_install_test_setup(autodir, True)
- # run test
- self.base_autotest.install()
+ # run test
+ self.base_autotest.install()
- # check
- self.assertTrue(self.base_autotest.installed)
- self.god.check_playback()
+ # check
+ self.assertTrue(self.base_autotest.installed)
+ self.god.check_playback()
- # put back
- self.common_install_test_teardown(old_get_autodir)
+ # put back
+ self.common_install_test_teardown(old_get_autodir)
- def test_install2(self):
- # setup
- autodir = "autodir"
- old_get_autodir = self.common_install_test_setup(autodir, False)
- cmd = 'which svn'
- cmdresult = client_utils.CmdResult(cmd)
- self.utils_obj.run.expect_call(cmd).and_return(cmdresult)
- cmd = 'svn checkout %s %s' % (autotest.AUTOTEST_SVN, autodir)
- self.host.run.expect_call(cmd)
+ def test_install2(self):
+ # setup
+ autodir = "autodir"
+ old_get_autodir = self.common_install_test_setup(autodir, False)
+ cmd = 'which svn'
+ cmdresult = client_utils.CmdResult(cmd)
+ self.utils_obj.run.expect_call(cmd).and_return(cmdresult)
+ cmd = 'svn checkout %s %s' % (autotest.AUTOTEST_SVN, autodir)
+ self.host.run.expect_call(cmd)
- # run test
- self.base_autotest.install()
+ # run test
+ self.base_autotest.install()
- # check
- self.assertTrue(self.base_autotest.installed)
- self.god.check_playback()
+ # check
+ self.assertTrue(self.base_autotest.installed)
+ self.god.check_playback()
- # put back
- self.common_install_test_teardown(old_get_autodir)
+ # put back
+ self.common_install_test_teardown(old_get_autodir)
- def test_get(self):
- # setup
- location = "autotest_lib.client"
- cwd = "current_dir"
- self.os_obj.getcwd.expect_call().and_return(cwd)
- self.os_obj.chdir.expect_call(location)
- self.os_obj.system.expect_call('tools/make_clean')
- self.os_obj.chdir.expect_call(cwd)
+ def test_get(self):
+ # setup
+ location = "autotest_lib.client"
+ cwd = "current_dir"
+ self.os_obj.getcwd.expect_call().and_return(cwd)
+ self.os_obj.chdir.expect_call(location)
+ self.os_obj.system.expect_call('tools/make_clean')
+ self.os_obj.chdir.expect_call(cwd)
- # call method under test
- self.base_autotest.get(location)
+ # call method under test
+ self.base_autotest.get(location)
- # do tests
- self.assertTrue(self.base_autotest.got)
- self.god.check_playback()
+ # do tests
+ self.assertTrue(self.base_autotest.got)
+ self.god.check_playback()
- def test_get_default(self):
- # setup the test
- location = "autotest_lib.client"
- self.path_obj.join.expect_call(self.base_autotest.serverdir,
- '../client').and_return(location)
- self.path_obj.abspath.expect_call(location).and_return(location)
- cwd = "current_dir"
- self.os_obj.getcwd.expect_call().and_return(cwd)
- self.os_obj.chdir.expect_call(location)
- self.os_obj.system.expect_call('tools/make_clean')
- self.os_obj.chdir.expect_call(cwd)
+ def test_get_default(self):
+ # setup the test
+ location = "autotest_lib.client"
+ self.path_obj.join.expect_call(self.base_autotest.serverdir,
+ '../client').and_return(location)
+ self.path_obj.abspath.expect_call(location).and_return(location)
+ cwd = "current_dir"
+ self.os_obj.getcwd.expect_call().and_return(cwd)
+ self.os_obj.chdir.expect_call(location)
+ self.os_obj.system.expect_call('tools/make_clean')
+ self.os_obj.chdir.expect_call(cwd)
- # call method under test
- self.base_autotest.get()
+ # call method under test
+ self.base_autotest.get()
- # do tests
- self.assertTrue(self.base_autotest.got)
- self.god.check_playback()
+ # do tests
+ self.assertTrue(self.base_autotest.got)
+ self.god.check_playback()
- def test_run_default(self):
- # need to stub out _get_host_and_setup
- old_func = self.base_autotest._get_host_and_setup
- name = "_get_host_and_setup"
- new_func = self.god.create_mock_function(name)
- self.base_autotest._get_host_and_setup = new_func
+ def test_run_default(self):
+ # need to stub out _get_host_and_setup
+ old_func = self.base_autotest._get_host_and_setup
+ name = "_get_host_and_setup"
+ new_func = self.god.create_mock_function(name)
+ self.base_autotest._get_host_and_setup = new_func
- # need to stub out _do_run
- old_do_run = self.base_autotest._do_run
- do_run = self.god.create_mock_function("_do_run")
- self.base_autotest._do_run = do_run
+ # need to stub out _do_run
+ old_do_run = self.base_autotest._do_run
+ do_run = self.god.create_mock_function("_do_run")
+ self.base_autotest._do_run = do_run
- # need a mock of _Run object
- run = self.god.create_mock_class(autotest._Run, "run")
+ # need a mock of _Run object
+ run = self.god.create_mock_class(autotest._Run, "run")
- # need a mock for _Run constuctor
- oldRun = autotest._Run
- newRun = self.god.create_mock_function("_Run")
- autotest._Run = newRun
+ # need a mock for _Run constuctor
+ oldRun = autotest._Run
+ newRun = self.god.create_mock_function("_Run")
+ autotest._Run = newRun
- new_func.expect_call(None).and_return(self.host)
- results_dir = "results_dir"
- self.path_obj.abspath.expect_call(".").and_return(results_dir)
- newRun.expect_call(self.host,
- results_dir, None, False).and_return(run)
- do_run.expect_call("control", results_dir, self.host, run, None)
+ new_func.expect_call(None).and_return(self.host)
+ results_dir = "results_dir"
+ self.path_obj.abspath.expect_call(".").and_return(results_dir)
+ newRun.expect_call(self.host,
+ results_dir, None, False).and_return(run)
+ do_run.expect_call("control", results_dir, self.host, run, None)
- # call method
- self.base_autotest.run("control")
+ # call method
+ self.base_autotest.run("control")
- # do test
- self.god.check_playback()
+ # do test
+ self.god.check_playback()
- # put things back
- self.base_autotest._get_host_and_setup = old_func
- self.base_autotest._do_run = old_do_run
- autotest._Run = oldRun
+ # put things back
+ self.base_autotest._get_host_and_setup = old_func
+ self.base_autotest._do_run = old_do_run
+ autotest._Run = oldRun
- def test_prepare_for_copying_logs1(self):
- src = "src"
- dest = "dest"
- keyval_path = ''
- dkeyval = "dest/keyval"
+ def test_prepare_for_copying_logs1(self):
+ src = "src"
+ dest = "dest"
+ keyval_path = ''
+ dkeyval = "dest/keyval"
- # setup
- self.path_obj.join.expect_call(dest,
- 'keyval').and_return(dkeyval)
- self.path_obj.exists.expect_call(dkeyval).and_return(False)
+ # setup
+ self.path_obj.join.expect_call(dest,
+ 'keyval').and_return(dkeyval)
+ self.path_obj.exists.expect_call(dkeyval).and_return(False)
- # run test
- self.base_autotest.prepare_for_copying_logs(src, dest,
- self.host)
+ # run test
+ self.base_autotest.prepare_for_copying_logs(src, dest,
+ self.host)
- # check
- self.god.check_playback()
+ # check
+ self.god.check_playback()
- def test_prepare_for_copying_logs2(self):
- src = "src"
- dest = "dest"
- keyval_path = ''
- dkeyval = "dest/keyval"
- skeyval = "src/keyval"
- file_path = (0, ".keyavl_host")
+ def test_prepare_for_copying_logs2(self):
+ src = "src"
+ dest = "dest"
+ keyval_path = ''
+ dkeyval = "dest/keyval"
+ skeyval = "src/keyval"
+ file_path = (0, ".keyavl_host")
- # make stub for tempfile.mkstemp
- old_mkstemp = autotest.tempfile.mkstemp
- mkstemp_obj = self.god.create_mock_function("tempfile.mkstemp")
- autotest.tempfile.mkstemp = mkstemp_obj
+ # make stub for tempfile.mkstemp
+ old_mkstemp = autotest.tempfile.mkstemp
+ mkstemp_obj = self.god.create_mock_function("tempfile.mkstemp")
+ autotest.tempfile.mkstemp = mkstemp_obj
- # setup
- self.path_obj.join.expect_call(dest,
- 'keyval').and_return(dkeyval)
- self.path_obj.exists.expect_call(dkeyval).and_return(True)
- mkstemp_obj.expect_call('.keyval_%s'
- % self.host.hostname).and_return(file_path)
- self.path_obj.join.expect_call(src,
- 'keyval').and_return(skeyval)
- self.host.get_file.expect_call(skeyval, file_path[1])
- self.path_obj.join.expect_call(src,
- 'keyval').and_return(skeyval)
- self.host.run.expect_call('rm -rf %s' % (skeyval))
+ # setup
+ self.path_obj.join.expect_call(dest,
+ 'keyval').and_return(dkeyval)
+ self.path_obj.exists.expect_call(dkeyval).and_return(True)
+ mkstemp_obj.expect_call('.keyval_%s'
+ % self.host.hostname).and_return(file_path)
+ self.path_obj.join.expect_call(src,
+ 'keyval').and_return(skeyval)
+ self.host.get_file.expect_call(skeyval, file_path[1])
+ self.path_obj.join.expect_call(src,
+ 'keyval').and_return(skeyval)
+ self.host.run.expect_call('rm -rf %s' % (skeyval))
- # run test
- self.base_autotest.prepare_for_copying_logs(src, dest,
- self.host)
+ # run test
+ self.base_autotest.prepare_for_copying_logs(src, dest,
+ self.host)
- # check results
- self.god.check_playback()
+ # check results
+ self.god.check_playback()
- # set things back
- autotest.tempfile.mkstemp = old_mkstemp
+ # set things back
+ autotest.tempfile.mkstemp = old_mkstemp
- def test_process_copied_logs_no_dest_keyval(self):
- # setup test
- dest = "dest"
- path = "keyval_path"
- self.path_obj.join.expect_call(dest, 'keyval').and_return(path)
- self.path_obj.exists.expect_call(path).and_return(False)
+ def test_process_copied_logs_no_dest_keyval(self):
+ # setup test
+ dest = "dest"
+ path = "keyval_path"
+ self.path_obj.join.expect_call(dest, 'keyval').and_return(path)
+ self.path_obj.exists.expect_call(path).and_return(False)
- # run test
- self.base_autotest.process_copied_logs(dest, self.host, path)
+ # run test
+ self.base_autotest.process_copied_logs(dest, self.host, path)
- # run check
- self.god.check_playback()
+ # run check
+ self.god.check_playback()
- def test_process_copied_logs_with_dest_keyval(self):
- # setup test
- dest = "dest"
- kpath = "keyval_path"
- path = "path"
- self.path_obj.join.expect_call(dest, 'keyval').and_return(path)
- self.path_obj.exists.expect_call(path).and_return(True)
+ def test_process_copied_logs_with_dest_keyval(self):
+ # setup test
+ dest = "dest"
+ kpath = "keyval_path"
+ path = "path"
+ self.path_obj.join.expect_call(dest, 'keyval').and_return(path)
+ self.path_obj.exists.expect_call(path).and_return(True)
- vals = {'version': 1, 'author': "wonder woman"}
- kvals = {'version': 1}
- mvals = {'author': "wonder woman"}
+ vals = {'version': 1, 'author': "wonder woman"}
+ kvals = {'version': 1}
+ mvals = {'author': "wonder woman"}
- self.utils_obj.read_keyval.expect_call(path).and_return(vals)
- self.path_obj.join.expect_call(dest, 'keyval').and_return(kpath)
- self.utils_obj.read_keyval.expect_call(kpath).and_return(kvals)
- self.path_obj.join.expect_call(dest, 'keyval').and_return(dest)
- self.utils_obj.write_keyval.expect_call(dest, mvals)
- self.os_obj.remove.expect_call(path)
+ self.utils_obj.read_keyval.expect_call(path).and_return(vals)
+ self.path_obj.join.expect_call(dest, 'keyval').and_return(kpath)
+ self.utils_obj.read_keyval.expect_call(kpath).and_return(kvals)
+ self.path_obj.join.expect_call(dest, 'keyval').and_return(dest)
+ self.utils_obj.write_keyval.expect_call(dest, mvals)
+ self.os_obj.remove.expect_call(path)
- # call test
- self.base_autotest.process_copied_logs(dest, self.host, path)
+ # call test
+ self.base_autotest.process_copied_logs(dest, self.host, path)
- # run check
- self.god.check_playback()
+ # run check
+ self.god.check_playback()
- def test_run_timed_test(self):
- pass
+ def test_run_timed_test(self):
+ pass
if __name__ == "__main__":
- unittest.main()
+ unittest.main()
diff --git a/server/deb_kernel.py b/server/deb_kernel.py
index 26523b9..d651329 100644
--- a/server/deb_kernel.py
+++ b/server/deb_kernel.py
@@ -5,7 +5,7 @@
"""
This module defines the Kernel class
- Kernel: an os kernel
+ Kernel: an os kernel
"""
__author__ = """
@@ -22,132 +22,132 @@
class DEBKernel(kernel.Kernel):
- """
- This class represents a .deb pre-built kernel.
+ """
+ This class represents a .deb pre-built kernel.
- It is used to obtain a built kernel and install it on a Host.
+ It is used to obtain a built kernel and install it on a Host.
- Implementation details:
- This is a leaf class in an abstract class hierarchy, it must
- implement the unimplemented methods in parent classes.
- """
- def __init__(self):
- super(DEBKernel, self).__init__()
+ Implementation details:
+ This is a leaf class in an abstract class hierarchy, it must
+ implement the unimplemented methods in parent classes.
+ """
+ def __init__(self):
+ super(DEBKernel, self).__init__()
- def install(self, host, **kwargs):
- """
- Install a kernel on the remote host.
-
- This will also invoke the guest's bootloader to set this
- kernel as the default kernel.
-
- Args:
- host: the host on which to install the kernel
- [kwargs]: remaining keyword arguments will be passed
- to Bootloader.add_kernel()
-
- Raises:
- AutoservError: no package has yet been obtained. Call
- DEBKernel.get() with a .deb package.
- """
- if self.source_material is None:
- raise error.AutoservError("A kernel must first be "
- "specified via get()")
-
- remote_tmpdir = host.get_tmp_dir()
- basename = os.path.basename(self.source_material)
- remote_filename = os.path.join(remote_tmpdir, basename)
- host.send_file(self.source_material, remote_filename)
- host.run('dpkg -i "%s"' % (utils.sh_escape(remote_filename),))
- host.run('mkinitramfs -o "%s" "%s"' % (
- utils.sh_escape(self.get_initrd_name()),
- utils.sh_escape(self.get_version()),))
-
- host.bootloader.add_kernel(self.get_image_name(),
- initrd=self.get_initrd_name(), **kwargs)
+ def install(self, host, **kwargs):
+ """
+ Install a kernel on the remote host.
+
+ This will also invoke the guest's bootloader to set this
+ kernel as the default kernel.
+
+ Args:
+ host: the host on which to install the kernel
+ [kwargs]: remaining keyword arguments will be passed
+ to Bootloader.add_kernel()
+
+ Raises:
+ AutoservError: no package has yet been obtained. Call
+ DEBKernel.get() with a .deb package.
+ """
+ if self.source_material is None:
+ raise error.AutoservError("A kernel must first be "
+ "specified via get()")
+
+ remote_tmpdir = host.get_tmp_dir()
+ basename = os.path.basename(self.source_material)
+ remote_filename = os.path.join(remote_tmpdir, basename)
+ host.send_file(self.source_material, remote_filename)
+ host.run('dpkg -i "%s"' % (utils.sh_escape(remote_filename),))
+ host.run('mkinitramfs -o "%s" "%s"' % (
+ utils.sh_escape(self.get_initrd_name()),
+ utils.sh_escape(self.get_version()),))
+
+ host.bootloader.add_kernel(self.get_image_name(),
+ initrd=self.get_initrd_name(), **kwargs)
- def get_version(self):
- """Get the version of the kernel to be installed.
-
- Returns:
- The version string, as would be returned
- by 'make kernelrelease'.
-
- Raises:
- AutoservError: no package has yet been obtained. Call
- DEBKernel.get() with a .deb package.
- """
- if self.source_material is None:
- raise error.AutoservError("A kernel must first be "
- "specified via get()")
-
- retval= utils.run('dpkg-deb -f "%s" version' %
- utils.sh_escape(self.source_material),)
- return retval.stdout.strip()
+ def get_version(self):
+ """Get the version of the kernel to be installed.
+
+ Returns:
+ The version string, as would be returned
+ by 'make kernelrelease'.
+
+ Raises:
+ AutoservError: no package has yet been obtained. Call
+ DEBKernel.get() with a .deb package.
+ """
+ if self.source_material is None:
+ raise error.AutoservError("A kernel must first be "
+ "specified via get()")
+
+ retval= utils.run('dpkg-deb -f "%s" version' %
+ utils.sh_escape(self.source_material),)
+ return retval.stdout.strip()
- def get_image_name(self):
- """Get the name of the kernel image to be installed.
-
- Returns:
- The full path to the kernel image file as it will be
- installed on the host.
-
- Raises:
- AutoservError: no package has yet been obtained. Call
- DEBKernel.get() with a .deb package.
- """
- return "/boot/vmlinuz-%s" % (self.get_version(),)
+ def get_image_name(self):
+ """Get the name of the kernel image to be installed.
+
+ Returns:
+ The full path to the kernel image file as it will be
+ installed on the host.
+
+ Raises:
+ AutoservError: no package has yet been obtained. Call
+ DEBKernel.get() with a .deb package.
+ """
+ return "/boot/vmlinuz-%s" % (self.get_version(),)
- def get_initrd_name(self):
- """Get the name of the initrd file to be installed.
-
- Returns:
- The full path to the initrd file as it will be
- installed on the host. If the package includes no
- initrd file, None is returned
-
- Raises:
- AutoservError: no package has yet been obtained. Call
- DEBKernel.get() with a .deb package.
- """
- if self.source_material is None:
- raise error.AutoservError("A kernel must first be "
- "specified via get()")
-
- return "/boot/initrd.img-%s" % (self.get_version(),)
-
- def extract(self, host):
- """Extract the kernel package.
-
- This function is only useful to access the content of the
- package (for example the kernel image) without
- installing it. It is not necessary to run this function to
- install the kernel.
-
- Args:
- host: the host on which to extract the kernel package.
-
- Returns:
- The full path to the temporary directory on host where
- the package was extracted.
-
- Raises:
- AutoservError: no package has yet been obtained. Call
- DEBKernel.get() with a .deb package.
- """
- if self.source_material is None:
- raise error.AutoservError("A kernel must first be "
- "specified via get()")
-
- remote_tmpdir = host.get_tmp_dir()
- basename = os.path.basename(self.source_material)
- remote_filename = os.path.join(remote_tmpdir, basename)
- host.send_file(self.source_material, remote_filename)
- content_dir= os.path.join(remote_tmpdir, "contents")
- host.run('dpkg -x "%s" "%s"' % (utils.sh_escape(remote_filename), utils.sh_escape(content_dir),))
-
- return content_dir
+ def get_initrd_name(self):
+ """Get the name of the initrd file to be installed.
+
+ Returns:
+ The full path to the initrd file as it will be
+ installed on the host. If the package includes no
+ initrd file, None is returned
+
+ Raises:
+ AutoservError: no package has yet been obtained. Call
+ DEBKernel.get() with a .deb package.
+ """
+ if self.source_material is None:
+ raise error.AutoservError("A kernel must first be "
+ "specified via get()")
+
+ return "/boot/initrd.img-%s" % (self.get_version(),)
+
+ def extract(self, host):
+ """Extract the kernel package.
+
+ This function is only useful to access the content of the
+ package (for example the kernel image) without
+ installing it. It is not necessary to run this function to
+ install the kernel.
+
+ Args:
+ host: the host on which to extract the kernel package.
+
+ Returns:
+ The full path to the temporary directory on host where
+ the package was extracted.
+
+ Raises:
+ AutoservError: no package has yet been obtained. Call
+ DEBKernel.get() with a .deb package.
+ """
+ if self.source_material is None:
+ raise error.AutoservError("A kernel must first be "
+ "specified via get()")
+
+ remote_tmpdir = host.get_tmp_dir()
+ basename = os.path.basename(self.source_material)
+ remote_filename = os.path.join(remote_tmpdir, basename)
+ host.send_file(self.source_material, remote_filename)
+ content_dir= os.path.join(remote_tmpdir, "contents")
+ host.run('dpkg -x "%s" "%s"' % (utils.sh_escape(remote_filename), utils.sh_escape(content_dir),))
+
+ return content_dir
diff --git a/server/git.py b/server/git.py
index 52812c4..72f5b81 100644
--- a/server/git.py
+++ b/server/git.py
@@ -18,168 +18,168 @@
class GitRepo(installable_object.InstallableObject):
- """
- This class represents a git repo.
+ """
+ This class represents a git repo.
- It is used to pull down a local copy of a git repo, check if the local
- repo is up-to-date, if not update. It delegates the install to
- implementation classes.
+ It is used to pull down a local copy of a git repo, check if the local
+ repo is up-to-date, if not update. It delegates the install to
+ implementation classes.
- """
+ """
- def __init__(self, repodir, giturl, weburl):
- super(installable_object.InstallableObject, self).__init__()
- if repodir == None:
- e_msg = 'You must provide a directory to hold the git repository'
- raise ValueError(e_msg)
- self.repodir = sh_escape(repodir)
- if giturl == None:
- raise ValueError('You must provide a git URL to the repository')
- self.giturl = giturl
- if weburl == None:
- raise ValueError('You must provide a http URL to the repository')
- self.weburl = weburl
+ def __init__(self, repodir, giturl, weburl):
+ super(installable_object.InstallableObject, self).__init__()
+ if repodir == None:
+ e_msg = 'You must provide a directory to hold the git repository'
+ raise ValueError(e_msg)
+ self.repodir = sh_escape(repodir)
+ if giturl == None:
+ raise ValueError('You must provide a git URL to the repository')
+ self.giturl = giturl
+ if weburl == None:
+ raise ValueError('You must provide a http URL to the repository')
+ self.weburl = weburl
- # path to .git dir
- self.gitpath = utils.sh_escape(os.path.join(self.repodir,'.git'))
+ # path to .git dir
+ self.gitpath = utils.sh_escape(os.path.join(self.repodir,'.git'))
- # base git command , pointing to gitpath git dir
- self.gitcmdbase = 'git --git-dir=%s' % self.gitpath
+ # base git command , pointing to gitpath git dir
+ self.gitcmdbase = 'git --git-dir=%s' % self.gitpath
- # default to same remote path as local
- self.__build = os.path.dirname(self.repodir)
+ # default to same remote path as local
+ self.__build = os.path.dirname(self.repodir)
- def run(self, command, timeout=None, ignore_status=False):
- return utils.run(r'%s' % (utils.sh_escape(command)),
- timeout, ignore_status)
+ def run(self, command, timeout=None, ignore_status=False):
+ return utils.run(r'%s' % (utils.sh_escape(command)),
+ timeout, ignore_status)
- # base install method
- def install(self, host, builddir=None):
- # allow override of target remote dir
- if builddir:
- self.__build = builddir
+ # base install method
+ def install(self, host, builddir=None):
+ # allow override of target remote dir
+ if builddir:
+ self.__build = builddir
- # push source to host for install
- print 'pushing %s to host:%s' %(self.source_material, self.__build)
- host.send_file(self.source_material, self.__build)
+ # push source to host for install
+ print 'pushing %s to host:%s' %(self.source_material, self.__build)
+ host.send_file(self.source_material, self.__build)
- def gitcmd(self, cmd, ignore_status=False):
- return self.run('%s %s'%(self.gitcmdbase, cmd),
- ignore_status=ignore_status)
+ def gitcmd(self, cmd, ignore_status=False):
+ return self.run('%s %s'%(self.gitcmdbase, cmd),
+ ignore_status=ignore_status)
- def get(self, **kwargs):
- """
- This method overrides baseclass get so we can do proper git
- clone/pulls, and check for updated versions. The result of
- this method will leave an up-to-date version of git repo at
- 'giturl' in 'repodir' directory to be used by build/install
- methods.
- """
+ def get(self, **kwargs):
+ """
+ This method overrides baseclass get so we can do proper git
+ clone/pulls, and check for updated versions. The result of
+ this method will leave an up-to-date version of git repo at
+ 'giturl' in 'repodir' directory to be used by build/install
+ methods.
+ """
- if not self.is_repo_initialized():
- # this is your first time ...
- print 'cloning repo...'
- cmd = 'clone %s %s ' %(self.giturl, self.repodir)
- rv = self.gitcmd(cmd, True)
- if rv.exit_status != 0:
- print rv.stderr
- raise error.CmdError('Failed to clone git url', rv)
- else:
- print rv.stdout
+ if not self.is_repo_initialized():
+ # this is your first time ...
+ print 'cloning repo...'
+ cmd = 'clone %s %s ' %(self.giturl, self.repodir)
+ rv = self.gitcmd(cmd, True)
+ if rv.exit_status != 0:
+ print rv.stderr
+ raise error.CmdError('Failed to clone git url', rv)
+ else:
+ print rv.stdout
- else:
- # exiting repo, check if we're up-to-date
- if self.is_out_of_date():
- print 'updating repo...'
- rv = self.gitcmd('pull', True)
- if rv.exit_status != 0:
- print rv.stderr
- e_msg = 'Failed to pull git repo data'
- raise error.CmdError(e_msg, rv)
- else:
- print 'repo up-to-date'
+ else:
+ # exiting repo, check if we're up-to-date
+ if self.is_out_of_date():
+ print 'updating repo...'
+ rv = self.gitcmd('pull', True)
+ if rv.exit_status != 0:
+ print rv.stderr
+ e_msg = 'Failed to pull git repo data'
+ raise error.CmdError(e_msg, rv)
+ else:
+ print 'repo up-to-date'
- # remember where the source is
- self.source_material = self.repodir
+ # remember where the source is
+ self.source_material = self.repodir
- def get_local_head(self):
- cmd = 'log --max-count=1'
- gitlog = self.gitcmd(cmd).stdout
+ def get_local_head(self):
+ cmd = 'log --max-count=1'
+ gitlog = self.gitcmd(cmd).stdout
- # parsing the commit checksum out of git log 's first entry.
- # Output looks like:
- #
- # commit 1dccba29b4e5bf99fb98c324f952386dda5b097f
- # Merge: 031b69b... df6af41...
- # Author: Avi Kivity <avi@qumranet.com>
- # Date: Tue Oct 23 10:36:11 2007 +0200
- #
- # Merge home:/home/avi/kvm/linux-2.6
- return str(gitlog.split('\n')[0]).split()[1]
+ # parsing the commit checksum out of git log 's first entry.
+ # Output looks like:
+ #
+ # commit 1dccba29b4e5bf99fb98c324f952386dda5b097f
+ # Merge: 031b69b... df6af41...
+ # Author: Avi Kivity <avi@qumranet.com>
+ # Date: Tue Oct 23 10:36:11 2007 +0200
+ #
+ # Merge home:/home/avi/kvm/linux-2.6
+ return str(gitlog.split('\n')[0]).split()[1]
- def get_remote_head(self):
- def __needs_refresh(lines):
- tag = '<meta http-equiv="refresh" content="0"/>'
- if len(filter(lambda x: x.startswith(tag), lines)) > 0:
- return True
+ def get_remote_head(self):
+ def __needs_refresh(lines):
+ tag = '<meta http-equiv="refresh" content="0"/>'
+ if len(filter(lambda x: x.startswith(tag), lines)) > 0:
+ return True
- return False
+ return False
- # scan git web interface for revision HEAD's commit tag
- gitwebaction=';a=commit;h=HEAD'
- url = self.weburl+gitwebaction
- max_refresh = 4
- r = 0
+ # scan git web interface for revision HEAD's commit tag
+ gitwebaction=';a=commit;h=HEAD'
+ url = self.weburl+gitwebaction
+ max_refresh = 4
+ r = 0
- print 'checking %s for changes' %(url)
- u = utils.urlopen(url)
- lines = u.read().split('\n')
+ print 'checking %s for changes' %(url)
+ u = utils.urlopen(url)
+ lines = u.read().split('\n')
- while __needs_refresh(lines) and r < max_refresh:
- print 'refreshing url'
- r = r+1
- u = utils.urlopen(url)
- lines = u.read().split('\n')
+ while __needs_refresh(lines) and r < max_refresh:
+ print 'refreshing url'
+ r = r+1
+ u = utils.urlopen(url)
+ lines = u.read().split('\n')
- if r >= max_refresh:
- e_msg = 'Failed to get remote repo status, refreshed %s times' % r
- raise IndexError(e_msg)
+ if r >= max_refresh:
+ e_msg = 'Failed to get remote repo status, refreshed %s times' % r
+ raise IndexError(e_msg)
- # looking for a line like:
- # <tr><td>commit</td><td # class="sha1">aadea67210c8b9e7a57744a1c2845501d2cdbac7</td></tr>
- commit_filter = lambda x: x.startswith('<tr><td>commit</td>')
- commit_line = filter(commit_filter, lines)
+ # looking for a line like:
+ # <tr><td>commit</td><td # class="sha1">aadea67210c8b9e7a57744a1c2845501d2cdbac7</td></tr>
+ commit_filter = lambda x: x.startswith('<tr><td>commit</td>')
+ commit_line = filter(commit_filter, lines)
- # extract the sha1 sum from the commit line
- return str(commit_line).split('>')[4].split('<')[0]
+ # extract the sha1 sum from the commit line
+ return str(commit_line).split('>')[4].split('<')[0]
- def is_out_of_date(self):
- local_head = self.get_local_head()
- remote_head = self.get_remote_head()
+ def is_out_of_date(self):
+ local_head = self.get_local_head()
+ remote_head = self.get_remote_head()
- # local is out-of-date, pull
- if local_head != remote_head:
- return True
+ # local is out-of-date, pull
+ if local_head != remote_head:
+ return True
- return False
+ return False
- def is_repo_initialized(self):
- # if we fail to get a rv of 0 out of the git log command
- # then the repo is bogus
+ def is_repo_initialized(self):
+ # if we fail to get a rv of 0 out of the git log command
+ # then the repo is bogus
- cmd = 'log --max-count=1'
- rv = self.gitcmd(cmd, True)
- if rv.exit_status == 0:
- return True
+ cmd = 'log --max-count=1'
+ rv = self.gitcmd(cmd, True)
+ if rv.exit_status == 0:
+ return True
- return False
+ return False
diff --git a/server/git_kernel.py b/server/git_kernel.py
index f993732..1f4273c 100644
--- a/server/git_kernel.py
+++ b/server/git_kernel.py
@@ -17,56 +17,56 @@
class GitKernel(git.GitRepo):
- """
- This class represents a git kernel repo.
+ """
+ This class represents a git kernel repo.
- It is used to pull down a local copy of a git repo, check if the local repo
- is up-to-date, if not update and then build the kernel from the git repo.
+ It is used to pull down a local copy of a git repo, check if the local repo
+ is up-to-date, if not update and then build the kernel from the git repo.
- """
- def __init__(self, repodir, giturl, weburl):
- git.GitRepo.__init__(self, repodir, giturl, weburl)
- self.__patches = []
- self.__config = None
- self.__build = None
+ """
+ def __init__(self, repodir, giturl, weburl):
+ git.GitRepo.__init__(self, repodir, giturl, weburl)
+ self.__patches = []
+ self.__config = None
+ self.__build = None
- def configure(self, config):
- self.__config = config
+ def configure(self, config):
+ self.__config = config
- def patch(self, patch):
- self.__patches.append(patch)
+ def patch(self, patch):
+ self.__patches.append(patch)
- def install(self, host, build=True, builddir=None):
- # use tmpdir if no builddir specified
- # NB: pass a builddir to install() method if you
- # need to ensure the build remains after the completion
- # of a job
- if not builddir:
- self.__build = os.path.join(host.get_tmp_dir(),"build")
- print 'warning: builddir %s is not persistent' %(self.__build)
+ def install(self, host, build=True, builddir=None):
+ # use tmpdir if no builddir specified
+ # NB: pass a builddir to install() method if you
+ # need to ensure the build remains after the completion
+ # of a job
+ if not builddir:
+ self.__build = os.path.join(host.get_tmp_dir(),"build")
+ print 'warning: builddir %s is not persistent' %(self.__build)
- # push source to host for install
- print 'pushing %s to host' %(self.source_material)
- host.send_file(self.source_material, self.__build)
- remote_source_material= os.path.join(self.__build,
- os.path.basename(self.source_material))
+ # push source to host for install
+ print 'pushing %s to host' %(self.source_material)
+ host.send_file(self.source_material, self.__build)
+ remote_source_material= os.path.join(self.__build,
+ os.path.basename(self.source_material))
- # use a source_kernel to configure, patch, build and install.
- sk = source_kernel.SourceKernel(remote_source_material)
+ # use a source_kernel to configure, patch, build and install.
+ sk = source_kernel.SourceKernel(remote_source_material)
- if build:
- # apply patches
- for p in self.__patches:
- sk.patch(p)
+ if build:
+ # apply patches
+ for p in self.__patches:
+ sk.patch(p)
- # configure
- sk.configure(self.__config)
+ # configure
+ sk.configure(self.__config)
- # build
- sk.build(host)
-
- # install
- sk.install(host)
+ # build
+ sk.build(host)
+
+ # install
+ sk.install(host)
diff --git a/server/hosts/base_classes.py b/server/hosts/base_classes.py
index ddcfcdd..0f637bd 100644
--- a/server/hosts/base_classes.py
+++ b/server/hosts/base_classes.py
@@ -8,8 +8,8 @@
Implementation details:
You should import the "hosts" package instead of importing each type of host.
- Host: a machine on which you can run programs
- RemoteHost: a remote machine on which you can run programs
+ Host: a machine on which you can run programs
+ RemoteHost: a remote machine on which you can run programs
"""
__author__ = """
@@ -26,91 +26,91 @@
class Host(object):
- """
- This class represents a machine on which you can run programs.
+ """
+ This class represents a machine on which you can run programs.
- It may be a local machine, the one autoserv is running on, a remote
- machine or a virtual machine.
+ It may be a local machine, the one autoserv is running on, a remote
+ machine or a virtual machine.
- Implementation details:
- This is an abstract class, leaf subclasses must implement the methods
- listed here. You must not instantiate this class but should
- instantiate one of those leaf subclasses.
- """
+ Implementation details:
+ This is an abstract class, leaf subclasses must implement the methods
+ listed here. You must not instantiate this class but should
+ instantiate one of those leaf subclasses.
+ """
- bootloader = None
+ bootloader = None
- def __init__(self):
- super(Host, self).__init__()
- self.serverdir = utils.get_server_dir()
- self.bootloader= bootloader.Bootloader(self)
- self.env = {}
+ def __init__(self):
+ super(Host, self).__init__()
+ self.serverdir = utils.get_server_dir()
+ self.bootloader= bootloader.Bootloader(self)
+ self.env = {}
- def run(self, command):
- pass
+ def run(self, command):
+ pass
- def reboot(self):
- pass
+ def reboot(self):
+ pass
- def reboot_setup(self):
- pass
+ def reboot_setup(self):
+ pass
- def reboot_followup(self):
- pass
+ def reboot_followup(self):
+ pass
- def get_file(self, source, dest):
- pass
+ def get_file(self, source, dest):
+ pass
- def send_file(self, source, dest):
- pass
+ def send_file(self, source, dest):
+ pass
- def get_tmp_dir(self):
- pass
+ def get_tmp_dir(self):
+ pass
- def is_up(self):
- pass
+ def is_up(self):
+ pass
- def get_wait_up_processes(self):
- """
- Gets the list of local processes to wait for in wait_up.
- """
- get_config = global_config.global_config.get_config_value
- proc_list = get_config("HOSTS", "wait_up_processes",
- default="").strip()
- processes = set(p.strip() for p in proc_list.split(","))
- processes.discard("")
- return processes
+ def get_wait_up_processes(self):
+ """
+ Gets the list of local processes to wait for in wait_up.
+ """
+ get_config = global_config.global_config.get_config_value
+ proc_list = get_config("HOSTS", "wait_up_processes",
+ default="").strip()
+ processes = set(p.strip() for p in proc_list.split(","))
+ processes.discard("")
+ return processes
- def wait_up(self, timeout):
- pass
+ def wait_up(self, timeout):
+ pass
- def wait_down(self, timeout):
- pass
+ def wait_down(self, timeout):
+ pass
- def get_num_cpu(self):
- pass
+ def get_num_cpu(self):
+ pass
- def machine_install(self):
- raise NotImplementedError('Machine install not implemented!')
+ def machine_install(self):
+ raise NotImplementedError('Machine install not implemented!')
- def install(self, installableObject):
- installableObject.install(self)
+ def install(self, installableObject):
+ installableObject.install(self)
- def get_crashdumps(self, test_start_time):
- pass
+ def get_crashdumps(self, test_start_time):
+ pass
- def get_autodir(self):
- return None
+ def get_autodir(self):
+ return None
diff --git a/server/hosts/bootloader.py b/server/hosts/bootloader.py
index 7994a8a..4fb8d51 100644
--- a/server/hosts/bootloader.py
+++ b/server/hosts/bootloader.py
@@ -5,7 +5,7 @@
"""
This module defines the Bootloader class.
- Bootloader: a program to boot Kernels on a Host.
+ Bootloader: a program to boot Kernels on a Host.
"""
__author__ = """
@@ -26,172 +26,172 @@
class Bootloader(object):
- """
- This class represents a bootloader.
+ """
+ This class represents a bootloader.
- It can be used to add a kernel to the list of kernels that can be
- booted by a bootloader. It can also make sure that this kernel will
- be the one chosen at next reboot.
- """
+ It can be used to add a kernel to the list of kernels that can be
+ booted by a bootloader. It can also make sure that this kernel will
+ be the one chosen at next reboot.
+ """
- def __init__(self, host, xen_mode=False):
- super(Bootloader, self).__init__()
- self._host = weakref.ref(host)
- self._boottool_path = None
- self.xen_mode = xen_mode
+ def __init__(self, host, xen_mode=False):
+ super(Bootloader, self).__init__()
+ self._host = weakref.ref(host)
+ self._boottool_path = None
+ self.xen_mode = xen_mode
- def get_type(self):
- return self._run_boottool('--bootloader-probe').stdout.strip()
+ def get_type(self):
+ return self._run_boottool('--bootloader-probe').stdout.strip()
- def get_architecture(self):
- return self._run_boottool('--arch-probe').stdout.strip()
+ def get_architecture(self):
+ return self._run_boottool('--arch-probe').stdout.strip()
- def get_titles(self):
- return self._run_boottool('--info all | grep title | '
- 'cut -d " " -f2-').stdout.strip().split('\n')
+ def get_titles(self):
+ return self._run_boottool('--info all | grep title | '
+ 'cut -d " " -f2-').stdout.strip().split('\n')
- def get_default(self):
- return self._run_boottool('--default').stdout.strip()
+ def get_default(self):
+ return self._run_boottool('--default').stdout.strip()
- def _get_info(self, info_id):
- retval = self._run_boottool('--info=%s' % info_id).stdout
+ def _get_info(self, info_id):
+ retval = self._run_boottool('--info=%s' % info_id).stdout
- results = []
- info = {}
- for line in retval.splitlines():
- if not line.strip():
- if info:
- results.append(info)
- info = {}
- else:
- key, val = line.split(":", 1)
- info[key.strip()] = val.strip()
- if info:
- results.append(info)
+ results = []
+ info = {}
+ for line in retval.splitlines():
+ if not line.strip():
+ if info:
+ results.append(info)
+ info = {}
+ else:
+ key, val = line.split(":", 1)
+ info[key.strip()] = val.strip()
+ if info:
+ results.append(info)
- return results
+ return results
- def get_info(self, index):
- results = self._get_info(index)
- if results:
- return results[0]
- else:
- return {}
+ def get_info(self, index):
+ results = self._get_info(index)
+ if results:
+ return results[0]
+ else:
+ return {}
- def get_all_info(self):
- return self._get_info('all')
+ def get_all_info(self):
+ return self._get_info('all')
- def set_default(self, index):
- self._run_boottool('--set-default=%s' % index)
+ def set_default(self, index):
+ self._run_boottool('--set-default=%s' % index)
- # 'kernel' can be a position number or a title
- def add_args(self, kernel, args):
- parameters = '--update-kernel=%s --args="%s"' % (kernel, args)
-
- #add parameter if this is a Xen entry
- if self.xen_mode:
- parameters += ' --xen'
-
- self._run_boottool(parameters)
+ # 'kernel' can be a position number or a title
+ def add_args(self, kernel, args):
+ parameters = '--update-kernel=%s --args="%s"' % (kernel, args)
+
+ #add parameter if this is a Xen entry
+ if self.xen_mode:
+ parameters += ' --xen'
+
+ self._run_boottool(parameters)
- def add_xen_hypervisor_args(self, kernel, args):
- self._run_boottool('--xen --update-xenhyper=%s --xha="%s"' \
- % (kernel, args))
+ def add_xen_hypervisor_args(self, kernel, args):
+ self._run_boottool('--xen --update-xenhyper=%s --xha="%s"' \
+ % (kernel, args))
- def remove_args(self, kernel, args):
- params = '--update-kernel=%s --remove-args="%s"' % (kernel, args)
-
- #add parameter if this is a Xen entry
- if self.xen_mode:
- params += ' --xen'
-
- self._run_boottool(params)
+ def remove_args(self, kernel, args):
+ params = '--update-kernel=%s --remove-args="%s"' % (kernel, args)
+
+ #add parameter if this is a Xen entry
+ if self.xen_mode:
+ params += ' --xen'
+
+ self._run_boottool(params)
- def remove_xen_hypervisor_args(self, kernel, args):
- self._run_boottool('--xen --update-xenhyper=%s '
- '--remove-args="%s"') % (kernel, args)
+ def remove_xen_hypervisor_args(self, kernel, args):
+ self._run_boottool('--xen --update-xenhyper=%s '
+ '--remove-args="%s"') % (kernel, args)
- def add_kernel(self, path, title='autoserv', root=None, args=None,
- initrd=None, xen_hypervisor=None, default=True):
- """
- If an entry with the same title is already present, it will be
- replaced.
- """
- if title in self.get_titles():
- self._run_boottool('--remove-kernel "%s"' % (
- utils.sh_escape(title),))
-
- parameters = '--add-kernel "%s" --title "%s"' % (
- utils.sh_escape(path), utils.sh_escape(title),)
-
- if root:
- parameters += ' --root "%s"' % (utils.sh_escape(root),)
-
- if args:
- parameters += ' --args "%s"' % (utils.sh_escape(args),)
-
- # add an initrd now or forever hold your peace
- if initrd:
- parameters += ' --initrd "%s"' % (
- utils.sh_escape(initrd),)
-
- if default:
- parameters += ' --make-default'
-
- # add parameter if this is a Xen entry
- if self.xen_mode:
- parameters += ' --xen'
- if xen_hypervisor:
- parameters += ' --xenhyper "%s"' % (
- utils.sh_escape(xen_hypervisor),)
-
- self._run_boottool(parameters)
+ def add_kernel(self, path, title='autoserv', root=None, args=None,
+ initrd=None, xen_hypervisor=None, default=True):
+ """
+ If an entry with the same title is already present, it will be
+ replaced.
+ """
+ if title in self.get_titles():
+ self._run_boottool('--remove-kernel "%s"' % (
+ utils.sh_escape(title),))
+
+ parameters = '--add-kernel "%s" --title "%s"' % (
+ utils.sh_escape(path), utils.sh_escape(title),)
+
+ if root:
+ parameters += ' --root "%s"' % (utils.sh_escape(root),)
+
+ if args:
+ parameters += ' --args "%s"' % (utils.sh_escape(args),)
+
+ # add an initrd now or forever hold your peace
+ if initrd:
+ parameters += ' --initrd "%s"' % (
+ utils.sh_escape(initrd),)
+
+ if default:
+ parameters += ' --make-default'
+
+ # add parameter if this is a Xen entry
+ if self.xen_mode:
+ parameters += ' --xen'
+ if xen_hypervisor:
+ parameters += ' --xenhyper "%s"' % (
+ utils.sh_escape(xen_hypervisor),)
+
+ self._run_boottool(parameters)
- def remove_kernel(self, kernel):
- self._run_boottool('--remove-kernel=%s' % kernel)
+ def remove_kernel(self, kernel):
+ self._run_boottool('--remove-kernel=%s' % kernel)
- def boot_once(self, title):
- self._run_boottool('--boot-once --title=%s' % title)
+ def boot_once(self, title):
+ self._run_boottool('--boot-once --title=%s' % title)
- def install_boottool(self):
- if self._host() is None:
- raise error.AutoservError(
- "Host does not exist anymore")
- tmpdir = self._host().get_tmp_dir()
- self._host().send_file(os.path.abspath(os.path.join(
- utils.get_server_dir(), BOOTTOOL_SRC)), tmpdir)
- self._boottool_path= os.path.join(tmpdir,
- os.path.basename(BOOTTOOL_SRC))
+ def install_boottool(self):
+ if self._host() is None:
+ raise error.AutoservError(
+ "Host does not exist anymore")
+ tmpdir = self._host().get_tmp_dir()
+ self._host().send_file(os.path.abspath(os.path.join(
+ utils.get_server_dir(), BOOTTOOL_SRC)), tmpdir)
+ self._boottool_path= os.path.join(tmpdir,
+ os.path.basename(BOOTTOOL_SRC))
- def _get_boottool_path(self):
- if not self._boottool_path:
- self.install_boottool()
- return self._boottool_path
+ def _get_boottool_path(self):
+ if not self._boottool_path:
+ self.install_boottool()
+ return self._boottool_path
- def _set_boottool_path(self, path):
- self._boottool_path = path
-
-
- boottool_path = property(_get_boottool_path, _set_boottool_path)
+ def _set_boottool_path(self, path):
+ self._boottool_path = path
- def _run_boottool(self, cmd):
- return self._host().run(self.boottool_path + ' ' + cmd)
+ boottool_path = property(_get_boottool_path, _set_boottool_path)
+
+
+ def _run_boottool(self, cmd):
+ return self._host().run(self.boottool_path + ' ' + cmd)
diff --git a/server/hosts/bootloader_unittest.py b/server/hosts/bootloader_unittest.py
index 9161a3e..a6f6756 100644
--- a/server/hosts/bootloader_unittest.py
+++ b/server/hosts/bootloader_unittest.py
@@ -10,347 +10,347 @@
class test_bootloader_install(unittest.TestCase):
- def setUp(self):
- self.god = mock.mock_god()
+ def setUp(self):
+ self.god = mock.mock_god()
- # mock out get_server_dir
- self.god.stub_function(utils, "get_server_dir")
+ # mock out get_server_dir
+ self.god.stub_function(utils, "get_server_dir")
- def tearDown(self):
- self.god.unstub_all()
+ def tearDown(self):
+ self.god.unstub_all()
- def create_mock_sshhost(self):
- # useful for building disposable SSHHost mocks
- return self.god.create_mock_class(ssh_host.SSHHost, "SSHHost")
+ def create_mock_sshhost(self):
+ # useful for building disposable SSHHost mocks
+ return self.god.create_mock_class(ssh_host.SSHHost, "SSHHost")
- def create_install_boottool_mock(self, loader, dst_dir):
- mock_install_boottool = \
- self.god.create_mock_function("install_boottool")
- def install_boottool():
- loader._boottool_path = dst_dir
- mock_install_boottool()
- loader.install_boottool = install_boottool
- return mock_install_boottool
+ def create_install_boottool_mock(self, loader, dst_dir):
+ mock_install_boottool = \
+ self.god.create_mock_function("install_boottool")
+ def install_boottool():
+ loader._boottool_path = dst_dir
+ mock_install_boottool()
+ loader.install_boottool = install_boottool
+ return mock_install_boottool
- def test_install_fails_without_host(self):
- host = self.create_mock_sshhost()
- loader = bootloader.Bootloader(host)
- del host
- self.assertRaises(error.AutoservError, loader.install_boottool)
+ def test_install_fails_without_host(self):
+ host = self.create_mock_sshhost()
+ loader = bootloader.Bootloader(host)
+ del host
+ self.assertRaises(error.AutoservError, loader.install_boottool)
- def test_installs_to_tmpdir(self):
- TMPDIR = "/unittest/tmp"
- SERVERDIR = "/unittest/server"
- BOOTTOOL_SRC = os.path.join(SERVERDIR, bootloader.BOOTTOOL_SRC)
- BOOTTOOL_SRC = os.path.abspath(BOOTTOOL_SRC)
- BOOTTOOL_DST = os.path.join(TMPDIR, "boottool")
- # set up the recording
- host = self.create_mock_sshhost()
- host.get_tmp_dir.expect_call().and_return(TMPDIR)
- utils.get_server_dir.expect_call().and_return(SERVERDIR)
- host.send_file.expect_call(BOOTTOOL_SRC, TMPDIR)
- # run the test
- loader = bootloader.Bootloader(host)
- loader.install_boottool()
- # assert the playback is correct
- self.god.check_playback()
- # assert the final dest is correct
- self.assertEquals(loader.boottool_path, BOOTTOOL_DST)
+ def test_installs_to_tmpdir(self):
+ TMPDIR = "/unittest/tmp"
+ SERVERDIR = "/unittest/server"
+ BOOTTOOL_SRC = os.path.join(SERVERDIR, bootloader.BOOTTOOL_SRC)
+ BOOTTOOL_SRC = os.path.abspath(BOOTTOOL_SRC)
+ BOOTTOOL_DST = os.path.join(TMPDIR, "boottool")
+ # set up the recording
+ host = self.create_mock_sshhost()
+ host.get_tmp_dir.expect_call().and_return(TMPDIR)
+ utils.get_server_dir.expect_call().and_return(SERVERDIR)
+ host.send_file.expect_call(BOOTTOOL_SRC, TMPDIR)
+ # run the test
+ loader = bootloader.Bootloader(host)
+ loader.install_boottool()
+ # assert the playback is correct
+ self.god.check_playback()
+ # assert the final dest is correct
+ self.assertEquals(loader.boottool_path, BOOTTOOL_DST)
- def test_get_path_automatically_installs(self):
- BOOTTOOL_DST = "/unittest/tmp/boottool"
- host = self.create_mock_sshhost()
- loader = bootloader.Bootloader(host)
- # mock out loader.install_boottool
- mock_install = \
- self.create_install_boottool_mock(loader, BOOTTOOL_DST)
- # set up the recording
- mock_install.expect_call()
- # run the test
- self.assertEquals(loader.boottool_path, BOOTTOOL_DST)
- self.god.check_playback()
+ def test_get_path_automatically_installs(self):
+ BOOTTOOL_DST = "/unittest/tmp/boottool"
+ host = self.create_mock_sshhost()
+ loader = bootloader.Bootloader(host)
+ # mock out loader.install_boottool
+ mock_install = \
+ self.create_install_boottool_mock(loader, BOOTTOOL_DST)
+ # set up the recording
+ mock_install.expect_call()
+ # run the test
+ self.assertEquals(loader.boottool_path, BOOTTOOL_DST)
+ self.god.check_playback()
- def test_install_is_only_called_once(self):
- BOOTTOOL_DST = "/unittest/tmp/boottool"
- host = self.create_mock_sshhost()
- loader = bootloader.Bootloader(host)
- # mock out loader.install_boottool
- mock_install = \
- self.create_install_boottool_mock(loader, BOOTTOOL_DST)
- # set up the recording
- mock_install.expect_call()
- # run the test
- self.assertEquals(loader.boottool_path, BOOTTOOL_DST)
- self.god.check_playback()
- self.assertEquals(loader.boottool_path, BOOTTOOL_DST)
- self.god.check_playback()
+ def test_install_is_only_called_once(self):
+ BOOTTOOL_DST = "/unittest/tmp/boottool"
+ host = self.create_mock_sshhost()
+ loader = bootloader.Bootloader(host)
+ # mock out loader.install_boottool
+ mock_install = \
+ self.create_install_boottool_mock(loader, BOOTTOOL_DST)
+ # set up the recording
+ mock_install.expect_call()
+ # run the test
+ self.assertEquals(loader.boottool_path, BOOTTOOL_DST)
+ self.god.check_playback()
+ self.assertEquals(loader.boottool_path, BOOTTOOL_DST)
+ self.god.check_playback()
class test_bootloader_methods(unittest.TestCase):
- def setUp(self):
- self.god = mock.mock_god()
- self.host = self.god.create_mock_class(ssh_host.SSHHost,
- "SSHHost")
- # creates a bootloader with _run_boottool mocked out
- self.loader = bootloader.Bootloader(self.host)
- self.god.stub_function(self.loader, "_run_boottool")
+ def setUp(self):
+ self.god = mock.mock_god()
+ self.host = self.god.create_mock_class(ssh_host.SSHHost,
+ "SSHHost")
+ # creates a bootloader with _run_boottool mocked out
+ self.loader = bootloader.Bootloader(self.host)
+ self.god.stub_function(self.loader, "_run_boottool")
- def tearDown(self):
- self.god.unstub_all()
+ def tearDown(self):
+ self.god.unstub_all()
- def expect_run_boottool(self, arg, result):
- result = common_utils.CmdResult(stdout=result, exit_status=0)
- self.loader._run_boottool.expect_call(arg).and_return(result)
+ def expect_run_boottool(self, arg, result):
+ result = common_utils.CmdResult(stdout=result, exit_status=0)
+ self.loader._run_boottool.expect_call(arg).and_return(result)
- def test_get_type(self):
- # set up the recording
- self.expect_run_boottool("--bootloader-probe", "lilo\n")
- # run the test
- self.assertEquals(self.loader.get_type(), "lilo")
- self.god.check_playback()
+ def test_get_type(self):
+ # set up the recording
+ self.expect_run_boottool("--bootloader-probe", "lilo\n")
+ # run the test
+ self.assertEquals(self.loader.get_type(), "lilo")
+ self.god.check_playback()
- def test_get_arch(self):
- # set up the recording
- self.expect_run_boottool("--arch-probe", "x86_64\n")
- # run the test
- self.assertEquals(self.loader.get_architecture(), "x86_64")
- self.god.check_playback()
+ def test_get_arch(self):
+ # set up the recording
+ self.expect_run_boottool("--arch-probe", "x86_64\n")
+ # run the test
+ self.assertEquals(self.loader.get_architecture(), "x86_64")
+ self.god.check_playback()
- def test_get_default(self):
- # set up the recording
- self.expect_run_boottool("--default", "0\n")
- # run the test
- self.assertEquals(self.loader.get_default(), "0")
- self.god.check_playback()
+ def test_get_default(self):
+ # set up the recording
+ self.expect_run_boottool("--default", "0\n")
+ # run the test
+ self.assertEquals(self.loader.get_default(), "0")
+ self.god.check_playback()
- def test_get_titles(self):
- # set up the recording
- self.expect_run_boottool(mock.regex_comparator(
- r"^--info all \|"), "title #1\ntitle #2\n")
- # run the test
- self.assertEquals(self.loader.get_titles(),
- ["title #1", "title #2"])
- self.god.check_playback()
+ def test_get_titles(self):
+ # set up the recording
+ self.expect_run_boottool(mock.regex_comparator(
+ r"^--info all \|"), "title #1\ntitle #2\n")
+ # run the test
+ self.assertEquals(self.loader.get_titles(),
+ ["title #1", "title #2"])
+ self.god.check_playback()
- def test_get_info_single_result(self):
- RESULT = (
- "index\t: 5\n"
- "args\t: ro single\n"
- "boot\t: (hd0,0)\n"
- "initrd\t: /boot/initrd.img-2.6.15-23-386\n"
- "kernel\t: /boot/vmlinuz-2.6.15-23-386\n"
- "root\t: UUID=07D7-0714\n"
- "savedefault\t: \n"
- "title\t: Distro, kernel 2.6.15-23-386\n"
- )
- # set up the recording
- self.expect_run_boottool("--info=5", RESULT)
- # run the test
- info = self.loader.get_info(5)
- self.god.check_playback()
- expected_info = {"index": "5", "args": "ro single",
- "boot": "(hd0,0)",
- "initrd": "/boot/initrd.img-2.6.15-23-386",
- "kernel": "/boot/vmlinuz-2.6.15-23-386",
- "root": "UUID=07D7-0714", "savedefault": "",
- "title": "Distro, kernel 2.6.15-23-386"}
- self.assertEquals(expected_info, info)
+ def test_get_info_single_result(self):
+ RESULT = (
+ "index\t: 5\n"
+ "args\t: ro single\n"
+ "boot\t: (hd0,0)\n"
+ "initrd\t: /boot/initrd.img-2.6.15-23-386\n"
+ "kernel\t: /boot/vmlinuz-2.6.15-23-386\n"
+ "root\t: UUID=07D7-0714\n"
+ "savedefault\t: \n"
+ "title\t: Distro, kernel 2.6.15-23-386\n"
+ )
+ # set up the recording
+ self.expect_run_boottool("--info=5", RESULT)
+ # run the test
+ info = self.loader.get_info(5)
+ self.god.check_playback()
+ expected_info = {"index": "5", "args": "ro single",
+ "boot": "(hd0,0)",
+ "initrd": "/boot/initrd.img-2.6.15-23-386",
+ "kernel": "/boot/vmlinuz-2.6.15-23-386",
+ "root": "UUID=07D7-0714", "savedefault": "",
+ "title": "Distro, kernel 2.6.15-23-386"}
+ self.assertEquals(expected_info, info)
- def test_get_info_missing_result(self):
- # set up the recording
- self.expect_run_boottool("--info=4", "")
- # run the test
- info = self.loader.get_info(4)
- self.god.check_playback()
- self.assertEquals({}, info)
+ def test_get_info_missing_result(self):
+ # set up the recording
+ self.expect_run_boottool("--info=4", "")
+ # run the test
+ info = self.loader.get_info(4)
+ self.god.check_playback()
+ self.assertEquals({}, info)
- def test_get_info_multiple_results(self):
- RESULT = (
- "index\t: 5\n"
- "args\t: ro single\n"
- "boot\t: (hd0,0)\n"
- "initrd\t: /boot/initrd.img-2.6.15-23-386\n"
- "kernel\t: /boot/vmlinuz-2.6.15-23-386\n"
- "root\t: UUID=07D7-0714\n"
- "savedefault\t: \n"
- "title\t: Distro, kernel 2.6.15-23-386\n"
- "\n"
- "index\t: 7\n"
- "args\t: ro single\n"
- "boot\t: (hd0,0)\n"
- "initrd\t: /boot/initrd.img-2.6.15-23-686\n"
- "kernel\t: /boot/vmlinuz-2.6.15-23-686\n"
- "root\t: UUID=07D7-0714\n"
- "savedefault\t: \n"
- "title\t: Distro, kernel 2.6.15-23-686\n"
- )
- # set up the recording
- self.expect_run_boottool("--info=all", RESULT)
- # run the test
- info = self.loader.get_all_info()
- self.god.check_playback()
- expected_info = [{"index": "5", "args": "ro single",
- "boot": "(hd0,0)",
- "initrd": "/boot/initrd.img-2.6.15-23-386",
- "kernel": "/boot/vmlinuz-2.6.15-23-386",
- "root": "UUID=07D7-0714", "savedefault": "",
- "title": "Distro, kernel 2.6.15-23-386"},
- {"index": "7", "args": "ro single",
- "boot": "(hd0,0)",
- "initrd": "/boot/initrd.img-2.6.15-23-686",
- "kernel": "/boot/vmlinuz-2.6.15-23-686",
- "root": "UUID=07D7-0714", "savedefault": "",
- "title": "Distro, kernel 2.6.15-23-686"}]
- self.assertEquals(expected_info, info)
+ def test_get_info_multiple_results(self):
+ RESULT = (
+ "index\t: 5\n"
+ "args\t: ro single\n"
+ "boot\t: (hd0,0)\n"
+ "initrd\t: /boot/initrd.img-2.6.15-23-386\n"
+ "kernel\t: /boot/vmlinuz-2.6.15-23-386\n"
+ "root\t: UUID=07D7-0714\n"
+ "savedefault\t: \n"
+ "title\t: Distro, kernel 2.6.15-23-386\n"
+ "\n"
+ "index\t: 7\n"
+ "args\t: ro single\n"
+ "boot\t: (hd0,0)\n"
+ "initrd\t: /boot/initrd.img-2.6.15-23-686\n"
+ "kernel\t: /boot/vmlinuz-2.6.15-23-686\n"
+ "root\t: UUID=07D7-0714\n"
+ "savedefault\t: \n"
+ "title\t: Distro, kernel 2.6.15-23-686\n"
+ )
+ # set up the recording
+ self.expect_run_boottool("--info=all", RESULT)
+ # run the test
+ info = self.loader.get_all_info()
+ self.god.check_playback()
+ expected_info = [{"index": "5", "args": "ro single",
+ "boot": "(hd0,0)",
+ "initrd": "/boot/initrd.img-2.6.15-23-386",
+ "kernel": "/boot/vmlinuz-2.6.15-23-386",
+ "root": "UUID=07D7-0714", "savedefault": "",
+ "title": "Distro, kernel 2.6.15-23-386"},
+ {"index": "7", "args": "ro single",
+ "boot": "(hd0,0)",
+ "initrd": "/boot/initrd.img-2.6.15-23-686",
+ "kernel": "/boot/vmlinuz-2.6.15-23-686",
+ "root": "UUID=07D7-0714", "savedefault": "",
+ "title": "Distro, kernel 2.6.15-23-686"}]
+ self.assertEquals(expected_info, info)
- def test_set_default(self):
- # set up the recording
- self.loader._run_boottool.expect_call("--set-default=41")
- # run the test
- self.loader.set_default(41)
- self.god.check_playback()
+ def test_set_default(self):
+ # set up the recording
+ self.loader._run_boottool.expect_call("--set-default=41")
+ # run the test
+ self.loader.set_default(41)
+ self.god.check_playback()
- def test_add_args(self):
- # set up the recording
- self.loader._run_boottool.expect_call(
- "--update-kernel=10 --args=\"some kernel args\"")
- # run the test
- self.loader.add_args(10, "some kernel args")
- self.god.check_playback()
+ def test_add_args(self):
+ # set up the recording
+ self.loader._run_boottool.expect_call(
+ "--update-kernel=10 --args=\"some kernel args\"")
+ # run the test
+ self.loader.add_args(10, "some kernel args")
+ self.god.check_playback()
- def test_remove_args(self):
- # set up the recording
- self.loader._run_boottool.expect_call(
- "--update-kernel=12 --remove-args=\"some kernel args\"")
- # run the test
- self.loader.remove_args(12, "some kernel args")
- self.god.check_playback()
+ def test_remove_args(self):
+ # set up the recording
+ self.loader._run_boottool.expect_call(
+ "--update-kernel=12 --remove-args=\"some kernel args\"")
+ # run the test
+ self.loader.remove_args(12, "some kernel args")
+ self.god.check_playback()
- def test_add_kernel_basic(self):
- self.loader.get_titles = self.god.create_mock_function(
- "get_titles")
- # set up the recording
- self.loader.get_titles.expect_call().and_return(["notmylabel"])
- self.loader._run_boottool.expect_call(
- "--add-kernel \"/unittest/kernels/vmlinuz\" "
- "--title \"mylabel\" --make-default")
- # run the test
- self.loader.add_kernel("/unittest/kernels/vmlinuz",
- "mylabel")
- self.god.check_playback()
+ def test_add_kernel_basic(self):
+ self.loader.get_titles = self.god.create_mock_function(
+ "get_titles")
+ # set up the recording
+ self.loader.get_titles.expect_call().and_return(["notmylabel"])
+ self.loader._run_boottool.expect_call(
+ "--add-kernel \"/unittest/kernels/vmlinuz\" "
+ "--title \"mylabel\" --make-default")
+ # run the test
+ self.loader.add_kernel("/unittest/kernels/vmlinuz",
+ "mylabel")
+ self.god.check_playback()
- def test_add_kernel_adds_root(self):
- self.loader.get_titles = self.god.create_mock_function(
- "get_titles")
- # set up the recording
- self.loader.get_titles.expect_call().and_return(["notmylabel"])
- self.loader._run_boottool.expect_call(
- "--add-kernel \"/unittest/kernels/vmlinuz\" "
- "--title \"mylabel\" --root \"/unittest/root\" "
- "--make-default")
- # run the test
- self.loader.add_kernel("/unittest/kernels/vmlinuz",
- "mylabel", root="/unittest/root")
- self.god.check_playback()
+ def test_add_kernel_adds_root(self):
+ self.loader.get_titles = self.god.create_mock_function(
+ "get_titles")
+ # set up the recording
+ self.loader.get_titles.expect_call().and_return(["notmylabel"])
+ self.loader._run_boottool.expect_call(
+ "--add-kernel \"/unittest/kernels/vmlinuz\" "
+ "--title \"mylabel\" --root \"/unittest/root\" "
+ "--make-default")
+ # run the test
+ self.loader.add_kernel("/unittest/kernels/vmlinuz",
+ "mylabel", root="/unittest/root")
+ self.god.check_playback()
- def test_add_kernel_adds_args(self):
- self.loader.get_titles = self.god.create_mock_function(
- "get_titles")
- # set up the recording
- self.loader.get_titles.expect_call().and_return(["notmylabel"])
- self.loader._run_boottool.expect_call(
- "--add-kernel \"/unittest/kernels/vmlinuz\" "
- "--title \"mylabel\" --args \"my kernel args\" "
- "--make-default")
- # run the test
- self.loader.add_kernel("/unittest/kernels/vmlinuz",
- "mylabel", args="my kernel args")
- self.god.check_playback()
+ def test_add_kernel_adds_args(self):
+ self.loader.get_titles = self.god.create_mock_function(
+ "get_titles")
+ # set up the recording
+ self.loader.get_titles.expect_call().and_return(["notmylabel"])
+ self.loader._run_boottool.expect_call(
+ "--add-kernel \"/unittest/kernels/vmlinuz\" "
+ "--title \"mylabel\" --args \"my kernel args\" "
+ "--make-default")
+ # run the test
+ self.loader.add_kernel("/unittest/kernels/vmlinuz",
+ "mylabel", args="my kernel args")
+ self.god.check_playback()
- def test_add_kernel_adds_initrd(self):
- self.loader.get_titles = self.god.create_mock_function(
- "get_titles")
- # set up the recording
- self.loader.get_titles.expect_call().and_return(["notmylabel"])
- self.loader._run_boottool.expect_call(
- "--add-kernel \"/unittest/kernels/vmlinuz\" "
- "--title \"mylabel\" --initrd \"/unittest/initrd\" "
- "--make-default")
- # run the test
- self.loader.add_kernel("/unittest/kernels/vmlinuz",
- "mylabel", initrd="/unittest/initrd")
- self.god.check_playback()
+ def test_add_kernel_adds_initrd(self):
+ self.loader.get_titles = self.god.create_mock_function(
+ "get_titles")
+ # set up the recording
+ self.loader.get_titles.expect_call().and_return(["notmylabel"])
+ self.loader._run_boottool.expect_call(
+ "--add-kernel \"/unittest/kernels/vmlinuz\" "
+ "--title \"mylabel\" --initrd \"/unittest/initrd\" "
+ "--make-default")
+ # run the test
+ self.loader.add_kernel("/unittest/kernels/vmlinuz",
+ "mylabel", initrd="/unittest/initrd")
+ self.god.check_playback()
- def test_add_kernel_disables_make_default(self):
- self.loader.get_titles = self.god.create_mock_function(
- "get_titles")
- # set up the recording
- self.loader.get_titles.expect_call().and_return(["notmylabel"])
- self.loader._run_boottool.expect_call(
- "--add-kernel \"/unittest/kernels/vmlinuz\" "
- "--title \"mylabel\"")
- # run the test
- self.loader.add_kernel("/unittest/kernels/vmlinuz",
- "mylabel", default=False)
- self.god.check_playback()
+ def test_add_kernel_disables_make_default(self):
+ self.loader.get_titles = self.god.create_mock_function(
+ "get_titles")
+ # set up the recording
+ self.loader.get_titles.expect_call().and_return(["notmylabel"])
+ self.loader._run_boottool.expect_call(
+ "--add-kernel \"/unittest/kernels/vmlinuz\" "
+ "--title \"mylabel\"")
+ # run the test
+ self.loader.add_kernel("/unittest/kernels/vmlinuz",
+ "mylabel", default=False)
+ self.god.check_playback()
- def test_add_kernel_removes_old(self):
- self.loader.get_titles = self.god.create_mock_function(
- "get_titles")
- # set up the recording
- self.loader.get_titles.expect_call().and_return(["mylabel"])
- self.loader._run_boottool.expect_call(
- "--remove-kernel \"mylabel\"")
- self.loader._run_boottool.expect_call(
- "--add-kernel \"/unittest/kernels/vmlinuz\" "
- "--title \"mylabel\" --make-default")
- # run the test
- self.loader.add_kernel("/unittest/kernels/vmlinuz",
- "mylabel")
- self.god.check_playback()
+ def test_add_kernel_removes_old(self):
+ self.loader.get_titles = self.god.create_mock_function(
+ "get_titles")
+ # set up the recording
+ self.loader.get_titles.expect_call().and_return(["mylabel"])
+ self.loader._run_boottool.expect_call(
+ "--remove-kernel \"mylabel\"")
+ self.loader._run_boottool.expect_call(
+ "--add-kernel \"/unittest/kernels/vmlinuz\" "
+ "--title \"mylabel\" --make-default")
+ # run the test
+ self.loader.add_kernel("/unittest/kernels/vmlinuz",
+ "mylabel")
+ self.god.check_playback()
- def test_remove_kernel(self):
- # set up the recording
- self.loader._run_boottool.expect_call("--remove-kernel=14")
- # run the test
- self.loader.remove_kernel(14)
- self.god.check_playback()
+ def test_remove_kernel(self):
+ # set up the recording
+ self.loader._run_boottool.expect_call("--remove-kernel=14")
+ # run the test
+ self.loader.remove_kernel(14)
+ self.god.check_playback()
- def test_boot_once(self):
- # set up the recording
- self.loader._run_boottool.expect_call(
- "--boot-once --title=autotest")
- # run the test
- self.loader.boot_once("autotest")
- self.god.check_playback()
+ def test_boot_once(self):
+ # set up the recording
+ self.loader._run_boottool.expect_call(
+ "--boot-once --title=autotest")
+ # run the test
+ self.loader.boot_once("autotest")
+ self.god.check_playback()
if __name__ == "__main__":
- unittest.main()
+ unittest.main()
diff --git a/server/hosts/guest.py b/server/hosts/guest.py
index 0d8270c..8fe6bc2 100644
--- a/server/hosts/guest.py
+++ b/server/hosts/guest.py
@@ -8,7 +8,7 @@
Implementation details:
You should import the "hosts" package instead of importing each type of host.
- Guest: a virtual machine on which you can run programs
+ Guest: a virtual machine on which you can run programs
"""
__author__ = """
@@ -22,49 +22,49 @@
class Guest(ssh_host.SSHHost):
- """
- This class represents a virtual machine on which you can run
- programs.
-
- It is not the machine autoserv is running on.
-
- Implementation details:
- This is an abstract class, leaf subclasses must implement the methods
- listed here and in parent classes which have no implementation. They
- may reimplement methods which already have an implementation. You
- must not instantiate this class but should instantiate one of those
- leaf subclasses.
- """
-
- controlling_hypervisor = None
+ """
+ This class represents a virtual machine on which you can run
+ programs.
-
- def __init__(self, controlling_hypervisor):
- """
- Construct a Guest object
-
- Args:
- controlling_hypervisor: Hypervisor object that is
- responsible for the creation and management of
- this guest
- """
- hostname= controlling_hypervisor.new_guest()
- super(Guest, self).__init__(hostname)
- self.controlling_hypervisor= controlling_hypervisor
+ It is not the machine autoserv is running on.
-
- def __del__(self):
- """
- Destroy a Guest object
- """
- self.controlling_hypervisor.delete_guest(self.hostname)
+ Implementation details:
+ This is an abstract class, leaf subclasses must implement the methods
+ listed here and in parent classes which have no implementation. They
+ may reimplement methods which already have an implementation. You
+ must not instantiate this class but should instantiate one of those
+ leaf subclasses.
+ """
-
- def hardreset(self, timeout=600, wait=True):
- """
- Perform a "hardreset" of the guest.
-
- It is restarted through the hypervisor. That will restart it
- even if the guest otherwise innaccessible through ssh.
- """
- return self.controlling_hypervisor.reset_guest(self.hostname)
+ controlling_hypervisor = None
+
+
+ def __init__(self, controlling_hypervisor):
+ """
+ Construct a Guest object
+
+ Args:
+ controlling_hypervisor: Hypervisor object that is
+ responsible for the creation and management of
+ this guest
+ """
+ hostname= controlling_hypervisor.new_guest()
+ super(Guest, self).__init__(hostname)
+ self.controlling_hypervisor= controlling_hypervisor
+
+
+ def __del__(self):
+ """
+ Destroy a Guest object
+ """
+ self.controlling_hypervisor.delete_guest(self.hostname)
+
+
+ def hardreset(self, timeout=600, wait=True):
+ """
+ Perform a "hardreset" of the guest.
+
+ It is restarted through the hypervisor. That will restart it
+ even if the guest otherwise innaccessible through ssh.
+ """
+ return self.controlling_hypervisor.reset_guest(self.hostname)
diff --git a/server/hosts/kvm_guest.py b/server/hosts/kvm_guest.py
index a676cba..ae0acc8 100644
--- a/server/hosts/kvm_guest.py
+++ b/server/hosts/kvm_guest.py
@@ -8,7 +8,7 @@
Implementation details:
You should import the "hosts" package instead of importing each type of host.
- KVMGuest: a KVM virtual machine on which you can run programs
+ KVMGuest: a KVM virtual machine on which you can run programs
"""
__author__ = """
@@ -22,26 +22,26 @@
class KVMGuest(guest.Guest):
- """This class represents a KVM virtual machine on which you can run
- programs.
-
- Implementation details:
- This is a leaf class in an abstract class hierarchy, it must
- implement the unimplemented methods in parent classes.
- """
-
- def __init__(self, controlling_hypervisor, qemu_options):
- """
- Construct a KVMGuest object
-
- Args:
- controlling_hypervisor: hypervisor object that is
- responsible for the creation and management of
- this guest
- qemu_options: options to pass to qemu, these should be
- appropriately shell escaped, if need be.
- """
- hostname= controlling_hypervisor.new_guest(qemu_options)
- # bypass Guest's __init__
- super(guest.Guest, self).__init__(hostname)
- self.controlling_hypervisor= controlling_hypervisor
+ """This class represents a KVM virtual machine on which you can run
+ programs.
+
+ Implementation details:
+ This is a leaf class in an abstract class hierarchy, it must
+ implement the unimplemented methods in parent classes.
+ """
+
+ def __init__(self, controlling_hypervisor, qemu_options):
+ """
+ Construct a KVMGuest object
+
+ Args:
+ controlling_hypervisor: hypervisor object that is
+ responsible for the creation and management of
+ this guest
+ qemu_options: options to pass to qemu, these should be
+ appropriately shell escaped, if need be.
+ """
+ hostname= controlling_hypervisor.new_guest(qemu_options)
+ # bypass Guest's __init__
+ super(guest.Guest, self).__init__(hostname)
+ self.controlling_hypervisor= controlling_hypervisor
diff --git a/server/hosts/remote.py b/server/hosts/remote.py
index 9f5e955..84bb400 100644
--- a/server/hosts/remote.py
+++ b/server/hosts/remote.py
@@ -1,32 +1,32 @@
"""This class defines the Remote host class, mixing in the SiteHost class
if it is available."""
-# site_host.py may be non-existant or empty, make sure that an appropriate
+# site_host.py may be non-existant or empty, make sure that an appropriate
# SiteHost class is created nevertheless
try:
- from site_host import SiteHost
+ from site_host import SiteHost
except ImportError:
- import base_classes
- class SiteHost(base_classes.Host):
- def __init__(self):
- super(SiteHost, self).__init__()
+ import base_classes
+ class SiteHost(base_classes.Host):
+ def __init__(self):
+ super(SiteHost, self).__init__()
class RemoteHost(SiteHost):
- """This class represents a remote machine on which you can run
- programs.
+ """This class represents a remote machine on which you can run
+ programs.
- It may be accessed through a network, a serial line, ...
- It is not the machine autoserv is running on.
+ It may be accessed through a network, a serial line, ...
+ It is not the machine autoserv is running on.
- Implementation details:
- This is an abstract class, leaf subclasses must implement the methods
- listed here and in parent classes which have no implementation. They
- may reimplement methods which already have an implementation. You
- must not instantiate this class but should instantiate one of those
- leaf subclasses."""
+ Implementation details:
+ This is an abstract class, leaf subclasses must implement the methods
+ listed here and in parent classes which have no implementation. They
+ may reimplement methods which already have an implementation. You
+ must not instantiate this class but should instantiate one of those
+ leaf subclasses."""
- hostname= None
+ hostname= None
- def __init__(self):
- super(RemoteHost, self).__init__()
+ def __init__(self):
+ super(RemoteHost, self).__init__()
diff --git a/server/hosts/site_host.py b/server/hosts/site_host.py
index 4be6a4d..c0b6693 100644
--- a/server/hosts/site_host.py
+++ b/server/hosts/site_host.py
@@ -11,7 +11,7 @@
Implementation details:
You should import the "hosts" package instead of importing each type of host.
- SiteHost: Host containing site-specific customizations.
+ SiteHost: Host containing site-specific customizations.
"""
__author__ = """
@@ -24,14 +24,14 @@
import base_classes, utils
class SiteHost(base_classes.Host):
- """
- Custom host to containing site-specific methods or attributes.
- """
-
- def __init__(self):
- super(SiteHost, self).__init__()
- self.serverdir = utils.get_server_dir()
+ """
+ Custom host to containing site-specific methods or attributes.
+ """
-
- def setup(self):
- return
+ def __init__(self):
+ super(SiteHost, self).__init__()
+ self.serverdir = utils.get_server_dir()
+
+
+ def setup(self):
+ return
diff --git a/server/hosts/ssh_host.py b/server/hosts/ssh_host.py
index 378d048..c4ce4ab 100644
--- a/server/hosts/ssh_host.py
+++ b/server/hosts/ssh_host.py
@@ -8,7 +8,7 @@
Implementation details:
You should import the "hosts" package instead of importing each type of host.
- SSHHost: a remote machine with a ssh access
+ SSHHost: a remote machine with a ssh access
"""
__author__ = """
@@ -26,869 +26,869 @@
class PermissionDeniedError(error.AutoservRunError):
- pass
+ pass
class SSHHost(remote.RemoteHost):
- """
- This class represents a remote machine controlled through an ssh
- session on which you can run programs.
+ """
+ This class represents a remote machine controlled through an ssh
+ session on which you can run programs.
- It is not the machine autoserv is running on. The machine must be
- configured for password-less login, for example through public key
- authentication.
+ It is not the machine autoserv is running on. The machine must be
+ configured for password-less login, for example through public key
+ authentication.
- It includes support for controlling the machine through a serial
- console on which you can run programs. If such a serial console is
- set up on the machine then capabilities such as hard reset and
- boot strap monitoring are available. If the machine does not have a
- serial console available then ordinary SSH-based commands will
- still be available, but attempts to use extensions such as
- console logging or hard reset will fail silently.
+ It includes support for controlling the machine through a serial
+ console on which you can run programs. If such a serial console is
+ set up on the machine then capabilities such as hard reset and
+ boot strap monitoring are available. If the machine does not have a
+ serial console available then ordinary SSH-based commands will
+ still be available, but attempts to use extensions such as
+ console logging or hard reset will fail silently.
- Implementation details:
- This is a leaf class in an abstract class hierarchy, it must
- implement the unimplemented methods in parent classes.
- """
+ Implementation details:
+ This is a leaf class in an abstract class hierarchy, it must
+ implement the unimplemented methods in parent classes.
+ """
- DEFAULT_REBOOT_TIMEOUT = 1800
- job = None
+ DEFAULT_REBOOT_TIMEOUT = 1800
+ job = None
- def __init__(self, hostname, user="root", port=22, initialize=True,
- conmux_log="console.log",
- conmux_server=None, conmux_attach=None,
- netconsole_log=None, netconsole_port=6666, autodir=None,
- password=''):
- """
- Construct a SSHHost object
-
- Args:
- hostname: network hostname or address of remote machine
- user: user to log in as on the remote machine
- port: port the ssh daemon is listening on on the remote
- machine
- """
- self.hostname= hostname
- self.user= user
- self.port= port
- self.tmp_dirs= []
- self.initialize = initialize
- self.autodir = autodir
- self.password = password
+ def __init__(self, hostname, user="root", port=22, initialize=True,
+ conmux_log="console.log",
+ conmux_server=None, conmux_attach=None,
+ netconsole_log=None, netconsole_port=6666, autodir=None,
+ password=''):
+ """
+ Construct a SSHHost object
- super(SSHHost, self).__init__()
+ Args:
+ hostname: network hostname or address of remote machine
+ user: user to log in as on the remote machine
+ port: port the ssh daemon is listening on on the remote
+ machine
+ """
+ self.hostname= hostname
+ self.user= user
+ self.port= port
+ self.tmp_dirs= []
+ self.initialize = initialize
+ self.autodir = autodir
+ self.password = password
- self.conmux_server = conmux_server
- if conmux_attach:
- self.conmux_attach = conmux_attach
- else:
- self.conmux_attach = os.path.abspath(os.path.join(
- self.serverdir, '..',
- 'conmux', 'conmux-attach'))
- self.logger_popen = None
- self.warning_stream = None
- self.__start_console_log(conmux_log)
+ super(SSHHost, self).__init__()
- self.bootloader = bootloader.Bootloader(self)
+ self.conmux_server = conmux_server
+ if conmux_attach:
+ self.conmux_attach = conmux_attach
+ else:
+ self.conmux_attach = os.path.abspath(os.path.join(
+ self.serverdir, '..',
+ 'conmux', 'conmux-attach'))
+ self.logger_popen = None
+ self.warning_stream = None
+ self.__start_console_log(conmux_log)
- self.__netconsole_param = ""
- self.netlogger_popen = None
- if netconsole_log:
- self.__init_netconsole_params(netconsole_port)
- self.__start_netconsole_log(netconsole_log, netconsole_port)
- self.__load_netconsole_module()
+ self.bootloader = bootloader.Bootloader(self)
+ self.__netconsole_param = ""
+ self.netlogger_popen = None
+ if netconsole_log:
+ self.__init_netconsole_params(netconsole_port)
+ self.__start_netconsole_log(netconsole_log, netconsole_port)
+ self.__load_netconsole_module()
- @staticmethod
- def __kill(popen):
- return_code = popen.poll()
- if return_code is None:
- try:
- os.kill(popen.pid, signal.SIGTERM)
- except OSError:
- pass
+ @staticmethod
+ def __kill(popen):
+ return_code = popen.poll()
+ if return_code is None:
+ try:
+ os.kill(popen.pid, signal.SIGTERM)
+ except OSError:
+ pass
- def __del__(self):
- """
- Destroy a SSHHost object
- """
- for dir in self.tmp_dirs:
- try:
- self.run('rm -rf "%s"' % (utils.sh_escape(dir)))
- except error.AutoservRunError:
- pass
- # kill the console logger
- if getattr(self, 'logger_popen', None):
- self.__kill(self.logger_popen)
- if self.job:
- self.job.warning_loggers.discard(
- self.warning_stream)
- self.warning_stream.close()
- # kill the netconsole logger
- if getattr(self, 'netlogger_popen', None):
- self.__unload_netconsole_module()
- self.__kill(self.netlogger_popen)
+ def __del__(self):
+ """
+ Destroy a SSHHost object
+ """
+ for dir in self.tmp_dirs:
+ try:
+ self.run('rm -rf "%s"' % (utils.sh_escape(dir)))
+ except error.AutoservRunError:
+ pass
+ # kill the console logger
+ if getattr(self, 'logger_popen', None):
+ self.__kill(self.logger_popen)
+ if self.job:
+ self.job.warning_loggers.discard(
+ self.warning_stream)
+ self.warning_stream.close()
+ # kill the netconsole logger
+ if getattr(self, 'netlogger_popen', None):
+ self.__unload_netconsole_module()
+ self.__kill(self.netlogger_popen)
- def __init_netconsole_params(self, port):
- """
- Connect to the remote machine and determine the values to use for the
- required netconsole parameters.
- """
- # PROBLEM: on machines with multiple IPs this may not make any sense
- # It also doesn't work with IPv6
- remote_ip = socket.gethostbyname(self.hostname)
- local_ip = socket.gethostbyname(socket.gethostname())
- # Get the gateway of the remote machine
- try:
- traceroute = self.run('traceroute -n %s' % local_ip)
- except error.AutoservRunError:
- return
- first_node = traceroute.stdout.split("\n")[0]
- match = re.search(r'\s+((\d+\.){3}\d+)\s+', first_node)
- if match:
- router_ip = match.group(1)
- else:
- return
- # Look up the MAC address of the gateway
- try:
- self.run('ping -c 1 %s' % router_ip)
- arp = self.run('arp -n -a %s' % router_ip)
- except error.AutoservRunError:
- return
- match = re.search(r'\s+(([0-9A-F]{2}:){5}[0-9A-F]{2})\s+', arp.stdout)
- if match:
- gateway_mac = match.group(1)
- else:
- return
- self.__netconsole_param = 'netconsole=@%s/,%s@%s/%s' % (remote_ip,
- port,
- local_ip,
- gateway_mac)
+ def __init_netconsole_params(self, port):
+ """
+ Connect to the remote machine and determine the values to use for the
+ required netconsole parameters.
+ """
+ # PROBLEM: on machines with multiple IPs this may not make any sense
+ # It also doesn't work with IPv6
+ remote_ip = socket.gethostbyname(self.hostname)
+ local_ip = socket.gethostbyname(socket.gethostname())
+ # Get the gateway of the remote machine
+ try:
+ traceroute = self.run('traceroute -n %s' % local_ip)
+ except error.AutoservRunError:
+ return
+ first_node = traceroute.stdout.split("\n")[0]
+ match = re.search(r'\s+((\d+\.){3}\d+)\s+', first_node)
+ if match:
+ router_ip = match.group(1)
+ else:
+ return
+ # Look up the MAC address of the gateway
+ try:
+ self.run('ping -c 1 %s' % router_ip)
+ arp = self.run('arp -n -a %s' % router_ip)
+ except error.AutoservRunError:
+ return
+ match = re.search(r'\s+(([0-9A-F]{2}:){5}[0-9A-F]{2})\s+', arp.stdout)
+ if match:
+ gateway_mac = match.group(1)
+ else:
+ return
+ self.__netconsole_param = 'netconsole=@%s/,%s@%s/%s' % (remote_ip,
+ port,
+ local_ip,
+ gateway_mac)
- def __start_netconsole_log(self, logfilename, port):
- """
- Log the output of netconsole to a specified file
- """
- if logfilename == None:
- return
- cmd = ['nc', '-u', '-l', '-p', str(port)]
- logfile = open(logfilename, 'a', 0)
- self.netlogger_popen = subprocess.Popen(cmd, stdout=logfile)
+ def __start_netconsole_log(self, logfilename, port):
+ """
+ Log the output of netconsole to a specified file
+ """
+ if logfilename == None:
+ return
+ cmd = ['nc', '-u', '-l', '-p', str(port)]
+ logfile = open(logfilename, 'a', 0)
+ self.netlogger_popen = subprocess.Popen(cmd, stdout=logfile)
- def __load_netconsole_module(self):
- """
- Make a best effort to load the netconsole module.
- Note that loading the module can fail even when the remote machine is
- working correctly if netconsole is already compiled into the kernel
- and started.
- """
- if not self.__netconsole_param:
- return
- try:
- self.run('modprobe netconsole %s' % self.__netconsole_param)
- except error.AutoservRunError:
- # if it fails there isn't much we can do, just keep going
- pass
+ def __load_netconsole_module(self):
+ """
+ Make a best effort to load the netconsole module.
+ Note that loading the module can fail even when the remote machine is
+ working correctly if netconsole is already compiled into the kernel
+ and started.
+ """
+ if not self.__netconsole_param:
+ return
+ try:
+ self.run('modprobe netconsole %s' % self.__netconsole_param)
+ except error.AutoservRunError:
+ # if it fails there isn't much we can do, just keep going
+ pass
- def __unload_netconsole_module(self):
- try:
- self.run('modprobe -r netconsole')
- except error.AutoservRunError:
- pass
+ def __unload_netconsole_module(self):
+ try:
+ self.run('modprobe -r netconsole')
+ except error.AutoservRunError:
+ pass
- def wait_for_restart(self, timeout=DEFAULT_REBOOT_TIMEOUT):
- if not self.wait_down(300): # Make sure he's dead, Jim
- self.__record("ABORT", None, "reboot.verify", "shutdown failed")
- raise error.AutoservRebootError(
- "Host did not shut down")
- self.wait_up(timeout)
- time.sleep(2) # this is needed for complete reliability
- if self.wait_up(timeout):
- self.__record("GOOD", None, "reboot.verify")
- else:
- self.__record("ABORT", None, "reboot.verify", "Host did not return from reboot")
- raise error.AutoservRebootError(
- "Host did not return from reboot")
- print "Reboot complete"
+ def wait_for_restart(self, timeout=DEFAULT_REBOOT_TIMEOUT):
+ if not self.wait_down(300): # Make sure he's dead, Jim
+ self.__record("ABORT", None, "reboot.verify", "shutdown failed")
+ raise error.AutoservRebootError(
+ "Host did not shut down")
+ self.wait_up(timeout)
+ time.sleep(2) # this is needed for complete reliability
+ if self.wait_up(timeout):
+ self.__record("GOOD", None, "reboot.verify")
+ else:
+ self.__record("ABORT", None, "reboot.verify", "Host did not return from reboot")
+ raise error.AutoservRebootError(
+ "Host did not return from reboot")
+ print "Reboot complete"
- def hardreset(self, timeout=DEFAULT_REBOOT_TIMEOUT, wait=True,
- conmux_command='hardreset'):
- """
- Reach out and slap the box in the power switch.
- Args:
- conmux_command: The command to run via the conmux interface
- timeout: timelimit in seconds before the machine is considered unreachable
- wait: Whether or not to wait for the machine to reboot
-
- """
- conmux_command = r"'~$%s'" % conmux_command
- if not self.__console_run(conmux_command):
- self.__record("ABORT", None, "reboot.start", "hard reset unavailable")
- raise error.AutoservUnsupportedError(
- 'Hard reset unavailable')
- if wait:
- self.wait_for_restart(timeout)
- self.__record("GOOD", None, "reboot.start", "hard reset")
+ def hardreset(self, timeout=DEFAULT_REBOOT_TIMEOUT, wait=True,
+ conmux_command='hardreset'):
+ """
+ Reach out and slap the box in the power switch.
+ Args:
+ conmux_command: The command to run via the conmux interface
+ timeout: timelimit in seconds before the machine is considered unreachable
+ wait: Whether or not to wait for the machine to reboot
+ """
+ conmux_command = r"'~$%s'" % conmux_command
+ if not self.__console_run(conmux_command):
+ self.__record("ABORT", None, "reboot.start", "hard reset unavailable")
+ raise error.AutoservUnsupportedError(
+ 'Hard reset unavailable')
- def __conmux_hostname(self):
- if self.conmux_server:
- return '%s/%s' % (self.conmux_server, self.hostname)
- else:
- return self.hostname
+ if wait:
+ self.wait_for_restart(timeout)
+ self.__record("GOOD", None, "reboot.start", "hard reset")
- def __start_console_log(self, logfilename):
- """
- Log the output of the console session to a specified file
- """
- if logfilename == None:
- return
- if not self.conmux_attach or not os.path.exists(self.conmux_attach):
- return
+ def __conmux_hostname(self):
+ if self.conmux_server:
+ return '%s/%s' % (self.conmux_server, self.hostname)
+ else:
+ return self.hostname
- r, w = os.pipe()
- script_path = os.path.join(self.serverdir,
- 'warning_monitor.py')
- cmd = [self.conmux_attach, self.__conmux_hostname(),
- '%s %s %s %d' % (sys.executable, script_path,
- logfilename, w)]
- dev_null = open(os.devnull, 'w')
- self.warning_stream = os.fdopen(r, 'r', 0)
- if self.job:
- self.job.warning_loggers.add(self.warning_stream)
- self.logger_popen = subprocess.Popen(cmd, stderr=dev_null)
- os.close(w)
+ def __start_console_log(self, logfilename):
+ """
+ Log the output of the console session to a specified file
+ """
+ if logfilename == None:
+ return
+ if not self.conmux_attach or not os.path.exists(self.conmux_attach):
+ return
+ r, w = os.pipe()
+ script_path = os.path.join(self.serverdir,
+ 'warning_monitor.py')
+ cmd = [self.conmux_attach, self.__conmux_hostname(),
+ '%s %s %s %d' % (sys.executable, script_path,
+ logfilename, w)]
+ dev_null = open(os.devnull, 'w')
- def __console_run(self, cmd):
- """
- Send a command to the conmux session
- """
- if not self.conmux_attach or not os.path.exists(self.conmux_attach):
- return False
- cmd = '%s %s echo %s 2> /dev/null' % (self.conmux_attach,
- self.__conmux_hostname(),
- cmd)
- result = utils.system(cmd, ignore_status=True)
- return result == 0
+ self.warning_stream = os.fdopen(r, 'r', 0)
+ if self.job:
+ self.job.warning_loggers.add(self.warning_stream)
+ self.logger_popen = subprocess.Popen(cmd, stderr=dev_null)
+ os.close(w)
- def __run_reboot_group(self, reboot_func):
- if self.job:
- self.job.run_reboot(reboot_func, self.get_kernel_ver)
- else:
- reboot_func()
+ def __console_run(self, cmd):
+ """
+ Send a command to the conmux session
+ """
+ if not self.conmux_attach or not os.path.exists(self.conmux_attach):
+ return False
+ cmd = '%s %s echo %s 2> /dev/null' % (self.conmux_attach,
+ self.__conmux_hostname(),
+ cmd)
+ result = utils.system(cmd, ignore_status=True)
+ return result == 0
- def __record(self, status_code, subdir, operation, status = ''):
- if self.job:
- self.job.record(status_code, subdir, operation, status)
- else:
- if not subdir:
- subdir = "----"
- msg = "%s\t%s\t%s\t%s" % (status_code, subdir, operation, status)
- sys.stderr.write(msg + "\n")
+ def __run_reboot_group(self, reboot_func):
+ if self.job:
+ self.job.run_reboot(reboot_func, self.get_kernel_ver)
+ else:
+ reboot_func()
- def ssh_base_command(self, connect_timeout=30):
- SSH_BASE_COMMAND = '/usr/bin/ssh -a -x -o ' + \
- 'BatchMode=yes -o ConnectTimeout=%d ' + \
- '-o ServerAliveInterval=300'
- assert isinstance(connect_timeout, (int, long))
- assert connect_timeout > 0 # can't disable the timeout
- return SSH_BASE_COMMAND % connect_timeout
+ def __record(self, status_code, subdir, operation, status = ''):
+ if self.job:
+ self.job.record(status_code, subdir, operation, status)
+ else:
+ if not subdir:
+ subdir = "----"
+ msg = "%s\t%s\t%s\t%s" % (status_code, subdir, operation, status)
+ sys.stderr.write(msg + "\n")
- def ssh_command(self, connect_timeout=30, options=''):
- """Construct an ssh command with proper args for this host."""
- ssh = self.ssh_base_command(connect_timeout)
- return r'%s %s -l %s -p %d %s' % (ssh,
- options,
- self.user,
- self.port,
- self.hostname)
+ def ssh_base_command(self, connect_timeout=30):
+ SSH_BASE_COMMAND = '/usr/bin/ssh -a -x -o ' + \
+ 'BatchMode=yes -o ConnectTimeout=%d ' + \
+ '-o ServerAliveInterval=300'
+ assert isinstance(connect_timeout, (int, long))
+ assert connect_timeout > 0 # can't disable the timeout
+ return SSH_BASE_COMMAND % connect_timeout
- def _run(self, command, timeout, ignore_status, stdout, stderr,
- connect_timeout, env, options):
- """Helper function for run()."""
+ def ssh_command(self, connect_timeout=30, options=''):
+ """Construct an ssh command with proper args for this host."""
+ ssh = self.ssh_base_command(connect_timeout)
+ return r'%s %s -l %s -p %d %s' % (ssh,
+ options,
+ self.user,
+ self.port,
+ self.hostname)
- ssh_cmd = self.ssh_command(connect_timeout, options)
- echo_cmd = 'echo Connected. >&2'
- full_cmd = '%s "%s;%s %s"' % (ssh_cmd, echo_cmd, env,
- utils.sh_escape(command))
- result = utils.run(full_cmd, timeout, True, stdout, stderr)
- # The error messages will show up in band (indistinguishable
- # from stuff sent through the SSH connection), so we have the
- # remote computer echo the message "Connected." before running
- # any command. Since the following 2 errors have to do with
- # connecting, it's safe to do these checks.
- if result.exit_status == 255:
- if re.search(r'^ssh: connect to host .* port .*: '
- r'Connection timed out\r$', result.stderr):
- raise error.AutoservSSHTimeout("ssh timed out",
- result)
- if result.stderr == "Permission denied.\r\n":
- msg = "ssh permission denied"
- raise PermissionDeniedError(msg, result)
+ def _run(self, command, timeout, ignore_status, stdout, stderr,
+ connect_timeout, env, options):
+ """Helper function for run()."""
- if not ignore_status and result.exit_status > 0:
- raise error.AutoservRunError("command execution error",
- result)
+ ssh_cmd = self.ssh_command(connect_timeout, options)
+ echo_cmd = 'echo Connected. >&2'
+ full_cmd = '%s "%s;%s %s"' % (ssh_cmd, echo_cmd, env,
+ utils.sh_escape(command))
+ result = utils.run(full_cmd, timeout, True, stdout, stderr)
- return result
+ # The error messages will show up in band (indistinguishable
+ # from stuff sent through the SSH connection), so we have the
+ # remote computer echo the message "Connected." before running
+ # any command. Since the following 2 errors have to do with
+ # connecting, it's safe to do these checks.
+ if result.exit_status == 255:
+ if re.search(r'^ssh: connect to host .* port .*: '
+ r'Connection timed out\r$', result.stderr):
+ raise error.AutoservSSHTimeout("ssh timed out",
+ result)
+ if result.stderr == "Permission denied.\r\n":
+ msg = "ssh permission denied"
+ raise PermissionDeniedError(msg, result)
+
+ if not ignore_status and result.exit_status > 0:
+ raise error.AutoservRunError("command execution error",
+ result)
+
+ return result
+
+
+ def run(self, command, timeout=3600, ignore_status=False,
+ stdout_tee=None, stderr_tee=None, connect_timeout=30):
+ """
+ Run a command on the remote host.
+
+ Args:
+ command: the command line string
+ timeout: time limit in seconds before attempting to
+ kill the running process. The run() function
+ will take a few seconds longer than 'timeout'
+ to complete if it has to kill the process.
+ ignore_status: do not raise an exception, no matter
+ what the exit code of the command is.
+
+ Returns:
+ a hosts.base_classes.CmdResult object
+
+ Raises:
+ AutoservRunError: the exit code of the command
+ execution was not 0
+ AutoservSSHTimeout: ssh connection has timed out
+ """
+ stdout = stdout_tee or sys.stdout
+ stderr = stderr_tee or sys.stdout
+ print "ssh: %s" % command
+ env = " ".join("=".join(pair) for pair in self.env.iteritems())
+ try:
+ try:
+ return self._run(command, timeout,
+ ignore_status, stdout,
+ stderr, connect_timeout,
+ env, '')
+ except PermissionDeniedError:
+ print("Permission denied to ssh; re-running"
+ "with increased logging:")
+ return self._run(command, timeout,
+ ignore_status, stdout,
+ stderr, connect_timeout,
+ env, '-v -v -v')
+ except error.CmdError, cmderr:
+ # We get a CmdError here only if there is timeout of
+ # that command. Catch that and stuff it into
+ # AutoservRunError and raise it.
+ raise error.AutoservRunError(cmderr.args[0],
+ cmderr.args[1])
+
+
+ def run_short(self, command, **kwargs):
+ """
+ Calls the run() command with a short default timeout.
+
+ Args:
+ Takes the same arguments as does run(),
+ with the exception of the timeout argument which
+ here is fixed at 60 seconds.
+ It returns the result of run.
+ """
+ return self.run(command, timeout=60, **kwargs)
+
+
+ def run_grep(self, command, timeout=30, ignore_status=False,
+ stdout_ok_regexp=None, stdout_err_regexp=None,
+ stderr_ok_regexp=None, stderr_err_regexp=None,
+ connect_timeout=30):
+ """
+ Run a command on the remote host and look for regexp
+ in stdout or stderr to determine if the command was
+ successul or not.
+
+ Args:
+ command: the command line string
+ timeout: time limit in seconds before attempting to
+ kill the running process. The run() function
+ will take a few seconds longer than 'timeout'
+ to complete if it has to kill the process.
+ ignore_status: do not raise an exception, no matter
+ what the exit code of the command is.
+ stdout_ok_regexp: regexp that should be in stdout
+ if the command was successul.
+ stdout_err_regexp: regexp that should be in stdout
+ if the command failed.
+ stderr_ok_regexp: regexp that should be in stderr
+ if the command was successul.
+ stderr_err_regexp: regexp that should be in stderr
+ if the command failed.
+
+ Returns:
+ if the command was successul, raises an exception
+ otherwise.
+
+ Raises:
+ AutoservRunError:
+ - the exit code of the command execution was not 0.
+ - If stderr_err_regexp is found in stderr,
+ - If stdout_err_regexp is found in stdout,
+ - If stderr_ok_regexp is not found in stderr.
+ - If stdout_ok_regexp is not found in stdout,
+ """
+
+ # We ignore the status, because we will handle it at the end.
+ result = self.run(command, timeout, ignore_status=True,
+ connect_timeout=connect_timeout)
+
+ # Look for the patterns, in order
+ for (regexp, stream) in ((stderr_err_regexp, result.stderr),
+ (stdout_err_regexp, result.stdout)):
+ if regexp and stream:
+ err_re = re.compile (regexp)
+ if err_re.search(stream):
+ raise error.AutoservRunError(
+ '%s failed, found error pattern: '
+ '"%s"' % (command, regexp), result)
+
+ for (regexp, stream) in ((stderr_ok_regexp, result.stderr),
+ (stdout_ok_regexp, result.stdout)):
+ if regexp and stream:
+ ok_re = re.compile (regexp)
+ if ok_re.search(stream):
+ if ok_re.search(stream):
+ return
+
+ if not ignore_status and result.exit_status > 0:
+ raise error.AutoservRunError("command execution error",
+ result)
+
+
+ def reboot(self, timeout=DEFAULT_REBOOT_TIMEOUT, label=None,
+ kernel_args=None, wait=True):
+ """
+ Reboot the remote host.
+
+ Args:
+ timeout
+ """
+ self.reboot_setup()
+
+ # forcibly include the "netconsole" kernel arg
+ if self.__netconsole_param:
+ if kernel_args is None:
+ kernel_args = self.__netconsole_param
+ else:
+ kernel_args += " " + self.__netconsole_param
+ # unload the (possibly loaded) module to avoid shutdown issues
+ self.__unload_netconsole_module()
+ if label or kernel_args:
+ self.bootloader.install_boottool()
+ if label:
+ self.bootloader.set_default(label)
+ if kernel_args:
+ if not label:
+ default = int(self.bootloader.get_default())
+ label = self.bootloader.get_titles()[default]
+ self.bootloader.add_args(label, kernel_args)
+
+ # define a function for the reboot and run it in a group
+ print "Reboot: initiating reboot"
+ def reboot():
+ self.__record("GOOD", None, "reboot.start")
+ try:
+ self.run('(sleep 5; reboot) '
+ '</dev/null >/dev/null 2>&1 &')
+ except error.AutoservRunError:
+ self.__record("ABORT", None, "reboot.start",
+ "reboot command failed")
+ raise
+ if wait:
+ self.wait_for_restart(timeout)
+ self.reboot_followup()
+ self.__run_reboot_group(reboot)
+
+
+ def reboot_followup(self):
+ super(SSHHost, self).reboot_followup()
+ self.__load_netconsole_module() # if the builtin fails
+
+
+ def __copy_files(self, sources, dest):
+ """
+ Copy files from one machine to another.
+
+ This is for internal use by other methods that intend to move
+ files between machines. It expects a list of source files and
+ a destination (a filename if the source is a single file, a
+ destination otherwise). The names must already be
+ pre-processed into the appropriate rsync/scp friendly
+ format (%s@%s:%s).
+ """
+ # wait until there are only a small number of copies running
+ # before starting this one
+ get_config = global_config.global_config.get_config_value
+ max_simultaneous = get_config("HOSTS",
+ "max_simultaneous_file_copies",
+ type=int)
+ while True:
+ copy_count = 0
+ procs = utils.system_output('ps -ef')
+ for line in procs.splitlines():
+ if 'rsync ' in line or 'scp ' in line:
+ copy_count += 1
+ if copy_count < max_simultaneous:
+ break
+ time.sleep(60)
+
+ print '__copy_files: copying %s to %s' % (sources, dest)
+ try:
+ utils.run('rsync --rsh="%s" -az %s %s' % (
+ self.ssh_base_command(), ' '.join(sources), dest))
+ except Exception:
+ try:
+ utils.run('scp -rpq -P %d %s "%s"' % (
+ self.port, ' '.join(sources), dest))
+ except error.CmdError, cmderr:
+ raise error.AutoservRunError(cmderr.args[0],
+ cmderr.args[1])
+ def get_file(self, source, dest):
+ """
+ Copy files from the remote host to a local path.
- def run(self, command, timeout=3600, ignore_status=False,
- stdout_tee=None, stderr_tee=None, connect_timeout=30):
- """
- Run a command on the remote host.
-
- Args:
- command: the command line string
- timeout: time limit in seconds before attempting to
- kill the running process. The run() function
- will take a few seconds longer than 'timeout'
- to complete if it has to kill the process.
- ignore_status: do not raise an exception, no matter
- what the exit code of the command is.
-
- Returns:
- a hosts.base_classes.CmdResult object
-
- Raises:
- AutoservRunError: the exit code of the command
- execution was not 0
- AutoservSSHTimeout: ssh connection has timed out
- """
- stdout = stdout_tee or sys.stdout
- stderr = stderr_tee or sys.stdout
- print "ssh: %s" % command
- env = " ".join("=".join(pair) for pair in self.env.iteritems())
- try:
- try:
- return self._run(command, timeout,
- ignore_status, stdout,
- stderr, connect_timeout,
- env, '')
- except PermissionDeniedError:
- print("Permission denied to ssh; re-running"
- "with increased logging:")
- return self._run(command, timeout,
- ignore_status, stdout,
- stderr, connect_timeout,
- env, '-v -v -v')
- except error.CmdError, cmderr:
- # We get a CmdError here only if there is timeout of
- # that command. Catch that and stuff it into
- # AutoservRunError and raise it.
- raise error.AutoservRunError(cmderr.args[0],
- cmderr.args[1])
+ Directories will be copied recursively.
+ If a source component is a directory with a trailing slash,
+ the content of the directory will be copied, otherwise, the
+ directory itself and its content will be copied. This
+ behavior is similar to that of the program 'rsync'.
+ Args:
+ source: either
+ 1) a single file or directory, as a string
+ 2) a list of one or more (possibly mixed)
+ files or directories
+ dest: a file or a directory (if source contains a
+ directory or more than one element, you must
+ supply a directory dest)
- def run_short(self, command, **kwargs):
- """
- Calls the run() command with a short default timeout.
-
- Args:
- Takes the same arguments as does run(),
- with the exception of the timeout argument which
- here is fixed at 60 seconds.
- It returns the result of run.
- """
- return self.run(command, timeout=60, **kwargs)
+ Raises:
+ AutoservRunError: the scp command failed
+ """
+ if isinstance(source, types.StringTypes):
+ source= [source]
+ processed_source= []
+ for entry in source:
+ if entry.endswith('/'):
+ format_string= '%s@%s:"%s*"'
+ else:
+ format_string= '%s@%s:"%s"'
+ entry= format_string % (self.user, self.hostname,
+ utils.scp_remote_escape(entry))
+ processed_source.append(entry)
- def run_grep(self, command, timeout=30, ignore_status=False,
- stdout_ok_regexp=None, stdout_err_regexp=None,
- stderr_ok_regexp=None, stderr_err_regexp=None,
- connect_timeout=30):
- """
- Run a command on the remote host and look for regexp
- in stdout or stderr to determine if the command was
- successul or not.
+ processed_dest= os.path.abspath(dest)
+ if os.path.isdir(dest):
+ processed_dest= "%s/" % (utils.sh_escape(processed_dest),)
+ else:
+ processed_dest= utils.sh_escape(processed_dest)
- Args:
- command: the command line string
- timeout: time limit in seconds before attempting to
- kill the running process. The run() function
- will take a few seconds longer than 'timeout'
- to complete if it has to kill the process.
- ignore_status: do not raise an exception, no matter
- what the exit code of the command is.
- stdout_ok_regexp: regexp that should be in stdout
- if the command was successul.
- stdout_err_regexp: regexp that should be in stdout
- if the command failed.
- stderr_ok_regexp: regexp that should be in stderr
- if the command was successul.
- stderr_err_regexp: regexp that should be in stderr
- if the command failed.
+ self.__copy_files(processed_source, processed_dest)
- Returns:
- if the command was successul, raises an exception
- otherwise.
- Raises:
- AutoservRunError:
- - the exit code of the command execution was not 0.
- - If stderr_err_regexp is found in stderr,
- - If stdout_err_regexp is found in stdout,
- - If stderr_ok_regexp is not found in stderr.
- - If stdout_ok_regexp is not found in stdout,
- """
+ def send_file(self, source, dest):
+ """
+ Copy files from a local path to the remote host.
- # We ignore the status, because we will handle it at the end.
- result = self.run(command, timeout, ignore_status=True,
- connect_timeout=connect_timeout)
+ Directories will be copied recursively.
+ If a source component is a directory with a trailing slash,
+ the content of the directory will be copied, otherwise, the
+ directory itself and its content will be copied. This
+ behavior is similar to that of the program 'rsync'.
- # Look for the patterns, in order
- for (regexp, stream) in ((stderr_err_regexp, result.stderr),
- (stdout_err_regexp, result.stdout)):
- if regexp and stream:
- err_re = re.compile (regexp)
- if err_re.search(stream):
- raise error.AutoservRunError(
- '%s failed, found error pattern: '
- '"%s"' % (command, regexp), result)
+ Args:
+ source: either
+ 1) a single file or directory, as a string
+ 2) a list of one or more (possibly mixed)
+ files or directories
+ dest: a file or a directory (if source contains a
+ directory or more than one element, you must
+ supply a directory dest)
- for (regexp, stream) in ((stderr_ok_regexp, result.stderr),
- (stdout_ok_regexp, result.stdout)):
- if regexp and stream:
- ok_re = re.compile (regexp)
- if ok_re.search(stream):
- if ok_re.search(stream):
- return
+ Raises:
+ AutoservRunError: the scp command failed
+ """
+ if isinstance(source, types.StringTypes):
+ source= [source]
- if not ignore_status and result.exit_status > 0:
- raise error.AutoservRunError("command execution error",
- result)
+ processed_source= []
+ for entry in source:
+ if entry.endswith('/'):
+ format_string= '"%s/"*'
+ else:
+ format_string= '"%s"'
+ entry= format_string % (utils.sh_escape(os.path.abspath(entry)),)
+ processed_source.append(entry)
+ remote_dest = '%s@%s:"%s"' % (
+ self.user, self.hostname,
+ utils.scp_remote_escape(dest))
- def reboot(self, timeout=DEFAULT_REBOOT_TIMEOUT, label=None,
- kernel_args=None, wait=True):
- """
- Reboot the remote host.
-
- Args:
- timeout
- """
- self.reboot_setup()
+ self.__copy_files(processed_source, remote_dest)
+ self.run('find "%s" -type d | xargs -i -r chmod o+rx "{}"' % dest)
+ self.run('find "%s" -type f | xargs -i -r chmod o+r "{}"' % dest)
- # forcibly include the "netconsole" kernel arg
- if self.__netconsole_param:
- if kernel_args is None:
- kernel_args = self.__netconsole_param
- else:
- kernel_args += " " + self.__netconsole_param
- # unload the (possibly loaded) module to avoid shutdown issues
- self.__unload_netconsole_module()
- if label or kernel_args:
- self.bootloader.install_boottool()
- if label:
- self.bootloader.set_default(label)
- if kernel_args:
- if not label:
- default = int(self.bootloader.get_default())
- label = self.bootloader.get_titles()[default]
- self.bootloader.add_args(label, kernel_args)
+ def get_tmp_dir(self):
+ """
+ Return the pathname of a directory on the host suitable
+ for temporary file storage.
- # define a function for the reboot and run it in a group
- print "Reboot: initiating reboot"
- def reboot():
- self.__record("GOOD", None, "reboot.start")
- try:
- self.run('(sleep 5; reboot) '
- '</dev/null >/dev/null 2>&1 &')
- except error.AutoservRunError:
- self.__record("ABORT", None, "reboot.start",
- "reboot command failed")
- raise
- if wait:
- self.wait_for_restart(timeout)
- self.reboot_followup()
- self.__run_reboot_group(reboot)
+ The directory and its content will be deleted automatically
+ on the destruction of the Host object that was used to obtain
+ it.
+ """
+ dir_name= self.run("mktemp -d /tmp/autoserv-XXXXXX").stdout.rstrip(" \n")
+ self.tmp_dirs.append(dir_name)
+ return dir_name
- def reboot_followup(self):
- super(SSHHost, self).reboot_followup()
- self.__load_netconsole_module() # if the builtin fails
+ def is_up(self):
+ """
+ Check if the remote host is up.
+ Returns:
+ True if the remote host is up, False otherwise
+ """
+ try:
+ self.ssh_ping()
+ except:
+ return False
+ return True
- def __copy_files(self, sources, dest):
- """
- Copy files from one machine to another.
- This is for internal use by other methods that intend to move
- files between machines. It expects a list of source files and
- a destination (a filename if the source is a single file, a
- destination otherwise). The names must already be
- pre-processed into the appropriate rsync/scp friendly
- format (%s@%s:%s).
- """
- # wait until there are only a small number of copies running
- # before starting this one
- get_config = global_config.global_config.get_config_value
- max_simultaneous = get_config("HOSTS",
- "max_simultaneous_file_copies",
- type=int)
- while True:
- copy_count = 0
- procs = utils.system_output('ps -ef')
- for line in procs.splitlines():
- if 'rsync ' in line or 'scp ' in line:
- copy_count += 1
- if copy_count < max_simultaneous:
- break
- time.sleep(60)
+ def _is_wait_up_process_up(self):
+ """
+ Checks if any SSHHOST waitup processes are running yet on the
+ remote host.
- print '__copy_files: copying %s to %s' % (sources, dest)
- try:
- utils.run('rsync --rsh="%s" -az %s %s' % (
- self.ssh_base_command(), ' '.join(sources), dest))
- except Exception:
- try:
- utils.run('scp -rpq -P %d %s "%s"' % (
- self.port, ' '.join(sources), dest))
- except error.CmdError, cmderr:
- raise error.AutoservRunError(cmderr.args[0],
- cmderr.args[1])
+ Returns True if any the waitup processes are running, False
+ otherwise.
+ """
+ processes = self.get_wait_up_processes()
+ if len(processes) == 0:
+ return True # wait up processes aren't being used
+ for procname in processes:
+ exit_status = self.run("ps -e | grep '%s'" % procname,
+ ignore_status=True).exit_status
+ if exit_status == 0:
+ return True
+ return False
- def get_file(self, source, dest):
- """
- Copy files from the remote host to a local path.
-
- Directories will be copied recursively.
- If a source component is a directory with a trailing slash,
- the content of the directory will be copied, otherwise, the
- directory itself and its content will be copied. This
- behavior is similar to that of the program 'rsync'.
-
- Args:
- source: either
- 1) a single file or directory, as a string
- 2) a list of one or more (possibly mixed)
- files or directories
- dest: a file or a directory (if source contains a
- directory or more than one element, you must
- supply a directory dest)
-
- Raises:
- AutoservRunError: the scp command failed
- """
- if isinstance(source, types.StringTypes):
- source= [source]
-
- processed_source= []
- for entry in source:
- if entry.endswith('/'):
- format_string= '%s@%s:"%s*"'
- else:
- format_string= '%s@%s:"%s"'
- entry= format_string % (self.user, self.hostname,
- utils.scp_remote_escape(entry))
- processed_source.append(entry)
-
- processed_dest= os.path.abspath(dest)
- if os.path.isdir(dest):
- processed_dest= "%s/" % (utils.sh_escape(processed_dest),)
- else:
- processed_dest= utils.sh_escape(processed_dest)
- self.__copy_files(processed_source, processed_dest)
+ def wait_up(self, timeout=None):
+ """
+ Wait until the remote host is up or the timeout expires.
+ In fact, it will wait until an ssh connection to the remote
+ host can be established, and getty is running.
- def send_file(self, source, dest):
- """
- Copy files from a local path to the remote host.
-
- Directories will be copied recursively.
- If a source component is a directory with a trailing slash,
- the content of the directory will be copied, otherwise, the
- directory itself and its content will be copied. This
- behavior is similar to that of the program 'rsync'.
-
- Args:
- source: either
- 1) a single file or directory, as a string
- 2) a list of one or more (possibly mixed)
- files or directories
- dest: a file or a directory (if source contains a
- directory or more than one element, you must
- supply a directory dest)
-
- Raises:
- AutoservRunError: the scp command failed
- """
- if isinstance(source, types.StringTypes):
- source= [source]
-
- processed_source= []
- for entry in source:
- if entry.endswith('/'):
- format_string= '"%s/"*'
- else:
- format_string= '"%s"'
- entry= format_string % (utils.sh_escape(os.path.abspath(entry)),)
- processed_source.append(entry)
+ Args:
+ timeout: time limit in seconds before returning even
+ if the host is not up.
- remote_dest = '%s@%s:"%s"' % (
- self.user, self.hostname,
- utils.scp_remote_escape(dest))
+ Returns:
+ True if the host was found to be up, False otherwise
+ """
+ if timeout:
+ end_time= time.time() + timeout
- self.__copy_files(processed_source, remote_dest)
- self.run('find "%s" -type d | xargs -i -r chmod o+rx "{}"' % dest)
- self.run('find "%s" -type f | xargs -i -r chmod o+r "{}"' % dest)
+ while not timeout or time.time() < end_time:
+ try:
+ self.ssh_ping()
+ except (error.AutoservRunError,
+ error.AutoservSSHTimeout):
+ pass
+ else:
+ try:
+ if self._is_wait_up_process_up():
+ return True
+ except (error.AutoservRunError,
+ error.AutoservSSHTimeout):
+ pass
+ time.sleep(1)
- def get_tmp_dir(self):
- """
- Return the pathname of a directory on the host suitable
- for temporary file storage.
-
- The directory and its content will be deleted automatically
- on the destruction of the Host object that was used to obtain
- it.
- """
- dir_name= self.run("mktemp -d /tmp/autoserv-XXXXXX").stdout.rstrip(" \n")
- self.tmp_dirs.append(dir_name)
- return dir_name
+ return False
- def is_up(self):
- """
- Check if the remote host is up.
-
- Returns:
- True if the remote host is up, False otherwise
- """
- try:
- self.ssh_ping()
- except:
- return False
- return True
+ def wait_down(self, timeout=None):
+ """
+ Wait until the remote host is down or the timeout expires.
+ In fact, it will wait until an ssh connection to the remote
+ host fails.
- def _is_wait_up_process_up(self):
- """
- Checks if any SSHHOST waitup processes are running yet on the
- remote host.
+ Args:
+ timeout: time limit in seconds before returning even
+ if the host is not up.
- Returns True if any the waitup processes are running, False
- otherwise.
- """
- processes = self.get_wait_up_processes()
- if len(processes) == 0:
- return True # wait up processes aren't being used
- for procname in processes:
- exit_status = self.run("ps -e | grep '%s'" % procname,
- ignore_status=True).exit_status
- if exit_status == 0:
- return True
- return False
+ Returns:
+ True if the host was found to be down, False otherwise
+ """
+ if timeout:
+ end_time= time.time() + timeout
+ while not timeout or time.time() < end_time:
+ try:
+ self.ssh_ping()
+ except:
+ return True
+ time.sleep(1)
- def wait_up(self, timeout=None):
- """
- Wait until the remote host is up or the timeout expires.
-
- In fact, it will wait until an ssh connection to the remote
- host can be established, and getty is running.
-
- Args:
- timeout: time limit in seconds before returning even
- if the host is not up.
-
- Returns:
- True if the host was found to be up, False otherwise
- """
- if timeout:
- end_time= time.time() + timeout
-
- while not timeout or time.time() < end_time:
- try:
- self.ssh_ping()
- except (error.AutoservRunError,
- error.AutoservSSHTimeout):
- pass
- else:
- try:
- if self._is_wait_up_process_up():
- return True
- except (error.AutoservRunError,
- error.AutoservSSHTimeout):
- pass
- time.sleep(1)
-
- return False
+ return False
- def wait_down(self, timeout=None):
- """
- Wait until the remote host is down or the timeout expires.
-
- In fact, it will wait until an ssh connection to the remote
- host fails.
-
- Args:
- timeout: time limit in seconds before returning even
- if the host is not up.
-
- Returns:
- True if the host was found to be down, False otherwise
- """
- if timeout:
- end_time= time.time() + timeout
-
- while not timeout or time.time() < end_time:
- try:
- self.ssh_ping()
- except:
- return True
- time.sleep(1)
-
- return False
+ def ensure_up(self):
+ """
+ Ensure the host is up if it is not then do not proceed;
+ this prevents cacading failures of tests
+ """
+ print 'Ensuring that %s is up before continuing' % self.hostname
+ if hasattr(self, 'hardreset') and not self.wait_up(300):
+ print "Performing a hardreset on %s" % self.hostname
+ try:
+ self.hardreset()
+ except error.AutoservUnsupportedError:
+ print "Hardreset is unsupported on %s" % self.hostname
+ if not self.wait_up(60 * 30):
+ # 30 minutes should be more than enough
+ raise error.AutoservHostError
+ print 'Host up, continuing'
- def ensure_up(self):
- """
- Ensure the host is up if it is not then do not proceed;
- this prevents cacading failures of tests
- """
- print 'Ensuring that %s is up before continuing' % self.hostname
- if hasattr(self, 'hardreset') and not self.wait_up(300):
- print "Performing a hardreset on %s" % self.hostname
- try:
- self.hardreset()
- except error.AutoservUnsupportedError:
- print "Hardreset is unsupported on %s" % self.hostname
- if not self.wait_up(60 * 30):
- # 30 minutes should be more than enough
- raise error.AutoservHostError
- print 'Host up, continuing'
+ def get_num_cpu(self):
+ """
+ Get the number of CPUs in the host according to
+ /proc/cpuinfo.
+ Returns:
+ The number of CPUs
+ """
- def get_num_cpu(self):
- """
- Get the number of CPUs in the host according to
- /proc/cpuinfo.
-
- Returns:
- The number of CPUs
- """
-
- proc_cpuinfo = self.run("cat /proc/cpuinfo",
- stdout_tee=open('/dev/null', 'w')).stdout
- cpus = 0
- for line in proc_cpuinfo.splitlines():
- if line.startswith('processor'):
- cpus += 1
- return cpus
+ proc_cpuinfo = self.run("cat /proc/cpuinfo",
+ stdout_tee=open('/dev/null', 'w')).stdout
+ cpus = 0
+ for line in proc_cpuinfo.splitlines():
+ if line.startswith('processor'):
+ cpus += 1
+ return cpus
- def check_uptime(self):
- """
- Check that uptime is available and monotonically increasing.
- """
- if not self.ping():
- raise error.AutoservHostError('Client is not pingable')
- result = self.run("/bin/cat /proc/uptime", 30)
- return result.stdout.strip().split()[0]
+ def check_uptime(self):
+ """
+ Check that uptime is available and monotonically increasing.
+ """
+ if not self.ping():
+ raise error.AutoservHostError('Client is not pingable')
+ result = self.run("/bin/cat /proc/uptime", 30)
+ return result.stdout.strip().split()[0]
- def get_arch(self):
- """
- Get the hardware architecture of the remote machine
- """
- arch = self.run('/bin/uname -m').stdout.rstrip()
- if re.match(r'i\d86$', arch):
- arch = 'i386'
- return arch
+ def get_arch(self):
+ """
+ Get the hardware architecture of the remote machine
+ """
+ arch = self.run('/bin/uname -m').stdout.rstrip()
+ if re.match(r'i\d86$', arch):
+ arch = 'i386'
+ return arch
- def get_kernel_ver(self):
- """
- Get the kernel version of the remote machine
- """
- return self.run('/bin/uname -r').stdout.rstrip()
+ def get_kernel_ver(self):
+ """
+ Get the kernel version of the remote machine
+ """
+ return self.run('/bin/uname -r').stdout.rstrip()
- def get_cmdline(self):
- """
- Get the kernel command line of the remote machine
- """
- return self.run('cat /proc/cmdline').stdout.rstrip()
+ def get_cmdline(self):
+ """
+ Get the kernel command line of the remote machine
+ """
+ return self.run('cat /proc/cmdline').stdout.rstrip()
- def ping(self):
- """
- Ping the remote system, and return whether it's available
- """
- fpingcmd = "%s -q %s" % ('/usr/bin/fping', self.hostname)
- rc = utils.system(fpingcmd, ignore_status = 1)
- return (rc == 0)
+ def ping(self):
+ """
+ Ping the remote system, and return whether it's available
+ """
+ fpingcmd = "%s -q %s" % ('/usr/bin/fping', self.hostname)
+ rc = utils.system(fpingcmd, ignore_status = 1)
+ return (rc == 0)
- def ssh_ping(self, timeout = 60):
- try:
- self.run('true', timeout = timeout, connect_timeout = timeout)
- except error.AutoservSSHTimeout:
- msg = "ssh ping timed out. timeout = %s" % timeout
- raise error.AutoservSSHTimeout(msg)
- except error.AutoservRunError, exc:
- msg = "command true failed in ssh ping"
- raise error.AutoservRunError(msg, exc.args[1])
+ def ssh_ping(self, timeout = 60):
+ try:
+ self.run('true', timeout = timeout, connect_timeout = timeout)
+ except error.AutoservSSHTimeout:
+ msg = "ssh ping timed out. timeout = %s" % timeout
+ raise error.AutoservSSHTimeout(msg)
+ except error.AutoservRunError, exc:
+ msg = "command true failed in ssh ping"
+ raise error.AutoservRunError(msg, exc.args[1])
- def get_autodir(self):
- return self.autodir
+ def get_autodir(self):
+ return self.autodir
- def ssh_setup_key(self):
- try:
- print 'Performing ssh key setup on %s:%d as %s' % \
- (self.hostname, self.port, self.user)
+ def ssh_setup_key(self):
+ try:
+ print 'Performing ssh key setup on %s:%d as %s' % \
+ (self.hostname, self.port, self.user)
- host = pxssh.pxssh()
- host.login(self.hostname, self.user, self.password,
- port=self.port)
+ host = pxssh.pxssh()
+ host.login(self.hostname, self.user, self.password,
+ port=self.port)
- try:
- public_key = utils.get_public_key()
+ try:
+ public_key = utils.get_public_key()
- host.sendline('mkdir -p ~/.ssh')
- host.prompt()
- host.sendline('chmod 700 ~/.ssh')
- host.prompt()
- host.sendline("echo '%s' >> ~/.ssh/authorized_keys; " %
- (public_key))
- host.prompt()
- host.sendline('chmod 600 ~/.ssh/authorized_keys')
- host.prompt()
+ host.sendline('mkdir -p ~/.ssh')
+ host.prompt()
+ host.sendline('chmod 700 ~/.ssh')
+ host.prompt()
+ host.sendline("echo '%s' >> ~/.ssh/authorized_keys; " %
+ (public_key))
+ host.prompt()
+ host.sendline('chmod 600 ~/.ssh/authorized_keys')
+ host.prompt()
- print 'SSH key setup complete'
+ print 'SSH key setup complete'
- finally:
- host.logout()
+ finally:
+ host.logout()
- except:
- pass
+ except:
+ pass
- def setup(self):
- if not self.password == '':
- try:
- self.ssh_ping()
- except error.AutoservRunError:
- self.ssh_setup_key()
+ def setup(self):
+ if not self.password == '':
+ try:
+ self.ssh_ping()
+ except error.AutoservRunError:
+ self.ssh_setup_key()
diff --git a/server/hypervisor.py b/server/hypervisor.py
index ac213cc..75ad321 100644
--- a/server/hypervisor.py
+++ b/server/hypervisor.py
@@ -5,7 +5,7 @@
"""
This module defines the Hypervisor class
- Hypervisor: a virtual machine monitor
+ Hypervisor: a virtual machine monitor
"""
__author__ = """
@@ -19,32 +19,32 @@
class Hypervisor(installable_object.InstallableObject):
- """
- This class represents a virtual machine monitor.
+ """
+ This class represents a virtual machine monitor.
- Implementation details:
- This is an abstract class, leaf subclasses must implement the methods
- listed here and in parent classes which have no implementation. They
- may reimplement methods which already have an implementation. You
- must not instantiate this class but should instantiate one of those
- leaf subclasses.
- """
+ Implementation details:
+ This is an abstract class, leaf subclasses must implement the methods
+ listed here and in parent classes which have no implementation. They
+ may reimplement methods which already have an implementation. You
+ must not instantiate this class but should instantiate one of those
+ leaf subclasses.
+ """
- host = None
- guests = None
+ host = None
+ guests = None
- def __init__(self, host):
- super(Hypervisor, self).__init__()
- self.host= host
+ def __init__(self, host):
+ super(Hypervisor, self).__init__()
+ self.host= host
- def new_guest(self):
- pass
+ def new_guest(self):
+ pass
- def delete_guest(self, guest_hostname):
- pass
+ def delete_guest(self, guest_hostname):
+ pass
- def reset_guest(self, guest_hostname):
- pass
+ def reset_guest(self, guest_hostname):
+ pass
diff --git a/server/installable_object.py b/server/installable_object.py
index 0492595..3c61e05 100644
--- a/server/installable_object.py
+++ b/server/installable_object.py
@@ -5,7 +5,7 @@
"""
This module defines the InstallableObject class
- InstallableObject: a software package that can be installed on a Host
+ InstallableObject: a software package that can be installed on a Host
"""
__author__ = """
@@ -19,37 +19,37 @@
class InstallableObject(object):
- """
- This class represents a software package that can be installed on
- a Host.
-
- Implementation details:
- This is an abstract class, leaf subclasses must implement the methods
- listed here. You must not instantiate this class but should
- instantiate one of those leaf subclasses.
- """
-
- source_material= None
-
- def __init__(self):
- super(InstallableObject, self).__init__()
+ """
+ This class represents a software package that can be installed on
+ a Host.
-
- def get(self, location):
- """
- Get the source material required to install the object.
-
- Through the utils.get() function, the argument passed will be
- saved in a temporary location on the LocalHost. That location
- is saved in the source_material attribute.
-
- Args:
- location: the path to the source material. This path
- may be of any type that the utils.get()
- function will accept.
- """
- self.source_material= utils.get(location)
+ Implementation details:
+ This is an abstract class, leaf subclasses must implement the methods
+ listed here. You must not instantiate this class but should
+ instantiate one of those leaf subclasses.
+ """
-
- def install(self, host):
- pass
+ source_material= None
+
+ def __init__(self):
+ super(InstallableObject, self).__init__()
+
+
+ def get(self, location):
+ """
+ Get the source material required to install the object.
+
+ Through the utils.get() function, the argument passed will be
+ saved in a temporary location on the LocalHost. That location
+ is saved in the source_material attribute.
+
+ Args:
+ location: the path to the source material. This path
+ may be of any type that the utils.get()
+ function will accept.
+ """
+ self.source_material= utils.get(location)
+
+
+ def install(self, host):
+ pass
diff --git a/server/kernel.py b/server/kernel.py
index 63743cd..5fcaba7 100644
--- a/server/kernel.py
+++ b/server/kernel.py
@@ -5,7 +5,7 @@
"""
This module defines the Kernel class
- Kernel: an os kernel
+ Kernel: an os kernel
"""
__author__ = """
@@ -19,27 +19,27 @@
class Kernel(installable_object.InstallableObject):
- """
- This class represents a kernel.
-
- It is used to obtain a built kernel or create one from source and
- install it on a Host.
-
- Implementation details:
- This is an abstract class, leaf subclasses must implement the methods
- listed here and in parent classes which have no implementation. They
- may reimplement methods which already have an implementation. You
- must not instantiate this class but should instantiate one of those
- leaf subclasses.
- """
-
- def get_version():
- pass
+ """
+ This class represents a kernel.
+
+ It is used to obtain a built kernel or create one from source and
+ install it on a Host.
+
+ Implementation details:
+ This is an abstract class, leaf subclasses must implement the methods
+ listed here and in parent classes which have no implementation. They
+ may reimplement methods which already have an implementation. You
+ must not instantiate this class but should instantiate one of those
+ leaf subclasses.
+ """
+
+ def get_version():
+ pass
- def get_image_name():
- pass
+ def get_image_name():
+ pass
- def get_initrd_name():
- pass
+ def get_initrd_name():
+ pass
diff --git a/server/kvm.py b/server/kvm.py
index 72e3e7d..0e57731 100644
--- a/server/kvm.py
+++ b/server/kvm.py
@@ -5,7 +5,7 @@
"""
This module defines the KVM class
- KVM: a KVM virtual machine monitor
+ KVM: a KVM virtual machine monitor
"""
__author__ = """
@@ -31,15 +31,15 @@
_check_process_script= """\
if [ -f "%(pid_file_name)s" ]
then
- pid=$(cat "%(pid_file_name)s")
- if [ -L /proc/$pid/exe ] && stat /proc/$pid/exe |
- grep -q -- "-> \`%(qemu_binary)s\'\$"
- then
- echo "process present"
- else
- rm -f "%(pid_file_name)s"
- rm -f "%(monitor_file_name)s"
- fi
+ pid=$(cat "%(pid_file_name)s")
+ if [ -L /proc/$pid/exe ] && stat /proc/$pid/exe |
+ grep -q -- "-> \`%(qemu_binary)s\'\$"
+ then
+ echo "process present"
+ else
+ rm -f "%(pid_file_name)s"
+ rm -f "%(monitor_file_name)s"
+ fi
fi
"""
@@ -54,430 +54,430 @@
_remove_modules_script= """\
if $(grep -q "^kvm_intel [[:digit:]]\+ 0" /proc/modules)
then
- rmmod kvm-intel
+ rmmod kvm-intel
fi
if $(grep -q "^kvm_amd [[:digit:]]\+ 0" /proc/modules)
then
- rmmod kvm-amd
+ rmmod kvm-amd
fi
if $(grep -q "^kvm [[:digit:]]\+ 0" /proc/modules)
then
- rmmod kvm
+ rmmod kvm
fi
"""
class KVM(hypervisor.Hypervisor):
- """
- This class represents a KVM virtual machine monitor.
+ """
+ This class represents a KVM virtual machine monitor.
- Implementation details:
- This is a leaf class in an abstract class hierarchy, it must
- implement the unimplemented methods in parent classes.
- """
+ Implementation details:
+ This is a leaf class in an abstract class hierarchy, it must
+ implement the unimplemented methods in parent classes.
+ """
- build_dir= None
- pid_dir= None
- support_dir= None
- addresses= []
- insert_modules= True
- modules= {}
+ build_dir= None
+ pid_dir= None
+ support_dir= None
+ addresses= []
+ insert_modules= True
+ modules= {}
- def __del__(self):
- """
- Destroy a KVM object.
+ def __del__(self):
+ """
+ Destroy a KVM object.
- Guests managed by this hypervisor that are still running will
- be killed.
- """
- self.deinitialize()
+ Guests managed by this hypervisor that are still running will
+ be killed.
+ """
+ self.deinitialize()
- def _insert_modules(self):
- """
- Insert the kvm modules into the kernel.
+ def _insert_modules(self):
+ """
+ Insert the kvm modules into the kernel.
- The modules inserted are the ones from the build directory, NOT
- the ones from the kernel.
+ The modules inserted are the ones from the build directory, NOT
+ the ones from the kernel.
- This function should only be called after install(). It will
- check that the modules are not already loaded before attempting
- to insert them.
- """
- cpu_flags= self.host.run('cat /proc/cpuinfo | '
- 'grep -e "^flags" | head -1 | cut -d " " -f 2-'
- ).stdout.strip()
+ This function should only be called after install(). It will
+ check that the modules are not already loaded before attempting
+ to insert them.
+ """
+ cpu_flags= self.host.run('cat /proc/cpuinfo | '
+ 'grep -e "^flags" | head -1 | cut -d " " -f 2-'
+ ).stdout.strip()
- if cpu_flags.find('vmx') != -1:
- module_type= "intel"
- elif cpu_flags.find('svm') != -1:
- module_type= "amd"
- else:
- raise error.AutoservVirtError("No harware "
- "virtualization extensions found, "
- "KVM cannot run")
+ if cpu_flags.find('vmx') != -1:
+ module_type= "intel"
+ elif cpu_flags.find('svm') != -1:
+ module_type= "amd"
+ else:
+ raise error.AutoservVirtError("No harware "
+ "virtualization extensions found, "
+ "KVM cannot run")
- self.host.run('if ! $(grep -q "^kvm " /proc/modules); '
- 'then insmod "%s"; fi' % (utils.sh_escape(
- os.path.join(self.build_dir, "kernel/kvm.ko")),))
- if module_type == "intel":
- self.host.run('if ! $(grep -q "^kvm_intel " '
- '/proc/modules); then insmod "%s"; fi' %
- (utils.sh_escape(os.path.join(self.build_dir,
- "kernel/kvm-intel.ko")),))
- elif module_type == "amd":
- self.host.run('if ! $(grep -q "^kvm_amd " '
- '/proc/modules); then insmod "%s"; fi' %
- (utils.sh_escape(os.path.join(self.build_dir,
- "kernel/kvm-amd.ko")),))
+ self.host.run('if ! $(grep -q "^kvm " /proc/modules); '
+ 'then insmod "%s"; fi' % (utils.sh_escape(
+ os.path.join(self.build_dir, "kernel/kvm.ko")),))
+ if module_type == "intel":
+ self.host.run('if ! $(grep -q "^kvm_intel " '
+ '/proc/modules); then insmod "%s"; fi' %
+ (utils.sh_escape(os.path.join(self.build_dir,
+ "kernel/kvm-intel.ko")),))
+ elif module_type == "amd":
+ self.host.run('if ! $(grep -q "^kvm_amd " '
+ '/proc/modules); then insmod "%s"; fi' %
+ (utils.sh_escape(os.path.join(self.build_dir,
+ "kernel/kvm-amd.ko")),))
- def _remove_modules(self):
- """
- Remove the kvm modules from the kernel.
+ def _remove_modules(self):
+ """
+ Remove the kvm modules from the kernel.
- This function checks that they're not in use before trying to
- remove them.
- """
- self.host.run(_remove_modules_script)
+ This function checks that they're not in use before trying to
+ remove them.
+ """
+ self.host.run(_remove_modules_script)
- def install(self, addresses, build=True, insert_modules=True, syncdir=None):
- """
- Compile the kvm software on the host that the object was
- initialized with.
+ def install(self, addresses, build=True, insert_modules=True, syncdir=None):
+ """
+ Compile the kvm software on the host that the object was
+ initialized with.
- The kvm kernel modules are compiled, for this, the kernel
- sources must be available. A custom qemu is also compiled.
- Note that 'make install' is not run, the kernel modules and
- qemu are run from where they were built, therefore not
- conflicting with what might already be installed.
+ The kvm kernel modules are compiled, for this, the kernel
+ sources must be available. A custom qemu is also compiled.
+ Note that 'make install' is not run, the kernel modules and
+ qemu are run from where they were built, therefore not
+ conflicting with what might already be installed.
- Args:
- addresses: a list of dict entries of the form
- {"mac" : "xx:xx:xx:xx:xx:xx",
- "ip" : "yyy.yyy.yyy.yyy"} where x and y
- are replaced with sensible values. The ip
- address may be a hostname or an IPv6 instead.
+ Args:
+ addresses: a list of dict entries of the form
+ {"mac" : "xx:xx:xx:xx:xx:xx",
+ "ip" : "yyy.yyy.yyy.yyy"} where x and y
+ are replaced with sensible values. The ip
+ address may be a hostname or an IPv6 instead.
- When a new virtual machine is created, the
- first available entry in that list will be
- used. The network card in the virtual machine
- will be assigned the specified mac address and
- autoserv will use the specified ip address to
- connect to the virtual host via ssh. The virtual
- machine os must therefore be configured to
- configure its network with the ip corresponding
- to the mac.
- build: build kvm from the source material, if False,
- it is assumed that the package contains the
- source tree after a 'make'.
- insert_modules: build kvm modules from the source
- material and insert them. Otherwise, the
- running kernel is assumed to already have
- kvm support and nothing will be done concerning
- the modules.
+ When a new virtual machine is created, the
+ first available entry in that list will be
+ used. The network card in the virtual machine
+ will be assigned the specified mac address and
+ autoserv will use the specified ip address to
+ connect to the virtual host via ssh. The virtual
+ machine os must therefore be configured to
+ configure its network with the ip corresponding
+ to the mac.
+ build: build kvm from the source material, if False,
+ it is assumed that the package contains the
+ source tree after a 'make'.
+ insert_modules: build kvm modules from the source
+ material and insert them. Otherwise, the
+ running kernel is assumed to already have
+ kvm support and nothing will be done concerning
+ the modules.
- TODO(poirier): check dependencies before building
- kvm needs:
- libasound2-dev
- libsdl1.2-dev (or configure qemu with --disable-gfx-check, how?)
- bridge-utils
- """
- self.addresses= [
- {"mac" : address["mac"],
- "ip" : address["ip"],
- "is_used" : False} for address in addresses]
+ TODO(poirier): check dependencies before building
+ kvm needs:
+ libasound2-dev
+ libsdl1.2-dev (or configure qemu with --disable-gfx-check, how?)
+ bridge-utils
+ """
+ self.addresses= [
+ {"mac" : address["mac"],
+ "ip" : address["ip"],
+ "is_used" : False} for address in addresses]
- self.build_dir = self.host.get_tmp_dir()
- self.support_dir= self.host.get_tmp_dir()
+ self.build_dir = self.host.get_tmp_dir()
+ self.support_dir= self.host.get_tmp_dir()
- self.host.run('echo "%s" > "%s"' % (
- utils.sh_escape(_qemu_ifup_script),
- utils.sh_escape(os.path.join(self.support_dir,
- "qemu-ifup.sh")),))
- self.host.run('chmod a+x "%s"' % (
- utils.sh_escape(os.path.join(self.support_dir,
- "qemu-ifup.sh")),))
+ self.host.run('echo "%s" > "%s"' % (
+ utils.sh_escape(_qemu_ifup_script),
+ utils.sh_escape(os.path.join(self.support_dir,
+ "qemu-ifup.sh")),))
+ self.host.run('chmod a+x "%s"' % (
+ utils.sh_escape(os.path.join(self.support_dir,
+ "qemu-ifup.sh")),))
- self.host.send_file(self.source_material, self.build_dir)
- remote_source_material= os.path.join(self.build_dir,
- os.path.basename(self.source_material))
+ self.host.send_file(self.source_material, self.build_dir)
+ remote_source_material= os.path.join(self.build_dir,
+ os.path.basename(self.source_material))
- self.build_dir= utils.unarchive(self.host,
- remote_source_material)
+ self.build_dir= utils.unarchive(self.host,
+ remote_source_material)
- if insert_modules:
- configure_modules= ""
- self.insert_modules= True
- else:
- configure_modules= "--with-patched-kernel "
- self.insert_modules= False
+ if insert_modules:
+ configure_modules= ""
+ self.insert_modules= True
+ else:
+ configure_modules= "--with-patched-kernel "
+ self.insert_modules= False
- # build
- if build:
- try:
- self.host.run('make -C "%s" clean' % (
- utils.sh_escape(self.build_dir),),
- timeout=600)
- except error.AutoservRunError:
- # directory was already clean and contained
- # no makefile
- pass
- self.host.run('cd "%s" && ./configure %s' % (
- utils.sh_escape(self.build_dir),
- configure_modules,), timeout=600)
- if syncdir:
- cmd = 'cd "%s/kernel" && make sync LINUX=%s' % (
- utils.sh_escape(self.build_dir),
- utils.sh_escape(syncdir))
- self.host.run(cmd)
- self.host.run('make -j%d -C "%s"' % (
- self.host.get_num_cpu() * 2,
- utils.sh_escape(self.build_dir),), timeout=3600)
- # remember path to modules
- self.modules['kvm'] = "%s" %(
- utils.sh_escape(os.path.join(self.build_dir,
- "kernel/kvm.ko")))
- self.modules['kvm-intel'] = "%s" %(
- utils.sh_escape(os.path.join(self.build_dir,
- "kernel/kvm-intel.ko")))
- self.modules['kvm-amd'] = "%s" %(
- utils.sh_escape(os.path.join(self.build_dir,
- "kernel/kvm-amd.ko")))
- print self.modules
+ # build
+ if build:
+ try:
+ self.host.run('make -C "%s" clean' % (
+ utils.sh_escape(self.build_dir),),
+ timeout=600)
+ except error.AutoservRunError:
+ # directory was already clean and contained
+ # no makefile
+ pass
+ self.host.run('cd "%s" && ./configure %s' % (
+ utils.sh_escape(self.build_dir),
+ configure_modules,), timeout=600)
+ if syncdir:
+ cmd = 'cd "%s/kernel" && make sync LINUX=%s' % (
+ utils.sh_escape(self.build_dir),
+ utils.sh_escape(syncdir))
+ self.host.run(cmd)
+ self.host.run('make -j%d -C "%s"' % (
+ self.host.get_num_cpu() * 2,
+ utils.sh_escape(self.build_dir),), timeout=3600)
+ # remember path to modules
+ self.modules['kvm'] = "%s" %(
+ utils.sh_escape(os.path.join(self.build_dir,
+ "kernel/kvm.ko")))
+ self.modules['kvm-intel'] = "%s" %(
+ utils.sh_escape(os.path.join(self.build_dir,
+ "kernel/kvm-intel.ko")))
+ self.modules['kvm-amd'] = "%s" %(
+ utils.sh_escape(os.path.join(self.build_dir,
+ "kernel/kvm-amd.ko")))
+ print self.modules
- self.initialize()
+ self.initialize()
- def initialize(self):
- """
- Initialize the hypervisor.
+ def initialize(self):
+ """
+ Initialize the hypervisor.
- Loads needed kernel modules and creates temporary directories.
- The logic is that you could compile once and
- initialize - deinitialize many times. But why you would do that
- has yet to be figured.
+ Loads needed kernel modules and creates temporary directories.
+ The logic is that you could compile once and
+ initialize - deinitialize many times. But why you would do that
+ has yet to be figured.
- Raises:
- AutoservVirtError: cpuid doesn't report virtualization
- extentions (vmx for intel or svm for amd), in
- this case, kvm cannot run.
- """
- self.pid_dir= self.host.get_tmp_dir()
+ Raises:
+ AutoservVirtError: cpuid doesn't report virtualization
+ extentions (vmx for intel or svm for amd), in
+ this case, kvm cannot run.
+ """
+ self.pid_dir= self.host.get_tmp_dir()
- if self.insert_modules:
- self._remove_modules()
- self._insert_modules()
+ if self.insert_modules:
+ self._remove_modules()
+ self._insert_modules()
- def deinitialize(self):
- """
- Terminate the hypervisor.
+ def deinitialize(self):
+ """
+ Terminate the hypervisor.
- Kill all the virtual machines that are still running and
- unload the kernel modules.
- """
- self.refresh_guests()
- for address in self.addresses:
- if address["is_used"]:
- self.delete_guest(address["ip"])
- self.pid_dir= None
+ Kill all the virtual machines that are still running and
+ unload the kernel modules.
+ """
+ self.refresh_guests()
+ for address in self.addresses:
+ if address["is_used"]:
+ self.delete_guest(address["ip"])
+ self.pid_dir= None
- if self.insert_modules:
- self._remove_modules()
+ if self.insert_modules:
+ self._remove_modules()
- def new_guest(self, qemu_options):
- """
- Start a new guest ("virtual machine").
+ def new_guest(self, qemu_options):
+ """
+ Start a new guest ("virtual machine").
- Returns:
- The ip that was picked from the list supplied to
- install() and assigned to this guest.
+ Returns:
+ The ip that was picked from the list supplied to
+ install() and assigned to this guest.
- Raises:
- AutoservVirtError: no more addresses are available.
- """
- for address in self.addresses:
- if not address["is_used"]:
- break
- else:
- raise error.AutoservVirtError(
- "No more addresses available")
+ Raises:
+ AutoservVirtError: no more addresses are available.
+ """
+ for address in self.addresses:
+ if not address["is_used"]:
+ break
+ else:
+ raise error.AutoservVirtError(
+ "No more addresses available")
- retval= self.host.run(
- '%s'
- # this is the line of options that can be modified
- ' %s '
- '-pidfile "%s" -daemonize -nographic '
- #~ '-serial telnet::4444,server '
- '-monitor unix:"%s",server,nowait '
- '-net nic,macaddr="%s" -net tap,script="%s" -L "%s"' % (
- utils.sh_escape(os.path.join(
- self.build_dir,
- "qemu/x86_64-softmmu/qemu-system-x86_64")),
- qemu_options,
- utils.sh_escape(os.path.join(
- self.pid_dir,
- "vhost%s_pid" % (address["ip"],))),
- utils.sh_escape(os.path.join(
- self.pid_dir,
- "vhost%s_monitor" % (address["ip"],))),
- utils.sh_escape(address["mac"]),
- utils.sh_escape(os.path.join(
- self.support_dir,
- "qemu-ifup.sh")),
- utils.sh_escape(os.path.join(
- self.build_dir,
- "qemu/pc-bios")),))
+ retval= self.host.run(
+ '%s'
+ # this is the line of options that can be modified
+ ' %s '
+ '-pidfile "%s" -daemonize -nographic '
+ #~ '-serial telnet::4444,server '
+ '-monitor unix:"%s",server,nowait '
+ '-net nic,macaddr="%s" -net tap,script="%s" -L "%s"' % (
+ utils.sh_escape(os.path.join(
+ self.build_dir,
+ "qemu/x86_64-softmmu/qemu-system-x86_64")),
+ qemu_options,
+ utils.sh_escape(os.path.join(
+ self.pid_dir,
+ "vhost%s_pid" % (address["ip"],))),
+ utils.sh_escape(os.path.join(
+ self.pid_dir,
+ "vhost%s_monitor" % (address["ip"],))),
+ utils.sh_escape(address["mac"]),
+ utils.sh_escape(os.path.join(
+ self.support_dir,
+ "qemu-ifup.sh")),
+ utils.sh_escape(os.path.join(
+ self.build_dir,
+ "qemu/pc-bios")),))
- address["is_used"]= True
- return address["ip"]
+ address["is_used"]= True
+ return address["ip"]
- def refresh_guests(self):
- """
- Refresh the list of guests addresses.
+ def refresh_guests(self):
+ """
+ Refresh the list of guests addresses.
- The is_used status will be updated according to the presence
- of the process specified in the pid file that was written when
- the virtual machine was started.
+ The is_used status will be updated according to the presence
+ of the process specified in the pid file that was written when
+ the virtual machine was started.
- TODO(poirier): there are a lot of race conditions in this code
- because the process might terminate on its own anywhere in
- between
- """
- for address in self.addresses:
- if address["is_used"]:
- pid_file_name= utils.sh_escape(os.path.join(
- self.pid_dir,
- "vhost%s_pid" % (address["ip"],)))
- monitor_file_name= utils.sh_escape(os.path.join(
- self.pid_dir,
- "vhost%s_monitor" % (address["ip"],)))
- retval= self.host.run(
- _check_process_script % {
- "pid_file_name" : pid_file_name,
- "monitor_file_name" : monitor_file_name,
- "qemu_binary" : utils.sh_escape(
- os.path.join(self.build_dir,
- "qemu/x86_64-softmmu/"
- "qemu-system-x86_64")),})
- if (retval.stdout.strip() !=
- "process present"):
- address["is_used"]= False
+ TODO(poirier): there are a lot of race conditions in this code
+ because the process might terminate on its own anywhere in
+ between
+ """
+ for address in self.addresses:
+ if address["is_used"]:
+ pid_file_name= utils.sh_escape(os.path.join(
+ self.pid_dir,
+ "vhost%s_pid" % (address["ip"],)))
+ monitor_file_name= utils.sh_escape(os.path.join(
+ self.pid_dir,
+ "vhost%s_monitor" % (address["ip"],)))
+ retval= self.host.run(
+ _check_process_script % {
+ "pid_file_name" : pid_file_name,
+ "monitor_file_name" : monitor_file_name,
+ "qemu_binary" : utils.sh_escape(
+ os.path.join(self.build_dir,
+ "qemu/x86_64-softmmu/"
+ "qemu-system-x86_64")),})
+ if (retval.stdout.strip() !=
+ "process present"):
+ address["is_used"]= False
- def delete_guest(self, guest_hostname):
- """
- Terminate a virtual machine.
+ def delete_guest(self, guest_hostname):
+ """
+ Terminate a virtual machine.
- Args:
- guest_hostname: the ip (as it was specified in the
- address list given to install()) of the guest
- to terminate.
+ Args:
+ guest_hostname: the ip (as it was specified in the
+ address list given to install()) of the guest
+ to terminate.
- Raises:
- AutoservVirtError: the guest_hostname argument is
- invalid
+ Raises:
+ AutoservVirtError: the guest_hostname argument is
+ invalid
- TODO(poirier): is there a difference in qemu between
- sending SIGTEM or quitting from the monitor?
- TODO(poirier): there are a lot of race conditions in this code
- because the process might terminate on its own anywhere in
- between
- """
- for address in self.addresses:
- if address["ip"] == guest_hostname:
- if address["is_used"]:
- break
- else:
- # Will happen if deinitialize() is
- # called while guest objects still
- # exit and these are del'ed after.
- # In that situation, nothing is to
- # be done here, don't throw an error
- # either because it will print an
- # ugly message during garbage
- # collection. The solution would be to
- # delete the guest objects before
- # calling deinitialize(), this can't be
- # done by the KVM class, it has no
- # reference to those objects and it
- # cannot have any either. The Guest
- # objects already need to have a
- # reference to their managing
- # hypervisor. If the hypervisor had a
- # reference to the Guest objects it
- # manages, it would create a circular
- # reference and those objects would
- # not be elligible for garbage
- # collection. In turn, this means that
- # the KVM object would not be
- # automatically del'ed at the end of
- # the program and guests that are still
- # running would be left unattended.
- # Note that this circular reference
- # problem could be avoided by using
- # weakref's in class KVM but the
- # control file will most likely also
- # have references to the guests.
- return
- else:
- raise error.AutoservVirtError("Unknown guest hostname")
+ TODO(poirier): is there a difference in qemu between
+ sending SIGTEM or quitting from the monitor?
+ TODO(poirier): there are a lot of race conditions in this code
+ because the process might terminate on its own anywhere in
+ between
+ """
+ for address in self.addresses:
+ if address["ip"] == guest_hostname:
+ if address["is_used"]:
+ break
+ else:
+ # Will happen if deinitialize() is
+ # called while guest objects still
+ # exit and these are del'ed after.
+ # In that situation, nothing is to
+ # be done here, don't throw an error
+ # either because it will print an
+ # ugly message during garbage
+ # collection. The solution would be to
+ # delete the guest objects before
+ # calling deinitialize(), this can't be
+ # done by the KVM class, it has no
+ # reference to those objects and it
+ # cannot have any either. The Guest
+ # objects already need to have a
+ # reference to their managing
+ # hypervisor. If the hypervisor had a
+ # reference to the Guest objects it
+ # manages, it would create a circular
+ # reference and those objects would
+ # not be elligible for garbage
+ # collection. In turn, this means that
+ # the KVM object would not be
+ # automatically del'ed at the end of
+ # the program and guests that are still
+ # running would be left unattended.
+ # Note that this circular reference
+ # problem could be avoided by using
+ # weakref's in class KVM but the
+ # control file will most likely also
+ # have references to the guests.
+ return
+ else:
+ raise error.AutoservVirtError("Unknown guest hostname")
- pid_file_name= utils.sh_escape(os.path.join(self.pid_dir,
- "vhost%s_pid" % (address["ip"],)))
- monitor_file_name= utils.sh_escape(os.path.join(self.pid_dir,
- "vhost%s_monitor" % (address["ip"],)))
+ pid_file_name= utils.sh_escape(os.path.join(self.pid_dir,
+ "vhost%s_pid" % (address["ip"],)))
+ monitor_file_name= utils.sh_escape(os.path.join(self.pid_dir,
+ "vhost%s_monitor" % (address["ip"],)))
- retval= self.host.run(
- _check_process_script % {
- "pid_file_name" : pid_file_name,
- "monitor_file_name" : monitor_file_name,
- "qemu_binary" : utils.sh_escape(os.path.join(
- self.build_dir,
- "qemu/x86_64-softmmu/qemu-system-x86_64")),})
- if retval.stdout.strip() == "process present":
- self.host.run('kill $(cat "%s")' %(
- pid_file_name,))
- self.host.run('rm -f "%s"' %(
- pid_file_name,))
- self.host.run('rm -f "%s"' %(
- monitor_file_name,))
- address["is_used"]= False
+ retval= self.host.run(
+ _check_process_script % {
+ "pid_file_name" : pid_file_name,
+ "monitor_file_name" : monitor_file_name,
+ "qemu_binary" : utils.sh_escape(os.path.join(
+ self.build_dir,
+ "qemu/x86_64-softmmu/qemu-system-x86_64")),})
+ if retval.stdout.strip() == "process present":
+ self.host.run('kill $(cat "%s")' %(
+ pid_file_name,))
+ self.host.run('rm -f "%s"' %(
+ pid_file_name,))
+ self.host.run('rm -f "%s"' %(
+ monitor_file_name,))
+ address["is_used"]= False
- def reset_guest(self, guest_hostname):
- """
- Perform a hard reset on a virtual machine.
+ def reset_guest(self, guest_hostname):
+ """
+ Perform a hard reset on a virtual machine.
- Args:
- guest_hostname: the ip (as it was specified in the
- address list given to install()) of the guest
- to terminate.
+ Args:
+ guest_hostname: the ip (as it was specified in the
+ address list given to install()) of the guest
+ to terminate.
- Raises:
- AutoservVirtError: the guest_hostname argument is
- invalid
- """
- for address in self.addresses:
- if address["ip"] is guest_hostname:
- if address["is_used"]:
- break
- else:
- raise error.AutoservVirtError("guest "
- "hostname not in use")
- else:
- raise error.AutoservVirtError("Unknown guest hostname")
+ Raises:
+ AutoservVirtError: the guest_hostname argument is
+ invalid
+ """
+ for address in self.addresses:
+ if address["ip"] is guest_hostname:
+ if address["is_used"]:
+ break
+ else:
+ raise error.AutoservVirtError("guest "
+ "hostname not in use")
+ else:
+ raise error.AutoservVirtError("Unknown guest hostname")
- monitor_file_name= utils.sh_escape(os.path.join(self.pid_dir,
- "vhost%s_monitor" % (address["ip"],)))
+ monitor_file_name= utils.sh_escape(os.path.join(self.pid_dir,
+ "vhost%s_monitor" % (address["ip"],)))
- self.host.run('python -c "%s"' % (utils.sh_escape(
- _hard_reset_script % {
- "monitor_file_name" : monitor_file_name,}),))
+ self.host.run('python -c "%s"' % (utils.sh_escape(
+ _hard_reset_script % {
+ "monitor_file_name" : monitor_file_name,}),))
diff --git a/server/rpm_kernel.py b/server/rpm_kernel.py
index adaf180..5b8c8c0 100644
--- a/server/rpm_kernel.py
+++ b/server/rpm_kernel.py
@@ -5,7 +5,7 @@
"""
This module defines the Kernel class
- Kernel: an os kernel
+ Kernel: an os kernel
"""
__author__ = """
@@ -20,147 +20,147 @@
class RPMKernel(kernel.Kernel):
- """
- This class represents a .rpm pre-built kernel.
+ """
+ This class represents a .rpm pre-built kernel.
- It is used to obtain a built kernel and install it on a Host.
+ It is used to obtain a built kernel and install it on a Host.
- Implementation details:
- This is a leaf class in an abstract class hierarchy, it must
- implement the unimplemented methods in parent classes.
- """
- def __init__(self):
- super(RPMKernel, self).__init__()
+ Implementation details:
+ This is a leaf class in an abstract class hierarchy, it must
+ implement the unimplemented methods in parent classes.
+ """
+ def __init__(self):
+ super(RPMKernel, self).__init__()
- def install(self, host, label='autoserv',
- default=False, kernel_args = '', install_vmlinux=False):
- """
- Install a kernel on the remote host.
-
- This will also invoke the guest's bootloader to set this
- kernel as the default kernel if default=True.
-
- Args:
- host: the host on which to install the kernel
- [kwargs]: remaining keyword arguments will be passed
- to Bootloader.add_kernel()
-
- Raises:
- AutoservError: no package has yet been obtained. Call
- RPMKernel.get() with a .rpm package.
- """
- if len(label) > 15:
- raise error.AutoservError("label for kernel is too long \
- (> 15 chars): %s" % label)
- if self.source_material is None:
- raise error.AutoservError("A kernel must first be \
- specified via get()")
- rpm = self.source_material
+ def install(self, host, label='autoserv',
+ default=False, kernel_args = '', install_vmlinux=False):
+ """
+ Install a kernel on the remote host.
- remote_tmpdir = host.get_tmp_dir()
- remote_rpm = os.path.join(remote_tmpdir, os.path.basename(rpm))
- rpm_package = utils.run('/usr/bin/rpm -q -p %s' % rpm).stdout
- vmlinuz = self.get_image_name()
- host.send_file(rpm, remote_rpm)
- host.run('rpm -e ' + rpm_package, ignore_status = True)
- host.run('rpm --force -i ' + remote_rpm)
+ This will also invoke the guest's bootloader to set this
+ kernel as the default kernel if default=True.
- # Copy over the uncompressed image if there is one
- if install_vmlinux:
- vmlinux = self.get_vmlinux_name()
- host.run('cd /;rpm2cpio %s | cpio -imuv .%s'
- % (remote_rpm, vmlinux))
- host.run('ls ' + vmlinux) # Verify
+ Args:
+ host: the host on which to install the kernel
+ [kwargs]: remaining keyword arguments will be passed
+ to Bootloader.add_kernel()
- host.bootloader.remove_kernel(label)
- host.bootloader.add_kernel(vmlinuz, label,
- args=kernel_args, default=default)
- if kernel_args:
- host.bootloader.add_args(label, kernel_args)
- if not default:
- host.bootloader.boot_once(label)
+ Raises:
+ AutoservError: no package has yet been obtained. Call
+ RPMKernel.get() with a .rpm package.
+ """
+ if len(label) > 15:
+ raise error.AutoservError("label for kernel is too long \
+ (> 15 chars): %s" % label)
+ if self.source_material is None:
+ raise error.AutoservError("A kernel must first be \
+ specified via get()")
+ rpm = self.source_material
+
+ remote_tmpdir = host.get_tmp_dir()
+ remote_rpm = os.path.join(remote_tmpdir, os.path.basename(rpm))
+ rpm_package = utils.run('/usr/bin/rpm -q -p %s' % rpm).stdout
+ vmlinuz = self.get_image_name()
+ host.send_file(rpm, remote_rpm)
+ host.run('rpm -e ' + rpm_package, ignore_status = True)
+ host.run('rpm --force -i ' + remote_rpm)
+
+ # Copy over the uncompressed image if there is one
+ if install_vmlinux:
+ vmlinux = self.get_vmlinux_name()
+ host.run('cd /;rpm2cpio %s | cpio -imuv .%s'
+ % (remote_rpm, vmlinux))
+ host.run('ls ' + vmlinux) # Verify
+
+ host.bootloader.remove_kernel(label)
+ host.bootloader.add_kernel(vmlinuz, label,
+ args=kernel_args, default=default)
+ if kernel_args:
+ host.bootloader.add_args(label, kernel_args)
+ if not default:
+ host.bootloader.boot_once(label)
- def get_version(self):
- """Get the version of the kernel to be installed.
-
- Returns:
- The version string, as would be returned
- by 'make kernelrelease'.
-
- Raises:
- AutoservError: no package has yet been obtained. Call
- RPMKernel.get() with a .rpm package.
- """
- if self.source_material is None:
- raise error.AutoservError("A kernel must first be \
- specified via get()")
-
- retval = utils.run('rpm -qpi %s | grep Version | \
- awk \'{print($3);}\'' % utils.sh_escape(self.source_material))
- return retval.stdout.strip()
+ def get_version(self):
+ """Get the version of the kernel to be installed.
+
+ Returns:
+ The version string, as would be returned
+ by 'make kernelrelease'.
+
+ Raises:
+ AutoservError: no package has yet been obtained. Call
+ RPMKernel.get() with a .rpm package.
+ """
+ if self.source_material is None:
+ raise error.AutoservError("A kernel must first be \
+ specified via get()")
+
+ retval = utils.run('rpm -qpi %s | grep Version | \
+ awk \'{print($3);}\'' % utils.sh_escape(self.source_material))
+ return retval.stdout.strip()
- def get_image_name(self):
- """Get the name of the kernel image to be installed.
-
- Returns:
- The full path to the kernel image file as it will be
- installed on the host.
-
- Raises:
- AutoservError: no package has yet been obtained. Call
- RPMKernel.get() with a .rpm package.
- """
- if self.source_material is None:
- raise error.AutoservError("A kernel must first be \
- specified via get()")
-
- vmlinuz = utils.run('rpm -q -l -p %s \
- | grep /boot/vmlinuz' % self.source_material).stdout.strip()
- return vmlinuz
+ def get_image_name(self):
+ """Get the name of the kernel image to be installed.
+
+ Returns:
+ The full path to the kernel image file as it will be
+ installed on the host.
+
+ Raises:
+ AutoservError: no package has yet been obtained. Call
+ RPMKernel.get() with a .rpm package.
+ """
+ if self.source_material is None:
+ raise error.AutoservError("A kernel must first be \
+ specified via get()")
+
+ vmlinuz = utils.run('rpm -q -l -p %s \
+ | grep /boot/vmlinuz' % self.source_material).stdout.strip()
+ return vmlinuz
- def get_vmlinux_name(self):
- """Get the name of the kernel image to be installed.
-
- Returns:
- The full path to the kernel image file as it will be
- installed on the host. It is the uncompressed and
- unstripped version of the kernel that can be used with
- oprofile.
-
- Raises:
- AutoservError: no package has yet been obtained. Call
- RPMKernel.get() with a .rpm package.
- """
- if self.source_material is None:
- raise error.AutoservError("A kernel must first be \
- specified via get()")
-
- vmlinux = utils.run('rpm -q -l -p %s \
- | grep /boot/vmlinux' % self.source_material).stdout.strip()
- return vmlinux
+ def get_vmlinux_name(self):
+ """Get the name of the kernel image to be installed.
+
+ Returns:
+ The full path to the kernel image file as it will be
+ installed on the host. It is the uncompressed and
+ unstripped version of the kernel that can be used with
+ oprofile.
+
+ Raises:
+ AutoservError: no package has yet been obtained. Call
+ RPMKernel.get() with a .rpm package.
+ """
+ if self.source_material is None:
+ raise error.AutoservError("A kernel must first be \
+ specified via get()")
+
+ vmlinux = utils.run('rpm -q -l -p %s \
+ | grep /boot/vmlinux' % self.source_material).stdout.strip()
+ return vmlinux
- def get_initrd_name(self):
- """Get the name of the initrd file to be installed.
-
- Returns:
- The full path to the initrd file as it will be
- installed on the host. If the package includes no
- initrd file, None is returned
-
- Raises:
- AutoservError: no package has yet been obtained. Call
- RPMKernel.get() with a .rpm package.
- """
- if self.source_material is None:
- raise error.AutoservError("A kernel must first be \
- specified via get()")
+ def get_initrd_name(self):
+ """Get the name of the initrd file to be installed.
- res = utils.run('rpm -q -l -p %s \
- | grep /boot/initrd' % self.source_material, ignore_status=True)
- if res.exit_status:
- return None
- return res.stdout.strip()
+ Returns:
+ The full path to the initrd file as it will be
+ installed on the host. If the package includes no
+ initrd file, None is returned
+
+ Raises:
+ AutoservError: no package has yet been obtained. Call
+ RPMKernel.get() with a .rpm package.
+ """
+ if self.source_material is None:
+ raise error.AutoservError("A kernel must first be \
+ specified via get()")
+
+ res = utils.run('rpm -q -l -p %s \
+ | grep /boot/initrd' % self.source_material, ignore_status=True)
+ if res.exit_status:
+ return None
+ return res.stdout.strip()
diff --git a/server/samples/autoserv_console.srv b/server/samples/autoserv_console.srv
index 50bb62a..b0c69e6 100755
--- a/server/samples/autoserv_console.srv
+++ b/server/samples/autoserv_console.srv
@@ -9,12 +9,12 @@
# -Steve Howard (showard@google.com)
if machines:
- host = hosts.SSHHost(machines[0])
+ host = hosts.SSHHost(machines[0])
try:
- import IPython
- ipshell = IPython.Shell.IPShellEmbed(argv=[])
- ipshell()
+ import IPython
+ ipshell = IPython.Shell.IPShellEmbed(argv=[])
+ ipshell()
except ImportError:
- import code
- code.interact("Autoserv console", raw_input, locals())
+ import code
+ code.interact("Autoserv console", raw_input, locals())
diff --git a/server/samples/continuous_reboot.srv b/server/samples/continuous_reboot.srv
index 900273b..adbfc43 100644
--- a/server/samples/continuous_reboot.srv
+++ b/server/samples/continuous_reboot.srv
@@ -1,6 +1,6 @@
def run(machine):
- host = hosts.SSHHost(machine)
- while True:
- host.reboot()
+ host = hosts.SSHHost(machine)
+ while True:
+ host.reboot()
job.parallel_simple(run, machines)
diff --git a/server/samples/failtest.srv b/server/samples/failtest.srv
index 975ab83..a793e59 100644
--- a/server/samples/failtest.srv
+++ b/server/samples/failtest.srv
@@ -1,6 +1,6 @@
def run(machine):
- host = hosts.SSHHost(machine)
- at = autotest.Autotest(host)
- at.run_test('failtest')
+ host = hosts.SSHHost(machine)
+ at = autotest.Autotest(host)
+ at.run_test('failtest')
job.parallel_simple(run, machines)
diff --git a/server/samples/info.srv b/server/samples/info.srv
index 0dd7f88..a5d2350 100644
--- a/server/samples/info.srv
+++ b/server/samples/info.srv
@@ -1,8 +1,8 @@
def run(machine):
- host = hosts.SSHHost(machine, initialize = False)
- print 'Uptime: ' + host.check_uptime()
- print 'Arch: ' + host.get_arch()
- print 'Kernel ver: ' + host.get_kernel_ver()
- print 'Kernel cmdline: ' + host.get_cmdline()
+ host = hosts.SSHHost(machine, initialize = False)
+ print 'Uptime: ' + host.check_uptime()
+ print 'Arch: ' + host.get_arch()
+ print 'Kernel ver: ' + host.get_kernel_ver()
+ print 'Kernel cmdline: ' + host.get_cmdline()
job.parallel_simple(run, machines)
diff --git a/server/samples/kernbench.srv b/server/samples/kernbench.srv
index 174e6e7..a8d84e3 100644
--- a/server/samples/kernbench.srv
+++ b/server/samples/kernbench.srv
@@ -1,6 +1,6 @@
def run(machine):
- host = hosts.SSHHost(machine)
- at = autotest.Autotest(host)
- at.run_test('kernbench', iterations=2, threads=5)
+ host = hosts.SSHHost(machine)
+ at = autotest.Autotest(host)
+ at.run_test('kernbench', iterations=2, threads=5)
job.parallel_simple(run, machines)
diff --git a/server/samples/netperf-guest-to-host-far.srv b/server/samples/netperf-guest-to-host-far.srv
index 57b67ae..10ac8c7 100644
--- a/server/samples/netperf-guest-to-host-far.srv
+++ b/server/samples/netperf-guest-to-host-far.srv
@@ -2,19 +2,19 @@
def check_kernel(host, version, package):
- if host.run("uname -r").stdout.strip() != version:
- print "XXXX installing kernel on %s" % (host.hostname,)
- package.install(host)
-
- host.reboot()
- host.wait_up()
+ if host.run("uname -r").stdout.strip() != version:
+ print "XXXX installing kernel on %s" % (host.hostname,)
+ package.install(host)
+
+ host.reboot()
+ host.wait_up()
def install_kvm(kvm_on_host_var_name, host, source, addresses):
- exec ("global %(var_name)s\n"
- "%(var_name)s= kvm.KVM(host)\n"
- "%(var_name)s.get(source)\n"
- "%(var_name)s.install(addresses)\n" % {"var_name": kvm_on_host_var_name})
+ exec ("global %(var_name)s\n"
+ "%(var_name)s= kvm.KVM(host)\n"
+ "%(var_name)s.get(source)\n"
+ "%(var_name)s.install(addresses)\n" % {"var_name": kvm_on_host_var_name})
print "XXXX creating SSHHost objects"
diff --git a/server/samples/parallel.srv b/server/samples/parallel.srv
index 2fdfc57..24dc1f0 100644
--- a/server/samples/parallel.srv
+++ b/server/samples/parallel.srv
@@ -1,7 +1,7 @@
at = autotest.Autotest()
def run(machine):
- host = hosts.SSHHost(machine)
- at.run_test('sleeptest', host = host)
+ host = hosts.SSHHost(machine)
+ at.run_test('sleeptest', host = host)
job.parallel_simple(run, machines)
diff --git a/server/samples/parallel_kvm.srv b/server/samples/parallel_kvm.srv
index 682c020..101b51d 100644
--- a/server/samples/parallel_kvm.srv
+++ b/server/samples/parallel_kvm.srv
@@ -21,9 +21,9 @@
num_guests= 5
g= []
for i in range(num_guests):
- g.append(hosts.KVMGuest(kvm_on_remote_host, qemu_options))
+ g.append(hosts.KVMGuest(kvm_on_remote_host, qemu_options))
for i in range(num_guests):
- g[i].wait_up()
+ g[i].wait_up()
print "XXXX running transfers"
@@ -35,10 +35,10 @@
args= range(1, num_guests)
def f(i):
- print "This is %s" % i
- tmp_dir= g[i].get_tmp_dir()
- g[i].run('scp "%s":"%s" "%s"' % (g[0].hostname, big_file, tmp_dir,))
- print g[i].run('sha1sum "%s"' % (os.path.join(tmp_dir, "big_file"),)).stdout.strip()
+ print "This is %s" % i
+ tmp_dir= g[i].get_tmp_dir()
+ g[i].run('scp "%s":"%s" "%s"' % (g[0].hostname, big_file, tmp_dir,))
+ print g[i].run('sha1sum "%s"' % (os.path.join(tmp_dir, "big_file"),)).stdout.strip()
job.parallel_simple(f, args)
diff --git a/server/samples/parallel_sleeptest.srv b/server/samples/parallel_sleeptest.srv
index 49bb0d1..c46353e 100644
--- a/server/samples/parallel_sleeptest.srv
+++ b/server/samples/parallel_sleeptest.srv
@@ -22,34 +22,34 @@
# Specify the path to the client control files and the tag names
# for the respective jobs here.
tests = [("client/tests/sleeptest/control", "sleeptag0"),
- ("client/tests/sleeptest/control", "sleeptag1"),
- ]
+ ("client/tests/sleeptest/control", "sleeptag1"),
+ ]
def run_client(at, machine_name, machine_num, instance):
- control = open(os.path.join(job.autodir,tests[instance][0])).read()
- '''
- The get_sync_control_file method basically does the setup of the barriers
- required to start these multiple tests at the same time and returns the
- modified control file (that contains the barrier code added to it)
- Check client/common_lib/utils.py for detailed documentation of how this
- method sets up the barriers.
- '''
- control_new = utils.get_sync_control_file(control, machine_name,
- machine_num, instance, len(tests))
- '''
- This control file is now simply passed in to the run method along with
- a tag name of the test and a 'parallel_flag' that identifies this scenario
- of running multiple tests on the same machine at the same time.
- '''
- at.run(control_new, tag='%s' % tests[instance][1], parallel_flag=True)
+ control = open(os.path.join(job.autodir,tests[instance][0])).read()
+ '''
+ The get_sync_control_file method basically does the setup of the barriers
+ required to start these multiple tests at the same time and returns the
+ modified control file (that contains the barrier code added to it)
+ Check client/common_lib/utils.py for detailed documentation of how this
+ method sets up the barriers.
+ '''
+ control_new = utils.get_sync_control_file(control, machine_name,
+ machine_num, instance, len(tests))
+ '''
+ This control file is now simply passed in to the run method along with
+ a tag name of the test and a 'parallel_flag' that identifies this scenario
+ of running multiple tests on the same machine at the same time.
+ '''
+ at.run(control_new, tag='%s' % tests[instance][1], parallel_flag=True)
def main(machine_name, machine_num):
- host = hosts.SSHHost(machine_name)
- at = autotest.Autotest(host)
- at.install()
+ host = hosts.SSHHost(machine_name)
+ at = autotest.Autotest(host)
+ at.install()
- parallel([subcommand(run_client, [at, machine_name, machine_num, i])
- for i in range(len(tests))])
+ parallel([subcommand(run_client, [at, machine_name, machine_num, i])
+ for i in range(len(tests))])
parallel([subcommand(main, [machines[i], i], machines[i])
- for i in range(len(machines))])
+ for i in range(len(machines))])
diff --git a/server/samples/profilertest.srv b/server/samples/profilertest.srv
index 2743b76..42f32f2 100644
--- a/server/samples/profilertest.srv
+++ b/server/samples/profilertest.srv
@@ -5,63 +5,63 @@
def add_profilers(at, profilers, timeout_sync, timeout_start, timeout_stop,
- machines, name):
- control_file = []
- for profiler in profilers:
- control_file.append("job.profilers.add(%s)"
- % str(profiler)[1:-1])
+ machines, name):
+ control_file = []
+ for profiler in profilers:
+ control_file.append("job.profilers.add(%s)"
+ % str(profiler)[1:-1])
- control_file.append("job.run_test('barriertest',%d,%d,%d,'%s','%s',%s)"
- % (timeout_sync, timeout_start, timeout_stop,
- at.host.hostname, "PROF_MASTER", str(machines)))
+ control_file.append("job.run_test('barriertest',%d,%d,%d,'%s','%s',%s)"
+ % (timeout_sync, timeout_start, timeout_stop,
+ at.host.hostname, "PROF_MASTER", str(machines)))
- for profiler in profilers:
- control_file.append("job.profilers.delete('%s')" % profiler[0])
+ for profiler in profilers:
+ control_file.append("job.profilers.delete('%s')" % profiler[0])
- params = ["\n".join(control_file), "profile-" + profiler[0], at.host]
- return subcommand(at.run, params, name)
+ params = ["\n".join(control_file), "profile-" + profiler[0], at.host]
+ return subcommand(at.run, params, name)
def wait_for_profilers(machines, timeout = 180):
- # wait until the profilers have started
- sync_bar = barrier("PROF_MASTER", "sync_profilers",
- timeout, port=63100)
- sync_bar.rendevous_servers("PROF_MASTER", *machines)
+ # wait until the profilers have started
+ sync_bar = barrier("PROF_MASTER", "sync_profilers",
+ timeout, port=63100)
+ sync_bar.rendevous_servers("PROF_MASTER", *machines)
def start_profilers(machines, timeout = 180):
- # wait until the profilers have started
- start_bar = barrier("PROF_MASTER", "start_profilers",
- timeout, port=63100)
- start_bar.rendevous_servers("PROF_MASTER", *machines)
+ # wait until the profilers have started
+ start_bar = barrier("PROF_MASTER", "start_profilers",
+ timeout, port=63100)
+ start_bar.rendevous_servers("PROF_MASTER", *machines)
def stop_profilers(machines, timeout = 120):
- stop_bar = barrier("PROF_MASTER", "stop_profilers", timeout, port=63100)
- stop_bar.rendevous_servers("PROF_MASTER", *machines)
+ stop_bar = barrier("PROF_MASTER", "stop_profilers", timeout, port=63100)
+ stop_bar.rendevous_servers("PROF_MASTER", *machines)
def server_sleep_test(seconds):
- wait_for_profilers(machines)
- start_profilers(machines)
- for i in range(seconds):
- print "%d of %d" % (i, seconds)
- time.sleep(1)
- stop_profilers(machines)
+ wait_for_profilers(machines)
+ start_profilers(machines)
+ for i in range(seconds):
+ print "%d of %d" % (i, seconds)
+ time.sleep(1)
+ stop_profilers(machines)
def main():
- timeout_sync = 180
- timeout_start = 60
- timeout_stop = 60
- profilers = [["vmstat"], ["iostat"]]
+ timeout_sync = 180
+ timeout_start = 60
+ timeout_stop = 60
+ profilers = [["vmstat"], ["iostat"]]
- tests = [subcommand(server_sleep_test, [20], "server_sleep_test")]
- for at in at_hosts:
- name = "profiled-%s" % at.host.hostname
- tests.append(add_profilers(at, profilers, timeout_sync,
- timeout_start, timeout_stop, machines, name))
- parallel(tests)
+ tests = [subcommand(server_sleep_test, [20], "server_sleep_test")]
+ for at in at_hosts:
+ name = "profiled-%s" % at.host.hostname
+ tests.append(add_profilers(at, profilers, timeout_sync,
+ timeout_start, timeout_stop, machines, name))
+ parallel(tests)
main()
diff --git a/server/samples/reboot.srv b/server/samples/reboot.srv
index 670bf5c..5837570 100644
--- a/server/samples/reboot.srv
+++ b/server/samples/reboot.srv
@@ -1,6 +1,6 @@
def run(machine):
- host = hosts.SSHHost(machine)
- print host.is_up()
- host.reboot()
+ host = hosts.SSHHost(machine)
+ print host.is_up()
+ host.reboot()
job.parallel_simple(run, machines)
diff --git a/server/samples/run_test.srv b/server/samples/run_test.srv
index 1220445..e8b1f36 100644
--- a/server/samples/run_test.srv
+++ b/server/samples/run_test.srv
@@ -5,38 +5,38 @@
def usage():
- print "usage: -t <test name> -m <machines> -l <log dir>"
+ print "usage: -t <test name> -m <machines> -l <log dir>"
def run(client):
- m = hosts.SSHHost(client)
- at = autotest.Autotest()
+ m = hosts.SSHHost(client)
+ at = autotest.Autotest()
- results_dir = os.path.join(logdir, client)
- at.run_test(test, results_dir, m)
+ results_dir = os.path.join(logdir, client)
+ at.run_test(test, results_dir, m)
def main():
- global test, logdir, args
+ global test, logdir, args
- try:
- opts, args = getopt.getopt(args, 't:l:', [])
- except getopt.GetoptError, e:
- usage()
- print e
- sys.exit(1)
+ try:
+ opts, args = getopt.getopt(args, 't:l:', [])
+ except getopt.GetoptError, e:
+ usage()
+ print e
+ sys.exit(1)
- for flag, value in opts:
- if flag == '-t':
- test = value
- elif flag == '-l':
- logdir = value
-
- if test == None or logdir == None:
- usage()
- sys.exit(1)
+ for flag, value in opts:
+ if flag == '-t':
+ test = value
+ elif flag == '-l':
+ logdir = value
- print "Going to launch %s on %r with log dir of %s." % (test, machines, logdir)
- parallel_simple(run, machines)
+ if test == None or logdir == None:
+ usage()
+ sys.exit(1)
+
+ print "Going to launch %s on %r with log dir of %s." % (test, machines, logdir)
+ parallel_simple(run, machines)
main()
diff --git a/server/samples/sleeptest.srv b/server/samples/sleeptest.srv
index 805ac097..85cbf08 100644
--- a/server/samples/sleeptest.srv
+++ b/server/samples/sleeptest.srv
@@ -1,6 +1,6 @@
def run(machine):
- host = hosts.SSHHost(machine)
- at = autotest.Autotest(host)
- at.run_test('sleeptest')
+ host = hosts.SSHHost(machine)
+ at = autotest.Autotest(host)
+ at.run_test('sleeptest')
job.parallel_simple(run, machines)
diff --git a/server/samples/uname.srv b/server/samples/uname.srv
index b141d5c..f67d626 100644
--- a/server/samples/uname.srv
+++ b/server/samples/uname.srv
@@ -1,5 +1,5 @@
hosts = [hosts.SSHHost(h, initialize=False) for h in machines]
for host in hosts:
- print host.hostname
- print host.run('uname -a').stdout.rstrip()
+ print host.hostname
+ print host.run('uname -a').stdout.rstrip()
diff --git a/server/self-test/alltests_suite.py b/server/self-test/alltests_suite.py
index 14ffc1d..1d9a8b0 100644
--- a/server/self-test/alltests_suite.py
+++ b/server/self-test/alltests_suite.py
@@ -12,7 +12,7 @@
# Adjust the path so Python can find the autoserv modules
src = os.path.abspath("%s/.." % (os.path.dirname(sys.argv[0]),))
if src not in sys.path:
- sys.path.insert(1, src)
+ sys.path.insert(1, src)
import unittest
@@ -22,9 +22,9 @@
def suite():
- return unittest.TestSuite([autotest_test.suite(),
- utils_test.suite()])
+ return unittest.TestSuite([autotest_test.suite(),
+ utils_test.suite()])
if __name__ == '__main__':
- unittest.TextTestRunner(verbosity=2).run(suite())
+ unittest.TextTestRunner(verbosity=2).run(suite())
diff --git a/server/self-test/autotest_test.py b/server/self-test/autotest_test.py
index 033b091..08dc482 100644
--- a/server/self-test/autotest_test.py
+++ b/server/self-test/autotest_test.py
@@ -14,7 +14,7 @@
# Adjust the path so Python can find the autoserv modules
src = os.path.abspath("%s/.." % (os.path.dirname(sys.argv[0]),))
if src not in sys.path:
- sys.path.insert(1, src)
+ sys.path.insert(1, src)
import utils
import autotest
@@ -22,119 +22,119 @@
class AutotestTestCase(unittest.TestCase):
- def setUp(self):
- self.autotest = autotest.Autotest()
-
- def tearDown(self):
- pass
+ def setUp(self):
+ self.autotest = autotest.Autotest()
+
+ def tearDown(self):
+ pass
- def testGetAutoDir(self):
- class MockInstallHost:
- def __init__(self):
- self.commands = []
- self.result = "autodir='/stuff/autotest'\n"
-
- def run(self, command):
- if command == "grep autodir= /etc/autotest.conf":
- result = hosts.CmdResult()
- result.stdout = self.result
- return result
- else:
- self.commands.append(command)
-
- host = MockInstallHost()
- self.assertEqual('/stuff/autotest',
- autotest._get_autodir(host))
- host.result = "autodir=/stuff/autotest\n"
- self.assertEqual('/stuff/autotest',
- autotest._get_autodir(host))
- host.result = 'autodir="/stuff/auto test"\n'
- self.assertEqual('/stuff/auto test',
- autotest._get_autodir(host))
+ def testGetAutoDir(self):
+ class MockInstallHost:
+ def __init__(self):
+ self.commands = []
+ self.result = "autodir='/stuff/autotest'\n"
+
+ def run(self, command):
+ if command == "grep autodir= /etc/autotest.conf":
+ result = hosts.CmdResult()
+ result.stdout = self.result
+ return result
+ else:
+ self.commands.append(command)
+
+ host = MockInstallHost()
+ self.assertEqual('/stuff/autotest',
+ autotest._get_autodir(host))
+ host.result = "autodir=/stuff/autotest\n"
+ self.assertEqual('/stuff/autotest',
+ autotest._get_autodir(host))
+ host.result = 'autodir="/stuff/auto test"\n'
+ self.assertEqual('/stuff/auto test',
+ autotest._get_autodir(host))
- def testInstallFromDir(self):
- class MockInstallHost:
- def __init__(self):
- self.commands = []
-
- def run(self, command):
- if command == "grep autodir= /etc/autotest.conf":
- result= hosts.CmdResult()
- result.stdout = "autodir=/usr/local/autotest\n"
- return result
- else:
- self.commands.append(command)
+ def testInstallFromDir(self):
+ class MockInstallHost:
+ def __init__(self):
+ self.commands = []
- def send_file(self, src, dst):
- self.commands.append("send_file: %s %s" % (src,
- dst))
-
- host = MockInstallHost()
- tmpdir = utils.get_tmp_dir()
- self.autotest.get(tmpdir)
- self.autotest.install(host)
- self.assertEqual(host.commands[0],
- 'mkdir -p /usr/local/autotest')
- self.assertTrue(host.commands[1].startswith('send_file: /tmp/'))
- self.assertTrue(host.commands[1].endswith(
- '/ /usr/local/autotest'))
+ def run(self, command):
+ if command == "grep autodir= /etc/autotest.conf":
+ result= hosts.CmdResult()
+ result.stdout = "autodir=/usr/local/autotest\n"
+ return result
+ else:
+ self.commands.append(command)
-
+ def send_file(self, src, dst):
+ self.commands.append("send_file: %s %s" % (src,
+ dst))
-
- def testInstallFromSVN(self):
- class MockInstallHost:
- def __init__(self):
- self.commands = []
-
- def run(self, command):
- if command == "grep autodir= /etc/autotest.conf":
- result= hosts.CmdResult()
- result.stdout = "autodir=/usr/local/autotest\n"
- return result
- else:
- self.commands.append(command)
-
- host = MockInstallHost()
- self.autotest.install(host)
- self.assertEqual(host.commands,
- ['svn checkout '
- + autotest.AUTOTEST_SVN + ' '
- + "/usr/local/autotest"])
+ host = MockInstallHost()
+ tmpdir = utils.get_tmp_dir()
+ self.autotest.get(tmpdir)
+ self.autotest.install(host)
+ self.assertEqual(host.commands[0],
+ 'mkdir -p /usr/local/autotest')
+ self.assertTrue(host.commands[1].startswith('send_file: /tmp/'))
+ self.assertTrue(host.commands[1].endswith(
+ '/ /usr/local/autotest'))
-
- def testFirstInstallFromSVNFails(self):
- class MockFirstInstallFailsHost:
- def __init__(self):
- self.commands = []
-
- def run(self, command):
- if command == "grep autodir= /etc/autotest.conf":
- result= hosts.CmdResult()
- result.stdout = "autodir=/usr/local/autotest\n"
- return result
- else:
- self.commands.append(command)
- first = ('svn checkout ' +
- autotest.AUTOTEST_SVN + ' ' +
- "/usr/local/autotest")
- if (command == first):
- raise autotest.AutoservRunError(
- "svn not found")
-
- host = MockFirstInstallFailsHost()
- self.autotest.install(host)
- self.assertEqual(host.commands,
- ['svn checkout ' + autotest.AUTOTEST_SVN +
- ' ' + "/usr/local/autotest",
- 'svn checkout ' + autotest.AUTOTEST_HTTP +
- ' ' + "/usr/local/autotest"])
+
+
+
+ def testInstallFromSVN(self):
+ class MockInstallHost:
+ def __init__(self):
+ self.commands = []
+
+ def run(self, command):
+ if command == "grep autodir= /etc/autotest.conf":
+ result= hosts.CmdResult()
+ result.stdout = "autodir=/usr/local/autotest\n"
+ return result
+ else:
+ self.commands.append(command)
+
+ host = MockInstallHost()
+ self.autotest.install(host)
+ self.assertEqual(host.commands,
+ ['svn checkout '
+ + autotest.AUTOTEST_SVN + ' '
+ + "/usr/local/autotest"])
+
+
+ def testFirstInstallFromSVNFails(self):
+ class MockFirstInstallFailsHost:
+ def __init__(self):
+ self.commands = []
+
+ def run(self, command):
+ if command == "grep autodir= /etc/autotest.conf":
+ result= hosts.CmdResult()
+ result.stdout = "autodir=/usr/local/autotest\n"
+ return result
+ else:
+ self.commands.append(command)
+ first = ('svn checkout ' +
+ autotest.AUTOTEST_SVN + ' ' +
+ "/usr/local/autotest")
+ if (command == first):
+ raise autotest.AutoservRunError(
+ "svn not found")
+
+ host = MockFirstInstallFailsHost()
+ self.autotest.install(host)
+ self.assertEqual(host.commands,
+ ['svn checkout ' + autotest.AUTOTEST_SVN +
+ ' ' + "/usr/local/autotest",
+ 'svn checkout ' + autotest.AUTOTEST_HTTP +
+ ' ' + "/usr/local/autotest"])
def suite():
- return unittest.TestLoader().loadTestsFromTestCase(AutotestTestCase)
+ return unittest.TestLoader().loadTestsFromTestCase(AutotestTestCase)
if __name__ == '__main__':
- unittest.TextTestRunner(verbosity=2).run(suite())
+ unittest.TextTestRunner(verbosity=2).run(suite())
diff --git a/server/self-test/utils_test.py b/server/self-test/utils_test.py
index 0978752..fffbf9a 100644
--- a/server/self-test/utils_test.py
+++ b/server/self-test/utils_test.py
@@ -15,62 +15,62 @@
# Adjust the path so Python can find the autoserv modules
src = os.path.abspath("%s/.." % (os.path.dirname(sys.argv[0]),))
if src not in sys.path:
- sys.path.insert(1, src)
+ sys.path.insert(1, src)
import utils
class UtilsTestCase(unittest.TestCase):
- def setUp(self):
- pass
+ def setUp(self):
+ pass
- def tearDown(self):
- pass
+ def tearDown(self):
+ pass
- def testGetWithOpenFile(self):
- tmpdir = utils.get_tmp_dir()
- tmppath = os.path.join(tmpdir, 'testfile')
- tmpfile = file(tmppath, 'w')
- print >> tmpfile, 'Test string'
- tmpfile.close()
- tmpfile = file(tmppath)
- newtmppath = utils.get(tmpfile)
- self.assertEqual(file(newtmppath).read(), 'Test string\n')
+ def testGetWithOpenFile(self):
+ tmpdir = utils.get_tmp_dir()
+ tmppath = os.path.join(tmpdir, 'testfile')
+ tmpfile = file(tmppath, 'w')
+ print >> tmpfile, 'Test string'
+ tmpfile.close()
+ tmpfile = file(tmppath)
+ newtmppath = utils.get(tmpfile)
+ self.assertEqual(file(newtmppath).read(), 'Test string\n')
- def testGetWithHTTP(self):
- # Yeah, this test is a bad idea, oh well
- url = 'http://www.kernel.org/pub/linux/kernel/README'
- tmppath = utils.get(url)
- f = file(tmppath)
- f.readline()
- self.assertTrue('Linux' in f.readline().split())
+ def testGetWithHTTP(self):
+ # Yeah, this test is a bad idea, oh well
+ url = 'http://www.kernel.org/pub/linux/kernel/README'
+ tmppath = utils.get(url)
+ f = file(tmppath)
+ f.readline()
+ self.assertTrue('Linux' in f.readline().split())
- def testGetWithPath(self):
- path = utils.get('/proc/cpuinfo')
- self.assertTrue(file(path).readline().startswith('processor'))
+ def testGetWithPath(self):
+ path = utils.get('/proc/cpuinfo')
+ self.assertTrue(file(path).readline().startswith('processor'))
- def testGetWithString(self):
- path = utils.get('/tmp loves rabbits!')
- self.assertTrue(file(path).readline().startswith('/tmp loves'))
+ def testGetWithString(self):
+ path = utils.get('/tmp loves rabbits!')
+ self.assertTrue(file(path).readline().startswith('/tmp loves'))
- def testGetWithDir(self):
- tmpdir = utils.get_tmp_dir()
- origpath = os.path.join(tmpdir, 'testGetWithDir')
- os.mkdir(origpath)
- dstpath = utils.get(origpath)
- self.assertTrue(dstpath.endswith('/'))
- self.assertTrue(os.path.isdir(dstpath))
+ def testGetWithDir(self):
+ tmpdir = utils.get_tmp_dir()
+ origpath = os.path.join(tmpdir, 'testGetWithDir')
+ os.mkdir(origpath)
+ dstpath = utils.get(origpath)
+ self.assertTrue(dstpath.endswith('/'))
+ self.assertTrue(os.path.isdir(dstpath))
def suite():
- return unittest.TestLoader().loadTestsFromTestCase(UtilsTestCase)
+ return unittest.TestLoader().loadTestsFromTestCase(UtilsTestCase)
if __name__ == '__main__':
- unittest.TextTestRunner(verbosity=2).run(suite())
+ unittest.TextTestRunner(verbosity=2).run(suite())
diff --git a/server/server_job.py b/server/server_job.py
index ed91713..8b6c2c6 100755
--- a/server/server_job.py
+++ b/server/server_job.py
@@ -23,12 +23,12 @@
# load up a control segment
# these are all stored in <server_dir>/control_segments
def load_control_segment(name):
- server_dir = os.path.dirname(os.path.abspath(__file__))
- script_file = os.path.join(server_dir, "control_segments", name)
- if os.path.exists(script_file):
- return file(script_file).read()
- else:
- return ""
+ server_dir = os.path.dirname(os.path.abspath(__file__))
+ script_file = os.path.join(server_dir, "control_segments", name)
+ if os.path.exists(script_file):
+ return file(script_file).read()
+ else:
+ return ""
preamble = """\
@@ -47,54 +47,54 @@
hosts.SSHHost.job = job
barrier = barrier.barrier
if len(machines) > 1:
- open('.machines', 'w').write('\\n'.join(machines) + '\\n')
+ open('.machines', 'w').write('\\n'.join(machines) + '\\n')
"""
client_wrapper = """
at = autotest.Autotest()
def run_client(machine):
- hostname, user, password, port = parse_machine(machine,
- ssh_user, ssh_port, ssh_pass)
+ hostname, user, password, port = parse_machine(machine,
+ ssh_user, ssh_port, ssh_pass)
- host = hosts.SSHHost(hostname, user, port, password=password)
- at.run(control, host=host)
+ host = hosts.SSHHost(hostname, user, port, password=password)
+ at.run(control, host=host)
job.parallel_simple(run_client, machines)
"""
crashdumps = """
def crashdumps(machine):
- hostname, user, password, port = parse_machine(machine,
- ssh_user, ssh_port, ssh_pass)
+ hostname, user, password, port = parse_machine(machine,
+ ssh_user, ssh_port, ssh_pass)
- host = hosts.SSHHost(hostname, user, port, initialize=False, \
- password=password)
- host.get_crashdumps(test_start_time)
+ host = hosts.SSHHost(hostname, user, port, initialize=False, \
+ password=password)
+ host.get_crashdumps(test_start_time)
job.parallel_simple(crashdumps, machines, log=False)
"""
reboot_segment="""\
def reboot(machine):
- hostname, user, password, port = parse_machine(machine,
- ssh_user, ssh_port, ssh_pass)
+ hostname, user, password, port = parse_machine(machine,
+ ssh_user, ssh_port, ssh_pass)
- host = hosts.SSHHost(hostname, user, port, initialize=False, \
- password=password)
- host.reboot()
+ host = hosts.SSHHost(hostname, user, port, initialize=False, \
+ password=password)
+ host.reboot()
job.parallel_simple(reboot, machines, log=False)
"""
install="""\
def install(machine):
- hostname, user, password, port = parse_machine(machine,
- ssh_user, ssh_port, ssh_pass)
+ hostname, user, password, port = parse_machine(machine,
+ ssh_user, ssh_port, ssh_pass)
- host = hosts.SSHHost(hostname, user, port, initialize=False, \
- password=password)
- host.machine_install()
+ host = hosts.SSHHost(hostname, user, port, initialize=False, \
+ password=password)
+ host.machine_install()
job.parallel_simple(install, machines, log=False)
"""
@@ -110,701 +110,701 @@
# load up site-specific code for generating site-specific job data
try:
- import site_job
- get_site_job_data = site_job.get_site_job_data
- del site_job
+ import site_job
+ get_site_job_data = site_job.get_site_job_data
+ del site_job
except ImportError:
- # by default provide a stub that generates no site data
- def get_site_job_data(job):
- return {}
+ # by default provide a stub that generates no site data
+ def get_site_job_data(job):
+ return {}
class base_server_job:
- """The actual job against which we do everything.
+ """The actual job against which we do everything.
- Properties:
- autodir
- The top level autotest directory (/usr/local/autotest).
- serverdir
- <autodir>/server/
- clientdir
- <autodir>/client/
- conmuxdir
- <autodir>/conmux/
- testdir
- <autodir>/server/tests/
- control
- the control file for this job
- """
+ Properties:
+ autodir
+ The top level autotest directory (/usr/local/autotest).
+ serverdir
+ <autodir>/server/
+ clientdir
+ <autodir>/client/
+ conmuxdir
+ <autodir>/conmux/
+ testdir
+ <autodir>/server/tests/
+ control
+ the control file for this job
+ """
- STATUS_VERSION = 1
+ STATUS_VERSION = 1
- def __init__(self, control, args, resultdir, label, user, machines,
- client=False, parse_job="",
- ssh_user='root', ssh_port=22, ssh_pass=''):
- """
- control
- The control file (pathname of)
- args
- args to pass to the control file
- resultdir
- where to throw the results
- label
- label for the job
- user
- Username for the job (email address)
- client
- True if a client-side control file
- """
- path = os.path.dirname(__file__)
- self.autodir = os.path.abspath(os.path.join(path, '..'))
- self.serverdir = os.path.join(self.autodir, 'server')
- self.testdir = os.path.join(self.serverdir, 'tests')
- self.tmpdir = os.path.join(self.serverdir, 'tmp')
- self.conmuxdir = os.path.join(self.autodir, 'conmux')
- self.clientdir = os.path.join(self.autodir, 'client')
- self.toolsdir = os.path.join(self.autodir, 'client/tools')
- if control:
- self.control = open(control, 'r').read()
- self.control = re.sub('\r', '', self.control)
- else:
- self.control = None
- self.resultdir = resultdir
- if not os.path.exists(resultdir):
- os.mkdir(resultdir)
- self.debugdir = os.path.join(resultdir, 'debug')
- if not os.path.exists(self.debugdir):
- os.mkdir(self.debugdir)
- self.status = os.path.join(resultdir, 'status')
- self.label = label
- self.user = user
- self.args = args
- self.machines = machines
- self.client = client
- self.record_prefix = ''
- self.warning_loggers = set()
- self.ssh_user = ssh_user
- self.ssh_port = ssh_port
- self.ssh_pass = ssh_pass
+ def __init__(self, control, args, resultdir, label, user, machines,
+ client=False, parse_job="",
+ ssh_user='root', ssh_port=22, ssh_pass=''):
+ """
+ control
+ The control file (pathname of)
+ args
+ args to pass to the control file
+ resultdir
+ where to throw the results
+ label
+ label for the job
+ user
+ Username for the job (email address)
+ client
+ True if a client-side control file
+ """
+ path = os.path.dirname(__file__)
+ self.autodir = os.path.abspath(os.path.join(path, '..'))
+ self.serverdir = os.path.join(self.autodir, 'server')
+ self.testdir = os.path.join(self.serverdir, 'tests')
+ self.tmpdir = os.path.join(self.serverdir, 'tmp')
+ self.conmuxdir = os.path.join(self.autodir, 'conmux')
+ self.clientdir = os.path.join(self.autodir, 'client')
+ self.toolsdir = os.path.join(self.autodir, 'client/tools')
+ if control:
+ self.control = open(control, 'r').read()
+ self.control = re.sub('\r', '', self.control)
+ else:
+ self.control = None
+ self.resultdir = resultdir
+ if not os.path.exists(resultdir):
+ os.mkdir(resultdir)
+ self.debugdir = os.path.join(resultdir, 'debug')
+ if not os.path.exists(self.debugdir):
+ os.mkdir(self.debugdir)
+ self.status = os.path.join(resultdir, 'status')
+ self.label = label
+ self.user = user
+ self.args = args
+ self.machines = machines
+ self.client = client
+ self.record_prefix = ''
+ self.warning_loggers = set()
+ self.ssh_user = ssh_user
+ self.ssh_port = ssh_port
+ self.ssh_pass = ssh_pass
- self.stdout = fd_stack.fd_stack(1, sys.stdout)
- self.stderr = fd_stack.fd_stack(2, sys.stderr)
+ self.stdout = fd_stack.fd_stack(1, sys.stdout)
+ self.stderr = fd_stack.fd_stack(2, sys.stderr)
- if os.path.exists(self.status):
- os.unlink(self.status)
- job_data = {'label' : label, 'user' : user,
- 'hostname' : ','.join(machines),
- 'status_version' : str(self.STATUS_VERSION)}
- job_data.update(get_site_job_data(self))
- utils.write_keyval(self.resultdir, job_data)
+ if os.path.exists(self.status):
+ os.unlink(self.status)
+ job_data = {'label' : label, 'user' : user,
+ 'hostname' : ','.join(machines),
+ 'status_version' : str(self.STATUS_VERSION)}
+ job_data.update(get_site_job_data(self))
+ utils.write_keyval(self.resultdir, job_data)
- self.parse_job = parse_job
- if self.parse_job and len(machines) == 1:
- self.using_parser = True
- self.init_parser(resultdir)
- else:
- self.using_parser = False
+ self.parse_job = parse_job
+ if self.parse_job and len(machines) == 1:
+ self.using_parser = True
+ self.init_parser(resultdir)
+ else:
+ self.using_parser = False
- def init_parser(self, resultdir):
- """Start the continuous parsing of resultdir. This sets up
- the database connection and inserts the basic job object into
- the database if necessary."""
- # redirect parser debugging to .parse.log
- parse_log = os.path.join(resultdir, '.parse.log')
- parse_log = open(parse_log, 'w', 0)
- tko_utils.redirect_parser_debugging(parse_log)
- # create a job model object and set up the db
- self.results_db = tko_db.db(autocommit=True)
- self.parser = status_lib.parser(self.STATUS_VERSION)
- self.job_model = self.parser.make_job(resultdir)
- self.parser.start(self.job_model)
- # check if a job already exists in the db and insert it if
- # it does not
- job_idx = self.results_db.find_job(self.parse_job)
- if job_idx is None:
- self.results_db.insert_job(self.parse_job,
- self.job_model)
- else:
- machine_idx = self.results_db.lookup_machine(
- self.job_model.machine)
- self.job_model.index = job_idx
- self.job_model.machine_idx = machine_idx
+ def init_parser(self, resultdir):
+ """Start the continuous parsing of resultdir. This sets up
+ the database connection and inserts the basic job object into
+ the database if necessary."""
+ # redirect parser debugging to .parse.log
+ parse_log = os.path.join(resultdir, '.parse.log')
+ parse_log = open(parse_log, 'w', 0)
+ tko_utils.redirect_parser_debugging(parse_log)
+ # create a job model object and set up the db
+ self.results_db = tko_db.db(autocommit=True)
+ self.parser = status_lib.parser(self.STATUS_VERSION)
+ self.job_model = self.parser.make_job(resultdir)
+ self.parser.start(self.job_model)
+ # check if a job already exists in the db and insert it if
+ # it does not
+ job_idx = self.results_db.find_job(self.parse_job)
+ if job_idx is None:
+ self.results_db.insert_job(self.parse_job,
+ self.job_model)
+ else:
+ machine_idx = self.results_db.lookup_machine(
+ self.job_model.machine)
+ self.job_model.index = job_idx
+ self.job_model.machine_idx = machine_idx
- def cleanup_parser(self):
- """This should be called after the server job is finished
- to carry out any remaining cleanup (e.g. flushing any
- remaining test results to the results db)"""
- if not self.using_parser:
- return
- final_tests = self.parser.end()
- for test in final_tests:
- self.__insert_test(test)
- self.using_parser = False
+ def cleanup_parser(self):
+ """This should be called after the server job is finished
+ to carry out any remaining cleanup (e.g. flushing any
+ remaining test results to the results db)"""
+ if not self.using_parser:
+ return
+ final_tests = self.parser.end()
+ for test in final_tests:
+ self.__insert_test(test)
+ self.using_parser = False
- def verify(self):
- if not self.machines:
- raise error.AutoservError(
- 'No machines specified to verify')
- try:
- namespace = {'machines' : self.machines, 'job' : self, \
- 'ssh_user' : self.ssh_user, \
- 'ssh_port' : self.ssh_port, \
- 'ssh_pass' : self.ssh_pass}
- exec(preamble + verify, namespace, namespace)
- except Exception, e:
- msg = ('Verify failed\n' + str(e) + '\n'
- + traceback.format_exc())
- self.record('ABORT', None, None, msg)
- raise
+ def verify(self):
+ if not self.machines:
+ raise error.AutoservError(
+ 'No machines specified to verify')
+ try:
+ namespace = {'machines' : self.machines, 'job' : self, \
+ 'ssh_user' : self.ssh_user, \
+ 'ssh_port' : self.ssh_port, \
+ 'ssh_pass' : self.ssh_pass}
+ exec(preamble + verify, namespace, namespace)
+ except Exception, e:
+ msg = ('Verify failed\n' + str(e) + '\n'
+ + traceback.format_exc())
+ self.record('ABORT', None, None, msg)
+ raise
- def repair(self):
- if not self.machines:
- raise error.AutoservError(
- 'No machines specified to repair')
- namespace = {'machines' : self.machines, 'job' : self, \
- 'ssh_user' : self.ssh_user, \
- 'ssh_port' : self.ssh_port, \
- 'ssh_pass' : self.ssh_pass}
- # no matter what happens during repair, go on to try to reverify
- try:
- exec(preamble + repair, namespace, namespace)
- except Exception, exc:
- print 'Exception occured during repair'
- traceback.print_exc()
- self.verify()
+ def repair(self):
+ if not self.machines:
+ raise error.AutoservError(
+ 'No machines specified to repair')
+ namespace = {'machines' : self.machines, 'job' : self, \
+ 'ssh_user' : self.ssh_user, \
+ 'ssh_port' : self.ssh_port, \
+ 'ssh_pass' : self.ssh_pass}
+ # no matter what happens during repair, go on to try to reverify
+ try:
+ exec(preamble + repair, namespace, namespace)
+ except Exception, exc:
+ print 'Exception occured during repair'
+ traceback.print_exc()
+ self.verify()
- def enable_external_logging(self):
- """Start or restart external logging mechanism.
- """
- pass
+ def enable_external_logging(self):
+ """Start or restart external logging mechanism.
+ """
+ pass
- def disable_external_logging(self):
- """ Pause or stop external logging mechanism.
- """
- pass
+ def disable_external_logging(self):
+ """ Pause or stop external logging mechanism.
+ """
+ pass
- def use_external_logging(self):
- """Return True if external logging should be used.
- """
- return False
+ def use_external_logging(self):
+ """Return True if external logging should be used.
+ """
+ return False
- def parallel_simple(self, function, machines, log=True, timeout=None):
- """Run 'function' using parallel_simple, with an extra
- wrapper to handle the necessary setup for continuous parsing,
- if possible. If continuous parsing is already properly
- initialized then this should just work."""
- is_forking = not (len(machines) == 1 and
- self.machines == machines)
- if self.parse_job and is_forking:
- def wrapper(machine):
- self.parse_job += "/" + machine
- self.using_parser = True
- self.machines = [machine]
- self.resultdir = os.path.join(self.resultdir,
- machine)
- self.init_parser(self.resultdir)
- result = function(machine)
- self.cleanup_parser()
- return result
- else:
- wrapper = function
- subcommand.parallel_simple(wrapper, machines, log, timeout)
+ def parallel_simple(self, function, machines, log=True, timeout=None):
+ """Run 'function' using parallel_simple, with an extra
+ wrapper to handle the necessary setup for continuous parsing,
+ if possible. If continuous parsing is already properly
+ initialized then this should just work."""
+ is_forking = not (len(machines) == 1 and
+ self.machines == machines)
+ if self.parse_job and is_forking:
+ def wrapper(machine):
+ self.parse_job += "/" + machine
+ self.using_parser = True
+ self.machines = [machine]
+ self.resultdir = os.path.join(self.resultdir,
+ machine)
+ self.init_parser(self.resultdir)
+ result = function(machine)
+ self.cleanup_parser()
+ return result
+ else:
+ wrapper = function
+ subcommand.parallel_simple(wrapper, machines, log, timeout)
- def run(self, reboot = False, install_before = False,
- install_after = False, collect_crashdumps = True,
- namespace = {}):
- # use a copy so changes don't affect the original dictionary
- namespace = namespace.copy()
- machines = self.machines
+ def run(self, reboot = False, install_before = False,
+ install_after = False, collect_crashdumps = True,
+ namespace = {}):
+ # use a copy so changes don't affect the original dictionary
+ namespace = namespace.copy()
+ machines = self.machines
- self.aborted = False
- namespace['machines'] = machines
- namespace['args'] = self.args
- namespace['job'] = self
- namespace['ssh_user'] = self.ssh_user
- namespace['ssh_port'] = self.ssh_port
- namespace['ssh_pass'] = self.ssh_pass
- test_start_time = int(time.time())
+ self.aborted = False
+ namespace['machines'] = machines
+ namespace['args'] = self.args
+ namespace['job'] = self
+ namespace['ssh_user'] = self.ssh_user
+ namespace['ssh_port'] = self.ssh_port
+ namespace['ssh_pass'] = self.ssh_pass
+ test_start_time = int(time.time())
- os.chdir(self.resultdir)
-
- self.enable_external_logging()
- status_log = os.path.join(self.resultdir, 'status.log')
- try:
- if install_before and machines:
- exec(preamble + install, namespace, namespace)
- if self.client:
- namespace['control'] = self.control
- open('control', 'w').write(self.control)
- open('control.srv', 'w').write(client_wrapper)
- server_control = client_wrapper
- else:
- open('control.srv', 'w').write(self.control)
- server_control = self.control
- exec(preamble + server_control, namespace, namespace)
+ os.chdir(self.resultdir)
- finally:
- if machines and collect_crashdumps:
- namespace['test_start_time'] = test_start_time
- exec(preamble + crashdumps,
- namespace, namespace)
- self.disable_external_logging()
- if reboot and machines:
- exec(preamble + reboot_segment,
- namespace, namespace)
- if install_after and machines:
- exec(preamble + install, namespace, namespace)
+ self.enable_external_logging()
+ status_log = os.path.join(self.resultdir, 'status.log')
+ try:
+ if install_before and machines:
+ exec(preamble + install, namespace, namespace)
+ if self.client:
+ namespace['control'] = self.control
+ open('control', 'w').write(self.control)
+ open('control.srv', 'w').write(client_wrapper)
+ server_control = client_wrapper
+ else:
+ open('control.srv', 'w').write(self.control)
+ server_control = self.control
+ exec(preamble + server_control, namespace, namespace)
+ finally:
+ if machines and collect_crashdumps:
+ namespace['test_start_time'] = test_start_time
+ exec(preamble + crashdumps,
+ namespace, namespace)
+ self.disable_external_logging()
+ if reboot and machines:
+ exec(preamble + reboot_segment,
+ namespace, namespace)
+ if install_after and machines:
+ exec(preamble + install, namespace, namespace)
- def run_test(self, url, *args, **dargs):
- """Summon a test object and run it.
-
- tag
- tag to add to testname
- url
- url of the test to run
- """
- (group, testname) = test.testname(url)
- tag = None
- subdir = testname
+ def run_test(self, url, *args, **dargs):
+ """Summon a test object and run it.
- if dargs.has_key('tag'):
- tag = dargs['tag']
- del dargs['tag']
- if tag:
- subdir += '.' + tag
+ tag
+ tag to add to testname
+ url
+ url of the test to run
+ """
+
+ (group, testname) = test.testname(url)
+ tag = None
+ subdir = testname
+
+ if dargs.has_key('tag'):
+ tag = dargs['tag']
+ del dargs['tag']
+ if tag:
+ subdir += '.' + tag
- outputdir = os.path.join(self.resultdir, subdir)
- if os.path.exists(outputdir):
- msg = ("%s already exists, test <%s> may have"
- " already run with tag <%s>"
- % (outputdir, testname, tag) )
- raise error.TestError(msg)
- os.mkdir(outputdir)
+ outputdir = os.path.join(self.resultdir, subdir)
+ if os.path.exists(outputdir):
+ msg = ("%s already exists, test <%s> may have"
+ " already run with tag <%s>"
+ % (outputdir, testname, tag) )
+ raise error.TestError(msg)
+ os.mkdir(outputdir)
- try:
- test.runtest(self, url, tag, args, dargs)
- self.record('GOOD', subdir, testname, 'completed successfully')
- except error.TestNAError, detail:
- self.record('TEST_NA', subdir, testname, str(detail))
- except Exception, detail:
- info = str(detail) + "\n" + traceback.format_exc()
- self.record('FAIL', subdir, testname, info)
+ try:
+ test.runtest(self, url, tag, args, dargs)
+ self.record('GOOD', subdir, testname, 'completed successfully')
+ except error.TestNAError, detail:
+ self.record('TEST_NA', subdir, testname, str(detail))
+ except Exception, detail:
+ info = str(detail) + "\n" + traceback.format_exc()
+ self.record('FAIL', subdir, testname, info)
- def run_group(self, function, *args, **dargs):
- """\
- function:
- subroutine to run
- *args:
- arguments for the function
- """
+ def run_group(self, function, *args, **dargs):
+ """\
+ function:
+ subroutine to run
+ *args:
+ arguments for the function
+ """
- result = None
- name = function.__name__
+ result = None
+ name = function.__name__
- # Allow the tag for the group to be specified.
- if dargs.has_key('tag'):
- tag = dargs['tag']
- del dargs['tag']
- if tag:
- name = tag
+ # Allow the tag for the group to be specified.
+ if dargs.has_key('tag'):
+ tag = dargs['tag']
+ del dargs['tag']
+ if tag:
+ name = tag
- old_record_prefix = self.record_prefix
- try:
- try:
- self.record('START', None, name)
- self.record_prefix += '\t'
- result = function(*args, **dargs)
- except Exception, e:
- self.record_prefix = old_record_prefix
- err_msg = str(e) + '\n'
- err_msg += traceback.format_exc()
- self.record('END FAIL', None, name, err_msg)
- else:
- self.record_prefix = old_record_prefix
- self.record('END GOOD', None, name)
+ old_record_prefix = self.record_prefix
+ try:
+ try:
+ self.record('START', None, name)
+ self.record_prefix += '\t'
+ result = function(*args, **dargs)
+ except Exception, e:
+ self.record_prefix = old_record_prefix
+ err_msg = str(e) + '\n'
+ err_msg += traceback.format_exc()
+ self.record('END FAIL', None, name, err_msg)
+ else:
+ self.record_prefix = old_record_prefix
+ self.record('END GOOD', None, name)
- # We don't want to raise up an error higher if it's just
- # a TestError - we want to carry on to other tests. Hence
- # this outer try/except block.
- except error.TestError:
- pass
- except:
- raise error.TestError(name + ' failed\n' +
- traceback.format_exc())
+ # We don't want to raise up an error higher if it's just
+ # a TestError - we want to carry on to other tests. Hence
+ # this outer try/except block.
+ except error.TestError:
+ pass
+ except:
+ raise error.TestError(name + ' failed\n' +
+ traceback.format_exc())
- return result
+ return result
- def run_reboot(self, reboot_func, get_kernel_func):
- """\
- A specialization of run_group meant specifically for handling
- a reboot. Includes support for capturing the kernel version
- after the reboot.
+ def run_reboot(self, reboot_func, get_kernel_func):
+ """\
+ A specialization of run_group meant specifically for handling
+ a reboot. Includes support for capturing the kernel version
+ after the reboot.
- reboot_func: a function that carries out the reboot
+ reboot_func: a function that carries out the reboot
- get_kernel_func: a function that returns a string
- representing the kernel version.
- """
+ get_kernel_func: a function that returns a string
+ representing the kernel version.
+ """
- old_record_prefix = self.record_prefix
- try:
- self.record('START', None, 'reboot')
- self.record_prefix += '\t'
- reboot_func()
- except Exception, e:
- self.record_prefix = old_record_prefix
- err_msg = str(e) + '\n' + traceback.format_exc()
- self.record('END FAIL', None, 'reboot', err_msg)
- else:
- kernel = get_kernel_func()
- self.record_prefix = old_record_prefix
- self.record('END GOOD', None, 'reboot',
- optional_fields={"kernel": kernel})
+ old_record_prefix = self.record_prefix
+ try:
+ self.record('START', None, 'reboot')
+ self.record_prefix += '\t'
+ reboot_func()
+ except Exception, e:
+ self.record_prefix = old_record_prefix
+ err_msg = str(e) + '\n' + traceback.format_exc()
+ self.record('END FAIL', None, 'reboot', err_msg)
+ else:
+ kernel = get_kernel_func()
+ self.record_prefix = old_record_prefix
+ self.record('END GOOD', None, 'reboot',
+ optional_fields={"kernel": kernel})
- def record(self, status_code, subdir, operation, status='',
- optional_fields=None):
- """
- Record job-level status
+ def record(self, status_code, subdir, operation, status='',
+ optional_fields=None):
+ """
+ Record job-level status
- The intent is to make this file both machine parseable and
- human readable. That involves a little more complexity, but
- really isn't all that bad ;-)
+ The intent is to make this file both machine parseable and
+ human readable. That involves a little more complexity, but
+ really isn't all that bad ;-)
- Format is <status code>\t<subdir>\t<operation>\t<status>
+ Format is <status code>\t<subdir>\t<operation>\t<status>
- status code: see common_lib.logging.is_valid_status()
- for valid status definition
+ status code: see common_lib.logging.is_valid_status()
+ for valid status definition
- subdir: MUST be a relevant subdirectory in the results,
- or None, which will be represented as '----'
+ subdir: MUST be a relevant subdirectory in the results,
+ or None, which will be represented as '----'
- operation: description of what you ran (e.g. "dbench", or
- "mkfs -t foobar /dev/sda9")
+ operation: description of what you ran (e.g. "dbench", or
+ "mkfs -t foobar /dev/sda9")
- status: error message or "completed sucessfully"
+ status: error message or "completed sucessfully"
- ------------------------------------------------------------
+ ------------------------------------------------------------
- Initial tabs indicate indent levels for grouping, and is
- governed by self.record_prefix
+ Initial tabs indicate indent levels for grouping, and is
+ governed by self.record_prefix
- multiline messages have secondary lines prefaced by a double
- space (' ')
+ multiline messages have secondary lines prefaced by a double
+ space (' ')
- Executing this method will trigger the logging of all new
- warnings to date from the various console loggers.
- """
- # poll all our warning loggers for new warnings
- warnings = self._read_warnings()
- for timestamp, msg in warnings:
- self.__record("WARN", None, None, msg, timestamp)
+ Executing this method will trigger the logging of all new
+ warnings to date from the various console loggers.
+ """
+ # poll all our warning loggers for new warnings
+ warnings = self._read_warnings()
+ for timestamp, msg in warnings:
+ self.__record("WARN", None, None, msg, timestamp)
- # write out the actual status log line
- self.__record(status_code, subdir, operation, status,
- optional_fields=optional_fields)
+ # write out the actual status log line
+ self.__record(status_code, subdir, operation, status,
+ optional_fields=optional_fields)
- def _read_warnings(self):
- warnings = []
- while True:
- # pull in a line of output from every logger that has
- # output ready to be read
- loggers, _, _ = select.select(self.warning_loggers,
- [], [], 0)
- closed_loggers = set()
- for logger in loggers:
- line = logger.readline()
- # record any broken pipes (aka line == empty)
- if len(line) == 0:
- closed_loggers.add(logger)
- continue
- timestamp, msg = line.split('\t', 1)
- warnings.append((int(timestamp), msg.strip()))
+ def _read_warnings(self):
+ warnings = []
+ while True:
+ # pull in a line of output from every logger that has
+ # output ready to be read
+ loggers, _, _ = select.select(self.warning_loggers,
+ [], [], 0)
+ closed_loggers = set()
+ for logger in loggers:
+ line = logger.readline()
+ # record any broken pipes (aka line == empty)
+ if len(line) == 0:
+ closed_loggers.add(logger)
+ continue
+ timestamp, msg = line.split('\t', 1)
+ warnings.append((int(timestamp), msg.strip()))
- # stop listening to loggers that are closed
- self.warning_loggers -= closed_loggers
+ # stop listening to loggers that are closed
+ self.warning_loggers -= closed_loggers
- # stop if none of the loggers have any output left
- if not loggers:
- break
+ # stop if none of the loggers have any output left
+ if not loggers:
+ break
- # sort into timestamp order
- warnings.sort()
- return warnings
+ # sort into timestamp order
+ warnings.sort()
+ return warnings
- def _render_record(self, status_code, subdir, operation, status='',
- epoch_time=None, record_prefix=None,
- optional_fields=None):
- """
- Internal Function to generate a record to be written into a
- status log. For use by server_job.* classes only.
- """
- if subdir:
- if re.match(r'[\n\t]', subdir):
- raise ValueError(
- 'Invalid character in subdir string')
- substr = subdir
- else:
- substr = '----'
+ def _render_record(self, status_code, subdir, operation, status='',
+ epoch_time=None, record_prefix=None,
+ optional_fields=None):
+ """
+ Internal Function to generate a record to be written into a
+ status log. For use by server_job.* classes only.
+ """
+ if subdir:
+ if re.match(r'[\n\t]', subdir):
+ raise ValueError(
+ 'Invalid character in subdir string')
+ substr = subdir
+ else:
+ substr = '----'
- if not logging.is_valid_status(status_code):
- raise ValueError('Invalid status code supplied: %s' %
- status_code)
- if not operation:
- operation = '----'
- if re.match(r'[\n\t]', operation):
- raise ValueError(
- 'Invalid character in operation string')
- operation = operation.rstrip()
- status = status.rstrip()
- status = re.sub(r"\t", " ", status)
- # Ensure any continuation lines are marked so we can
- # detect them in the status file to ensure it is parsable.
- status = re.sub(r"\n", "\n" + self.record_prefix + " ", status)
+ if not logging.is_valid_status(status_code):
+ raise ValueError('Invalid status code supplied: %s' %
+ status_code)
+ if not operation:
+ operation = '----'
+ if re.match(r'[\n\t]', operation):
+ raise ValueError(
+ 'Invalid character in operation string')
+ operation = operation.rstrip()
+ status = status.rstrip()
+ status = re.sub(r"\t", " ", status)
+ # Ensure any continuation lines are marked so we can
+ # detect them in the status file to ensure it is parsable.
+ status = re.sub(r"\n", "\n" + self.record_prefix + " ", status)
- if not optional_fields:
- optional_fields = {}
+ if not optional_fields:
+ optional_fields = {}
- # Generate timestamps for inclusion in the logs
- if epoch_time is None:
- epoch_time = int(time.time())
- local_time = time.localtime(epoch_time)
- optional_fields["timestamp"] = str(epoch_time)
- optional_fields["localtime"] = time.strftime("%b %d %H:%M:%S",
- local_time)
+ # Generate timestamps for inclusion in the logs
+ if epoch_time is None:
+ epoch_time = int(time.time())
+ local_time = time.localtime(epoch_time)
+ optional_fields["timestamp"] = str(epoch_time)
+ optional_fields["localtime"] = time.strftime("%b %d %H:%M:%S",
+ local_time)
- fields = [status_code, substr, operation]
- fields += ["%s=%s" % x for x in optional_fields.iteritems()]
- fields.append(status)
+ fields = [status_code, substr, operation]
+ fields += ["%s=%s" % x for x in optional_fields.iteritems()]
+ fields.append(status)
- if record_prefix is None:
- record_prefix = self.record_prefix
+ if record_prefix is None:
+ record_prefix = self.record_prefix
- msg = '\t'.join(str(x) for x in fields)
+ msg = '\t'.join(str(x) for x in fields)
- return record_prefix + msg + '\n'
+ return record_prefix + msg + '\n'
- def _record_prerendered(self, msg):
- """
- Record a pre-rendered msg into the status logs. The only
- change this makes to the message is to add on the local
- indentation. Should not be called outside of server_job.*
- classes. Unlike __record, this does not write the message
- to standard output.
- """
- lines = []
- status_file = os.path.join(self.resultdir, 'status.log')
- status_log = open(status_file, 'a')
- for line in msg.splitlines():
- line = self.record_prefix + line + '\n'
- lines.append(line)
- status_log.write(line)
- status_log.close()
- self.__parse_status(lines)
+ def _record_prerendered(self, msg):
+ """
+ Record a pre-rendered msg into the status logs. The only
+ change this makes to the message is to add on the local
+ indentation. Should not be called outside of server_job.*
+ classes. Unlike __record, this does not write the message
+ to standard output.
+ """
+ lines = []
+ status_file = os.path.join(self.resultdir, 'status.log')
+ status_log = open(status_file, 'a')
+ for line in msg.splitlines():
+ line = self.record_prefix + line + '\n'
+ lines.append(line)
+ status_log.write(line)
+ status_log.close()
+ self.__parse_status(lines)
- def __record(self, status_code, subdir, operation, status='',
- epoch_time=None, optional_fields=None):
- """
- Actual function for recording a single line into the status
- logs. Should never be called directly, only by job.record as
- this would bypass the console monitor logging.
- """
+ def __record(self, status_code, subdir, operation, status='',
+ epoch_time=None, optional_fields=None):
+ """
+ Actual function for recording a single line into the status
+ logs. Should never be called directly, only by job.record as
+ this would bypass the console monitor logging.
+ """
- msg = self._render_record(status_code, subdir, operation,
- status, epoch_time,
- optional_fields=optional_fields)
+ msg = self._render_record(status_code, subdir, operation,
+ status, epoch_time,
+ optional_fields=optional_fields)
- status_file = os.path.join(self.resultdir, 'status.log')
- sys.stdout.write(msg)
- open(status_file, "a").write(msg)
- if subdir:
- test_dir = os.path.join(self.resultdir, subdir)
- status_file = os.path.join(test_dir, 'status')
- open(status_file, "a").write(msg)
- self.__parse_status(msg.splitlines())
+ status_file = os.path.join(self.resultdir, 'status.log')
+ sys.stdout.write(msg)
+ open(status_file, "a").write(msg)
+ if subdir:
+ test_dir = os.path.join(self.resultdir, subdir)
+ status_file = os.path.join(test_dir, 'status')
+ open(status_file, "a").write(msg)
+ self.__parse_status(msg.splitlines())
- def __parse_status(self, new_lines):
- if not self.using_parser:
- return
- new_tests = self.parser.process_lines(new_lines)
- for test in new_tests:
- self.__insert_test(test)
+ def __parse_status(self, new_lines):
+ if not self.using_parser:
+ return
+ new_tests = self.parser.process_lines(new_lines)
+ for test in new_tests:
+ self.__insert_test(test)
- def __insert_test(self, test):
- """ An internal method to insert a new test result into the
- database. This method will not raise an exception, even if an
- error occurs during the insert, to avoid failing a test
- simply because of unexpected database issues."""
- try:
- self.results_db.insert_test(self.job_model, test)
- except Exception:
- msg = ("WARNING: An unexpected error occured while "
- "inserting test results into the database. "
- "Ignoring error.\n" + traceback.format_exc())
- print >> sys.stderr, msg
+ def __insert_test(self, test):
+ """ An internal method to insert a new test result into the
+ database. This method will not raise an exception, even if an
+ error occurs during the insert, to avoid failing a test
+ simply because of unexpected database issues."""
+ try:
+ self.results_db.insert_test(self.job_model, test)
+ except Exception:
+ msg = ("WARNING: An unexpected error occured while "
+ "inserting test results into the database. "
+ "Ignoring error.\n" + traceback.format_exc())
+ print >> sys.stderr, msg
# a file-like object for catching stderr from an autotest client and
# extracting status logs from it
class client_logger(object):
- """Partial file object to write to both stdout and
- the status log file. We only implement those methods
- utils.run() actually calls.
- """
- parser = re.compile(r"^AUTOTEST_STATUS:([^:]*):(.*)$")
- extract_indent = re.compile(r"^(\t*).*$")
+ """Partial file object to write to both stdout and
+ the status log file. We only implement those methods
+ utils.run() actually calls.
+ """
+ parser = re.compile(r"^AUTOTEST_STATUS:([^:]*):(.*)$")
+ extract_indent = re.compile(r"^(\t*).*$")
- def __init__(self, job):
- self.job = job
- self.leftover = ""
- self.last_line = ""
- self.logs = {}
+ def __init__(self, job):
+ self.job = job
+ self.leftover = ""
+ self.last_line = ""
+ self.logs = {}
- def _process_log_dict(self, log_dict):
- log_list = log_dict.pop("logs", [])
- for key in sorted(log_dict.iterkeys()):
- log_list += self._process_log_dict(log_dict.pop(key))
- return log_list
+ def _process_log_dict(self, log_dict):
+ log_list = log_dict.pop("logs", [])
+ for key in sorted(log_dict.iterkeys()):
+ log_list += self._process_log_dict(log_dict.pop(key))
+ return log_list
- def _process_logs(self):
- """Go through the accumulated logs in self.log and print them
- out to stdout and the status log. Note that this processes
- logs in an ordering where:
+ def _process_logs(self):
+ """Go through the accumulated logs in self.log and print them
+ out to stdout and the status log. Note that this processes
+ logs in an ordering where:
- 1) logs to different tags are never interleaved
- 2) logs to x.y come before logs to x.y.z for all z
- 3) logs to x.y come before x.z whenever y < z
+ 1) logs to different tags are never interleaved
+ 2) logs to x.y come before logs to x.y.z for all z
+ 3) logs to x.y come before x.z whenever y < z
- Note that this will in general not be the same as the
- chronological ordering of the logs. However, if a chronological
- ordering is desired that one can be reconstructed from the
- status log by looking at timestamp lines."""
- log_list = self._process_log_dict(self.logs)
- for line in log_list:
- self.job._record_prerendered(line + '\n')
- if log_list:
- self.last_line = log_list[-1]
+ Note that this will in general not be the same as the
+ chronological ordering of the logs. However, if a chronological
+ ordering is desired that one can be reconstructed from the
+ status log by looking at timestamp lines."""
+ log_list = self._process_log_dict(self.logs)
+ for line in log_list:
+ self.job._record_prerendered(line + '\n')
+ if log_list:
+ self.last_line = log_list[-1]
- def _process_quoted_line(self, tag, line):
- """Process a line quoted with an AUTOTEST_STATUS flag. If the
- tag is blank then we want to push out all the data we've been
- building up in self.logs, and then the newest line. If the
- tag is not blank, then push the line into the logs for handling
- later."""
- print line
- if tag == "":
- self._process_logs()
- self.job._record_prerendered(line + '\n')
- self.last_line = line
- else:
- tag_parts = [int(x) for x in tag.split(".")]
- log_dict = self.logs
- for part in tag_parts:
- log_dict = log_dict.setdefault(part, {})
- log_list = log_dict.setdefault("logs", [])
- log_list.append(line)
+ def _process_quoted_line(self, tag, line):
+ """Process a line quoted with an AUTOTEST_STATUS flag. If the
+ tag is blank then we want to push out all the data we've been
+ building up in self.logs, and then the newest line. If the
+ tag is not blank, then push the line into the logs for handling
+ later."""
+ print line
+ if tag == "":
+ self._process_logs()
+ self.job._record_prerendered(line + '\n')
+ self.last_line = line
+ else:
+ tag_parts = [int(x) for x in tag.split(".")]
+ log_dict = self.logs
+ for part in tag_parts:
+ log_dict = log_dict.setdefault(part, {})
+ log_list = log_dict.setdefault("logs", [])
+ log_list.append(line)
- def _process_line(self, line):
- """Write out a line of data to the appropriate stream. Status
- lines sent by autotest will be prepended with
- "AUTOTEST_STATUS", and all other lines are ssh error
- messages."""
- match = self.parser.search(line)
- if match:
- tag, line = match.groups()
- self._process_quoted_line(tag, line)
- else:
- print line
+ def _process_line(self, line):
+ """Write out a line of data to the appropriate stream. Status
+ lines sent by autotest will be prepended with
+ "AUTOTEST_STATUS", and all other lines are ssh error
+ messages."""
+ match = self.parser.search(line)
+ if match:
+ tag, line = match.groups()
+ self._process_quoted_line(tag, line)
+ else:
+ print line
- def _format_warnings(self, last_line, warnings):
- # use the indentation of whatever the last log line was
- indent = self.extract_indent.match(last_line).group(1)
- # if the last line starts a new group, add an extra indent
- if last_line.lstrip('\t').startswith("START\t"):
- indent += '\t'
- return [self.job._render_record("WARN", None, None, msg,
- timestamp, indent).rstrip('\n')
- for timestamp, msg in warnings]
+ def _format_warnings(self, last_line, warnings):
+ # use the indentation of whatever the last log line was
+ indent = self.extract_indent.match(last_line).group(1)
+ # if the last line starts a new group, add an extra indent
+ if last_line.lstrip('\t').startswith("START\t"):
+ indent += '\t'
+ return [self.job._render_record("WARN", None, None, msg,
+ timestamp, indent).rstrip('\n')
+ for timestamp, msg in warnings]
- def _process_warnings(self, last_line, log_dict, warnings):
- if log_dict.keys() in ([], ["logs"]):
- # there are no sub-jobs, just append the warnings here
- warnings = self._format_warnings(last_line, warnings)
- log_list = log_dict.setdefault("logs", [])
- log_list += warnings
- for warning in warnings:
- sys.stdout.write(warning + '\n')
- else:
- # there are sub-jobs, so put the warnings in there
- log_list = log_dict.get("logs", [])
- if log_list:
- last_line = log_list[-1]
- for key in sorted(log_dict.iterkeys()):
- if key != "logs":
- self._process_warnings(last_line,
- log_dict[key],
- warnings)
+ def _process_warnings(self, last_line, log_dict, warnings):
+ if log_dict.keys() in ([], ["logs"]):
+ # there are no sub-jobs, just append the warnings here
+ warnings = self._format_warnings(last_line, warnings)
+ log_list = log_dict.setdefault("logs", [])
+ log_list += warnings
+ for warning in warnings:
+ sys.stdout.write(warning + '\n')
+ else:
+ # there are sub-jobs, so put the warnings in there
+ log_list = log_dict.get("logs", [])
+ if log_list:
+ last_line = log_list[-1]
+ for key in sorted(log_dict.iterkeys()):
+ if key != "logs":
+ self._process_warnings(last_line,
+ log_dict[key],
+ warnings)
- def write(self, data):
- # first check for any new console warnings
- warnings = self.job._read_warnings()
- self._process_warnings(self.last_line, self.logs, warnings)
- # now process the newest data written out
- data = self.leftover + data
- lines = data.split("\n")
- # process every line but the last one
- for line in lines[:-1]:
- self._process_line(line)
- # save the last line for later processing
- # since we may not have the whole line yet
- self.leftover = lines[-1]
+ def write(self, data):
+ # first check for any new console warnings
+ warnings = self.job._read_warnings()
+ self._process_warnings(self.last_line, self.logs, warnings)
+ # now process the newest data written out
+ data = self.leftover + data
+ lines = data.split("\n")
+ # process every line but the last one
+ for line in lines[:-1]:
+ self._process_line(line)
+ # save the last line for later processing
+ # since we may not have the whole line yet
+ self.leftover = lines[-1]
- def flush(self):
- sys.stdout.flush()
+ def flush(self):
+ sys.stdout.flush()
- def close(self):
- if self.leftover:
- self._process_line(self.leftover)
- self._process_logs()
- self.flush()
+ def close(self):
+ if self.leftover:
+ self._process_line(self.leftover)
+ self._process_logs()
+ self.flush()
# site_server_job.py may be non-existant or empty, make sure that an
# appropriate site_server_job class is created nevertheless
try:
- from autotest_lib.server.site_server_job import site_server_job
+ from autotest_lib.server.site_server_job import site_server_job
except ImportError:
- class site_server_job(base_server_job):
- pass
-
+ class site_server_job(base_server_job):
+ pass
+
class server_job(site_server_job):
- pass
+ pass
diff --git a/server/site_autoserv_parser.py b/server/site_autoserv_parser.py
index f8703f7..03b8c4e 100644
--- a/server/site_autoserv_parser.py
+++ b/server/site_autoserv_parser.py
@@ -9,6 +9,6 @@
class site_autoserv_parser(base_autoserv_parser):
- def get_usage(self):
- usage = super(site_autoserv_parser, self).get_usage()
- return usage+add_usage
+ def get_usage(self):
+ usage = super(site_autoserv_parser, self).get_usage()
+ return usage+add_usage
diff --git a/server/source_kernel.py b/server/source_kernel.py
index 8d27d11..ae7f032 100644
--- a/server/source_kernel.py
+++ b/server/source_kernel.py
@@ -5,7 +5,7 @@
"""
This module defines the SourceKernel class
- SourceKernel: an linux kernel built from source
+ SourceKernel: an linux kernel built from source
"""
__author__ = """
@@ -21,60 +21,60 @@
class SourceKernel(kernel.Kernel):
- """
- This class represents a linux kernel built from source.
-
- It is used to obtain a built kernel or create one from source and
- install it on a Host.
-
- Implementation details:
- This is a leaf class in an abstract class hierarchy, it must
- implement the unimplemented methods in parent classes.
- """
- def __init__(self, k):
- super(kernel.Kernel, self).__init__()
- self.__kernel = k
- self.__patch_list = []
- self.__config_file = None
- self.__autotest = autotest.Autotest()
+ """
+ This class represents a linux kernel built from source.
+
+ It is used to obtain a built kernel or create one from source and
+ install it on a Host.
+
+ Implementation details:
+ This is a leaf class in an abstract class hierarchy, it must
+ implement the unimplemented methods in parent classes.
+ """
+ def __init__(self, k):
+ super(kernel.Kernel, self).__init__()
+ self.__kernel = k
+ self.__patch_list = []
+ self.__config_file = None
+ self.__autotest = autotest.Autotest()
- def configure(self, configFile):
- self.__config_file = configFile
+ def configure(self, configFile):
+ self.__config_file = configFile
- def patch(self, patchFile):
- self.__patch_list.append(patchFile)
+ def patch(self, patchFile):
+ self.__patch_list.append(patchFile)
- def build(self, host):
- ctlfile = self.__control_file(self.__kernel, self.__patch_list,
- self.__config_file)
- self.__autotest.run(ctlfile, host.get_tmp_dir(), host)
+ def build(self, host):
+ ctlfile = self.__control_file(self.__kernel, self.__patch_list,
+ self.__config_file)
+ self.__autotest.run(ctlfile, host.get_tmp_dir(), host)
- def install(self, host):
- self.__autotest.install(host)
- ctlfile = ("testkernel = job.kernel('%s')\n"
- "testkernel.install()\n"
- "testkernel.add_to_bootloader()\n" %(self.__kernel))
- self.__autotest.run(ctlfile, host.get_tmp_dir(), host)
-
+ def install(self, host):
+ self.__autotest.install(host)
+ ctlfile = ("testkernel = job.kernel('%s')\n"
+ "testkernel.install()\n"
+ "testkernel.add_to_bootloader()\n" %(self.__kernel))
+ self.__autotest.run(ctlfile, host.get_tmp_dir(), host)
- def __control_file(self, kernel, patch_list, config):
- ctl = ("testkernel = job.kernel('%s')\n" % kernel)
- if len(patch_list):
- patches = ', '.join(["'%s'" % x for x in patch_list])
- ctl += "testkernel.patch(%s)\n" % patches
+ def __control_file(self, kernel, patch_list, config):
+ ctl = ("testkernel = job.kernel('%s')\n" % kernel)
- if config:
- ctl += "testkernel.config('%s')\n" % config
- else:
- ctl += "testkernel.config('', None, True)\n"
+ if len(patch_list):
+ patches = ', '.join(["'%s'" % x for x in patch_list])
+ ctl += "testkernel.patch(%s)\n" % patches
- ctl += "testkernel.build()\n"
+ if config:
+ ctl += "testkernel.config('%s')\n" % config
+ else:
+ ctl += "testkernel.config('', None, True)\n"
- # copy back to server
+ ctl += "testkernel.build()\n"
- return ctl
+ # copy back to server
+
+ return ctl
diff --git a/server/standalone_profiler.py b/server/standalone_profiler.py
index 81bf769..f681ff4 100644
--- a/server/standalone_profiler.py
+++ b/server/standalone_profiler.py
@@ -15,35 +15,35 @@
def generate_test(machines, hostname, profilers, timeout_start, timeout_stop,
- timeout_sync=180):
- control_file = []
- for profiler in profilers:
- control_file.append("job.profilers.add(%s)"
- % str(profiler)[1:-1]) # Remove parens
+ timeout_sync=180):
+ control_file = []
+ for profiler in profilers:
+ control_file.append("job.profilers.add(%s)"
+ % str(profiler)[1:-1]) # Remove parens
- control_file.append("job.run_test('barriertest',%d,%d,%d,'%s','%s',%s)"
- % (timeout_sync, timeout_start, timeout_stop,
- hostname, "PROF_MASTER", str(machines)))
+ control_file.append("job.run_test('barriertest',%d,%d,%d,'%s','%s',%s)"
+ % (timeout_sync, timeout_start, timeout_stop,
+ hostname, "PROF_MASTER", str(machines)))
- for profiler in profilers:
- control_file.append("job.profilers.delete('%s')" % profiler[0])
+ for profiler in profilers:
+ control_file.append("job.profilers.delete('%s')" % profiler[0])
- return "\n".join(control_file)
+ return "\n".join(control_file)
def wait_for_profilers(machines, timeout = 300):
- sb = barrier.barrier("PROF_MASTER", "sync_profilers",
- timeout, port=63100)
- sb.rendevous_servers("PROF_MASTER", *machines)
+ sb = barrier.barrier("PROF_MASTER", "sync_profilers",
+ timeout, port=63100)
+ sb.rendevous_servers("PROF_MASTER", *machines)
def start_profilers(machines, timeout = 120):
- sb = barrier.barrier("PROF_MASTER", "start_profilers",
- timeout, port=63100)
- sb.rendevous_servers("PROF_MASTER", *machines)
+ sb = barrier.barrier("PROF_MASTER", "start_profilers",
+ timeout, port=63100)
+ sb.rendevous_servers("PROF_MASTER", *machines)
def stop_profilers(machines, timeout = 120):
- sb = barrier.barrier("PROF_MASTER", "stop_profilers",
- timeout, port=63100)
- sb.rendevous_servers("PROF_MASTER", *machines)
+ sb = barrier.barrier("PROF_MASTER", "stop_profilers",
+ timeout, port=63100)
+ sb.rendevous_servers("PROF_MASTER", *machines)
diff --git a/server/status.py b/server/status.py
index 382626a..b0f4408 100644
--- a/server/status.py
+++ b/server/status.py
@@ -3,194 +3,194 @@
class Machine:
- """
- Represents the current state of a machine. Possible values are:
- TESTING currently running a test
- REBOOTING currently rebooting
- BROKEN busted somehow (e.g. reboot timed out)
- OTHER none of the above
+ """
+ Represents the current state of a machine. Possible values are:
+ TESTING currently running a test
+ REBOOTING currently rebooting
+ BROKEN busted somehow (e.g. reboot timed out)
+ OTHER none of the above
- The implementation is basically that of a state machine. From an
- external point of view the only relevant attributes are:
- details text description of the current status
- test_count number of tests run
- """
- def __init__(self):
- self.state = "OTHER"
- self.details = "Running"
- self.test_name = ""
- self.test_count = 0
+ The implementation is basically that of a state machine. From an
+ external point of view the only relevant attributes are:
+ details text description of the current status
+ test_count number of tests run
+ """
+ def __init__(self):
+ self.state = "OTHER"
+ self.details = "Running"
+ self.test_name = ""
+ self.test_count = 0
- def process_line(self, line):
- self.handlers[self.state](self, line)
+ def process_line(self, line):
+ self.handlers[self.state](self, line)
- def _OTHER_handler(self, line):
- match = self.job_start.match(line)
- if match and match.group(2) != "----":
- self.state = "TESTING"
- self.tab_level = len(match.group(1))
- self.test_name = match.group(2)
- self.test_status = "GOOD"
- self.details = "Running %s" % self.test_name
- return
+ def _OTHER_handler(self, line):
+ match = self.job_start.match(line)
+ if match and match.group(2) != "----":
+ self.state = "TESTING"
+ self.tab_level = len(match.group(1))
+ self.test_name = match.group(2)
+ self.test_status = "GOOD"
+ self.details = "Running %s" % self.test_name
+ return
- match = self.reboot_start.match(line)
- if match:
- self.boot_status = match.group(1)
- if self.worse_status("GOOD", self.boot_status) == "GOOD":
- self.state = "REBOOTING"
- self.details = "Rebooting"
- else:
- self.state = "BROKEN"
- self.details = "Reboot failed - machine broken"
- return
+ match = self.reboot_start.match(line)
+ if match:
+ self.boot_status = match.group(1)
+ if self.worse_status("GOOD", self.boot_status) == "GOOD":
+ self.state = "REBOOTING"
+ self.details = "Rebooting"
+ else:
+ self.state = "BROKEN"
+ self.details = "Reboot failed - machine broken"
+ return
- def _TESTING_handler(self, line):
- match = self.job_status.match(line)
- if match:
- if len(match.group(1)) != self.tab_level + 1:
- return # we don't care about subgroups
- if self.test_name != match.group(3):
- return # we don't care about other tests
- self.test_status = self.worse_status(self.test_status,
- match.group(2))
- self.details = "Running %s: %s" % (self.test_name,
- match.group(4))
- return
+ def _TESTING_handler(self, line):
+ match = self.job_status.match(line)
+ if match:
+ if len(match.group(1)) != self.tab_level + 1:
+ return # we don't care about subgroups
+ if self.test_name != match.group(3):
+ return # we don't care about other tests
+ self.test_status = self.worse_status(self.test_status,
+ match.group(2))
+ self.details = "Running %s: %s" % (self.test_name,
+ match.group(4))
+ return
- match = self.job_end.match(line)
- if match:
- if len(match.group(1)) != self.tab_level:
- return # we don't care about subgroups
- if self.test_name != match.group(3):
- raise ValueError('Group START and END name mismatch')
- self.state = "OTHER"
- self.test_status = self.worse_status(self.test_status,
- match.group(2))
- self.test_name = ""
- del self.test_status
- self.details = "Running"
- self.test_count += 1
- return
+ match = self.job_end.match(line)
+ if match:
+ if len(match.group(1)) != self.tab_level:
+ return # we don't care about subgroups
+ if self.test_name != match.group(3):
+ raise ValueError('Group START and END name mismatch')
+ self.state = "OTHER"
+ self.test_status = self.worse_status(self.test_status,
+ match.group(2))
+ self.test_name = ""
+ del self.test_status
+ self.details = "Running"
+ self.test_count += 1
+ return
- def _REBOOTING_handler(self, line):
- match = self.reboot_done.match(line)
- if match:
- status = self.worse_status(self.boot_status,
- match.group(1))
- del self.boot_status
- if status == "GOOD":
- self.state = "OTHER"
- self.details = "Running"
- else:
- self.state = "BROKEN"
- self.details = "Reboot failed - machine broken"
- return
+ def _REBOOTING_handler(self, line):
+ match = self.reboot_done.match(line)
+ if match:
+ status = self.worse_status(self.boot_status,
+ match.group(1))
+ del self.boot_status
+ if status == "GOOD":
+ self.state = "OTHER"
+ self.details = "Running"
+ else:
+ self.state = "BROKEN"
+ self.details = "Reboot failed - machine broken"
+ return
- def _BROKEN_handler(self, line):
- pass # just do nothing - we're broken and staying broken
+ def _BROKEN_handler(self, line):
+ pass # just do nothing - we're broken and staying broken
- handlers = {"OTHER": _OTHER_handler,
- "TESTING": _TESTING_handler,
- "REBOOTING": _REBOOTING_handler,
- "BROKEN": _BROKEN_handler}
+ handlers = {"OTHER": _OTHER_handler,
+ "TESTING": _TESTING_handler,
+ "REBOOTING": _REBOOTING_handler,
+ "BROKEN": _BROKEN_handler}
- status_list = ["GOOD", "WARN", "FAIL", "ABORT", "ERROR"]
- order_dict = {None: -1}
- order_dict.update((status, i)
- for i, status in enumerate(status_list))
+ status_list = ["GOOD", "WARN", "FAIL", "ABORT", "ERROR"]
+ order_dict = {None: -1}
+ order_dict.update((status, i)
+ for i, status in enumerate(status_list))
- job_start = re.compile(r"^(\t*)START\t----\t([^\t]+).*$")
- job_status = re.compile(r"^(\t*)(%s)\t([^\t]+)\t(?:[^\t]+).*\t([^\t]+)$" %
- "|".join(status_list))
- job_end = re.compile(r"^(\t*)END (%s)\t----\t([^\t]+).*$" %
- "|".join(status_list))
- reboot_start = re.compile(r"^\t?(%s)\t[^\t]+\treboot\.start.*$" %
- "|".join(status_list))
- reboot_done = re.compile(r"^\t?(%s)\t[^\t]+\treboot\.verify.*$" %
- "|".join(status_list))
+ job_start = re.compile(r"^(\t*)START\t----\t([^\t]+).*$")
+ job_status = re.compile(r"^(\t*)(%s)\t([^\t]+)\t(?:[^\t]+).*\t([^\t]+)$" %
+ "|".join(status_list))
+ job_end = re.compile(r"^(\t*)END (%s)\t----\t([^\t]+).*$" %
+ "|".join(status_list))
+ reboot_start = re.compile(r"^\t?(%s)\t[^\t]+\treboot\.start.*$" %
+ "|".join(status_list))
+ reboot_done = re.compile(r"^\t?(%s)\t[^\t]+\treboot\.verify.*$" %
+ "|".join(status_list))
- @classmethod
- def worse_status(cls, old_status, new_status):
- if cls.order_dict[new_status] > cls.order_dict[old_status]:
- return new_status
- else:
- return old_status
+ @classmethod
+ def worse_status(cls, old_status, new_status):
+ if cls.order_dict[new_status] > cls.order_dict[old_status]:
+ return new_status
+ else:
+ return old_status
def parse_status(status_log):
- """\
- Parse the status from a single status log.
- Do not use with status logs from multi-machine tests.
- """
- parser = Machine()
- for line in file(status_log):
- parser.process_line(line)
- result = {
- "status": parser.details,
- "test_on": parser.test_name,
- "test_num_complete": parser.test_count
- }
- return result
+ """\
+ Parse the status from a single status log.
+ Do not use with status logs from multi-machine tests.
+ """
+ parser = Machine()
+ for line in file(status_log):
+ parser.process_line(line)
+ result = {
+ "status": parser.details,
+ "test_on": parser.test_name,
+ "test_num_complete": parser.test_count
+ }
+ return result
def _file_iterator(filename):
- """\
- Return an iterator over file(filename), or an empty iterator
- if the file does not exist.
- """
- if os.path.exists(filename):
- return iter(file(filename))
- else:
- return ()
+ """\
+ Return an iterator over file(filename), or an empty iterator
+ if the file does not exist.
+ """
+ if os.path.exists(filename):
+ return iter(file(filename))
+ else:
+ return ()
def parse_machine_status(root_path, name):
- """Parse the status for one machine (of a multi-machine test)"""
- general_log = _file_iterator(os.path.join(root_path, "status.log"))
- machine_log = _file_iterator(os.path.join(root_path, name, "status.log"))
- timestamp_regex = re.compile("\ttimestamp=(\d+)")
- # collect all the lines from both the root & machine-specific log
- lines = []
- timestamp = 0
- for line in itertools.chain(general_log, machine_log):
- timestamp_match = timestamp_regex.search(line)
- # if the log line has a timestamp, use it
- # otherwise, just use the timestamp from the previous line
- if timestamp_match:
- timestamp = int(timestamp_match.group(1))
- lines.append((timestamp, line))
- lines.sort() # this will sort the lines by timestamp
- # now actually run the lines through the parser
- parser = Machine()
- for timestamp, line in lines:
- parser.process_line(line)
- return {
- "status": parser.details,
- "test_on": parser.test_name,
- "test_num_complete": parser.test_count
- }
+ """Parse the status for one machine (of a multi-machine test)"""
+ general_log = _file_iterator(os.path.join(root_path, "status.log"))
+ machine_log = _file_iterator(os.path.join(root_path, name, "status.log"))
+ timestamp_regex = re.compile("\ttimestamp=(\d+)")
+ # collect all the lines from both the root & machine-specific log
+ lines = []
+ timestamp = 0
+ for line in itertools.chain(general_log, machine_log):
+ timestamp_match = timestamp_regex.search(line)
+ # if the log line has a timestamp, use it
+ # otherwise, just use the timestamp from the previous line
+ if timestamp_match:
+ timestamp = int(timestamp_match.group(1))
+ lines.append((timestamp, line))
+ lines.sort() # this will sort the lines by timestamp
+ # now actually run the lines through the parser
+ parser = Machine()
+ for timestamp, line in lines:
+ parser.process_line(line)
+ return {
+ "status": parser.details,
+ "test_on": parser.test_name,
+ "test_num_complete": parser.test_count
+ }
def parse_multimachine_status(root_path, machine_names):
- """Parse the status for a set of machines."""
- results = {}
- for name in machine_names:
- results[name] = parse_machine_status(root_path, name)
- return results
+ """Parse the status for a set of machines."""
+ results = {}
+ for name in machine_names:
+ results[name] = parse_machine_status(root_path, name)
+ return results
if __name__ == "__main__":
- args = sys.argv[1:]
- if len(args) != 1:
- print "USAGE: status.py status_log"
- sys.exit(1)
- print parse_status(args[0])
+ args = sys.argv[1:]
+ if len(args) != 1:
+ print "USAGE: status.py status_log"
+ sys.exit(1)
+ print parse_status(args[0])
diff --git a/server/subcommand.py b/server/subcommand.py
index 85898a1..8b29b82 100644
--- a/server/subcommand.py
+++ b/server/subcommand.py
@@ -6,196 +6,196 @@
def parallel(tasklist, timeout=None):
- """Run an set of predefined subcommands in parallel"""
- pids = []
- run_error = False
- for task in tasklist:
- task.fork_start()
+ """Run an set of predefined subcommands in parallel"""
+ pids = []
+ run_error = False
+ for task in tasklist:
+ task.fork_start()
- remaining_timeout = None
- if timeout:
- endtime = time.time() + timeout
+ remaining_timeout = None
+ if timeout:
+ endtime = time.time() + timeout
- for task in tasklist:
- if timeout:
- remaining_timeout = max(endtime - time.time(), 1)
- try:
- status = task.fork_waitfor(remaining_timeout)
- except error.AutoservSubcommandError:
- run_error = True
- else:
- if status != 0:
- run_error = True
+ for task in tasklist:
+ if timeout:
+ remaining_timeout = max(endtime - time.time(), 1)
+ try:
+ status = task.fork_waitfor(remaining_timeout)
+ except error.AutoservSubcommandError:
+ run_error = True
+ else:
+ if status != 0:
+ run_error = True
- if run_error:
- raise error.AutoservError('One or more subcommands failed')
+ if run_error:
+ raise error.AutoservError('One or more subcommands failed')
def parallel_simple(function, arglist, log=True, timeout=None):
- """Each element in the arglist used to create a subcommand object,
- where that arg is used both as a subdir name, and a single argument
- to pass to "function".
- We create a subcommand object for each element in the list,
- then execute those subcommand objects in parallel."""
+ """Each element in the arglist used to create a subcommand object,
+ where that arg is used both as a subdir name, and a single argument
+ to pass to "function".
+ We create a subcommand object for each element in the list,
+ then execute those subcommand objects in parallel."""
- # Bypass the multithreading if only one machine.
- if len (arglist) == 1:
- function(arglist[0])
- return
-
- subcommands = []
- for arg in arglist:
- args = [arg]
- if log:
- subdir = str(arg)
- else:
- subdir = None
- subcommands.append(subcommand(function, args, subdir))
- parallel(subcommands, timeout)
+ # Bypass the multithreading if only one machine.
+ if len (arglist) == 1:
+ function(arglist[0])
+ return
+
+ subcommands = []
+ for arg in arglist:
+ args = [arg]
+ if log:
+ subdir = str(arg)
+ else:
+ subdir = None
+ subcommands.append(subcommand(function, args, subdir))
+ parallel(subcommands, timeout)
def _where_art_thy_filehandles():
- os.system("ls -l /proc/%d/fd >> /dev/tty" % os.getpid())
+ os.system("ls -l /proc/%d/fd >> /dev/tty" % os.getpid())
def _print_to_tty(string):
- open('/dev/tty', 'w').write(string + '\n')
+ open('/dev/tty', 'w').write(string + '\n')
def _redirect_stream(fd, output):
- newfd = os.open(output, os.O_WRONLY | os.O_CREAT)
- os.dup2(newfd, fd)
- os.close(newfd)
- if fd == 1:
- sys.stdout = os.fdopen(fd, 'w')
- if fd == 2:
- sys.stderr = os.fdopen(fd, 'w')
+ newfd = os.open(output, os.O_WRONLY | os.O_CREAT)
+ os.dup2(newfd, fd)
+ os.close(newfd)
+ if fd == 1:
+ sys.stdout = os.fdopen(fd, 'w')
+ if fd == 2:
+ sys.stderr = os.fdopen(fd, 'w')
def _redirect_stream_tee(fd, output, tag):
- """Use the low-level fork & pipe operations here to get a fd,
- not a filehandle. This ensures that we get both the
- filehandle and fd for stdout/stderr redirected correctly."""
- r, w = os.pipe()
- pid = os.fork()
- if pid: # Parent
- os.dup2(w, fd)
- os.close(r)
- os.close(w)
- if fd == 1:
- sys.stdout = os.fdopen(fd, 'w', 1)
- if fd == 2:
- sys.stderr = os.fdopen(fd, 'w', 1)
- return
- else: # Child
- os.close(w)
- log = open(output, 'w')
- f = os.fdopen(r, 'r')
- for line in iter(f.readline, ''):
- # Tee straight to file
- log.write(line)
- log.flush()
- # Prepend stdout with the tag
- print tag + ' : ' + line,
- sys.stdout.flush()
- log.close()
- os._exit(0)
+ """Use the low-level fork & pipe operations here to get a fd,
+ not a filehandle. This ensures that we get both the
+ filehandle and fd for stdout/stderr redirected correctly."""
+ r, w = os.pipe()
+ pid = os.fork()
+ if pid: # Parent
+ os.dup2(w, fd)
+ os.close(r)
+ os.close(w)
+ if fd == 1:
+ sys.stdout = os.fdopen(fd, 'w', 1)
+ if fd == 2:
+ sys.stderr = os.fdopen(fd, 'w', 1)
+ return
+ else: # Child
+ os.close(w)
+ log = open(output, 'w')
+ f = os.fdopen(r, 'r')
+ for line in iter(f.readline, ''):
+ # Tee straight to file
+ log.write(line)
+ log.flush()
+ # Prepend stdout with the tag
+ print tag + ' : ' + line,
+ sys.stdout.flush()
+ log.close()
+ os._exit(0)
class subcommand:
- def __init__(self, func, args, subdir = None, stdprint = True):
- # func(args) - the subcommand to run
- # subdir - the subdirectory to log results in
- # stdprint - whether to print results to stdout/stderr
- if subdir:
- self.subdir = os.path.abspath(subdir)
- if not os.path.exists(self.subdir):
- os.mkdir(self.subdir)
- self.debug = os.path.join(self.subdir, 'debug')
- if not os.path.exists(self.debug):
- os.mkdir(self.debug)
- self.stdout = os.path.join(self.debug, 'stdout')
- self.stderr = os.path.join(self.debug, 'stderr')
- else:
- self.subdir = None
- self.debug = '/dev/null'
- self.stdout = '/dev/null'
- self.stderr = '/dev/null'
+ def __init__(self, func, args, subdir = None, stdprint = True):
+ # func(args) - the subcommand to run
+ # subdir - the subdirectory to log results in
+ # stdprint - whether to print results to stdout/stderr
+ if subdir:
+ self.subdir = os.path.abspath(subdir)
+ if not os.path.exists(self.subdir):
+ os.mkdir(self.subdir)
+ self.debug = os.path.join(self.subdir, 'debug')
+ if not os.path.exists(self.debug):
+ os.mkdir(self.debug)
+ self.stdout = os.path.join(self.debug, 'stdout')
+ self.stderr = os.path.join(self.debug, 'stderr')
+ else:
+ self.subdir = None
+ self.debug = '/dev/null'
+ self.stdout = '/dev/null'
+ self.stderr = '/dev/null'
- self.func = func
- self.args = args
- self.lambda_function = lambda: func(*args)
- self.pid = None
- self.stdprint = stdprint
+ self.func = func
+ self.args = args
+ self.lambda_function = lambda: func(*args)
+ self.pid = None
+ self.stdprint = stdprint
- def redirect_output(self):
- if self.stdprint:
- if self.subdir:
- tag = os.path.basename(self.subdir)
- _redirect_stream_tee(1, self.stdout, tag)
- _redirect_stream_tee(2, self.stderr, tag)
- else:
- _redirect_stream(1, self.stdout)
- _redirect_stream(2, self.stderr)
+ def redirect_output(self):
+ if self.stdprint:
+ if self.subdir:
+ tag = os.path.basename(self.subdir)
+ _redirect_stream_tee(1, self.stdout, tag)
+ _redirect_stream_tee(2, self.stderr, tag)
+ else:
+ _redirect_stream(1, self.stdout)
+ _redirect_stream(2, self.stderr)
- def fork_start(self):
- sys.stdout.flush()
- sys.stderr.flush()
- self.pid = os.fork()
+ def fork_start(self):
+ sys.stdout.flush()
+ sys.stderr.flush()
+ self.pid = os.fork()
- if self.pid: # I am the parent
- return
+ if self.pid: # I am the parent
+ return
- # We are the child from this point on. Never return.
- signal.signal(signal.SIGTERM, signal.SIG_DFL) # clear handler
- if self.subdir:
- os.chdir(self.subdir)
- self.redirect_output()
+ # We are the child from this point on. Never return.
+ signal.signal(signal.SIGTERM, signal.SIG_DFL) # clear handler
+ if self.subdir:
+ os.chdir(self.subdir)
+ self.redirect_output()
- try:
- self.lambda_function()
+ try:
+ self.lambda_function()
- except:
- traceback.print_exc()
- sys.stdout.flush()
- sys.stderr.flush()
- os._exit(1)
+ except:
+ traceback.print_exc()
+ sys.stdout.flush()
+ sys.stderr.flush()
+ os._exit(1)
- sys.stdout.flush()
- sys.stderr.flush()
- os._exit(0)
+ sys.stdout.flush()
+ sys.stderr.flush()
+ os._exit(0)
- def fork_waitfor(self, timeout=None):
- if not timeout:
- (pid, status) = os.waitpid(self.pid, 0)
- else:
- pid = None
- start_time = time.time()
- while time.time() <= start_time + timeout:
- (pid, status) = os.waitpid(self.pid, os.WNOHANG)
- if pid:
- break
- time.sleep(1)
+ def fork_waitfor(self, timeout=None):
+ if not timeout:
+ (pid, status) = os.waitpid(self.pid, 0)
+ else:
+ pid = None
+ start_time = time.time()
+ while time.time() <= start_time + timeout:
+ (pid, status) = os.waitpid(self.pid, os.WNOHANG)
+ if pid:
+ break
+ time.sleep(1)
- if not pid:
- utils.nuke_pid(self.pid)
- print "subcommand failed pid %d" % self.pid
- print "%s" % (self.func,)
- print "timeout after %ds" % timeout
- print
- return None
+ if not pid:
+ utils.nuke_pid(self.pid)
+ print "subcommand failed pid %d" % self.pid
+ print "%s" % (self.func,)
+ print "timeout after %ds" % timeout
+ print
+ return None
- if status != 0:
- print "subcommand failed pid %d" % pid
- print "%s" % (self.func,)
- print "rc=%d" % status
- print
- if os.path.exists(self.stderr):
- for line in open(self.stderr).readlines():
- print line,
- print "\n--------------------------------------------\n"
- raise error.AutoservSubcommandError(self.func, status)
- return status
+ if status != 0:
+ print "subcommand failed pid %d" % pid
+ print "%s" % (self.func,)
+ print "rc=%d" % status
+ print
+ if os.path.exists(self.stderr):
+ for line in open(self.stderr).readlines():
+ print line,
+ print "\n--------------------------------------------\n"
+ raise error.AutoservSubcommandError(self.func, status)
+ return status
diff --git a/server/test.py b/server/test.py
index 34de6a6..3176857 100755
--- a/server/test.py
+++ b/server/test.py
@@ -9,12 +9,12 @@
class test(common_test.base_test):
- pass
+ pass
testname = common_test.testname
def runtest(job, url, tag, args, dargs):
- common_test.runtest(job, url, tag, args, dargs,
- locals(), globals())
+ common_test.runtest(job, url, tag, args, dargs,
+ locals(), globals())
diff --git a/server/tests/sleeptest/sleeptest.py b/server/tests/sleeptest/sleeptest.py
index 7b9d12c..1ae3013 100755
--- a/server/tests/sleeptest/sleeptest.py
+++ b/server/tests/sleeptest/sleeptest.py
@@ -1,7 +1,7 @@
import test, time
class sleeptest(test.test):
- version = 1
+ version = 1
- def execute(self, seconds = 1):
- time.sleep(seconds)
+ def execute(self, seconds = 1):
+ time.sleep(seconds)
diff --git a/server/utils.py b/server/utils.py
index 56a102e..e5e0dd6 100644
--- a/server/utils.py
+++ b/server/utils.py
@@ -22,368 +22,368 @@
############# we need pass throughs for the methods in client/common_lib/utils
-def run(command, timeout=None, ignore_status=False,
- stdout_tee=None, stderr_tee=None):
- return utils.run(command, timeout, ignore_status,
- stdout_tee, stderr_tee)
+def run(command, timeout=None, ignore_status=False,
+ stdout_tee=None, stderr_tee=None):
+ return utils.run(command, timeout, ignore_status,
+ stdout_tee, stderr_tee)
def system(command, timeout=None, ignore_status=False):
- return utils.system(command, timeout, ignore_status)
+ return utils.system(command, timeout, ignore_status)
def system_output(command, timeout=None, ignore_status=False,
- retain_output=False):
- return utils.system_output(command, timeout, ignore_status,
- retain_output)
+ retain_output=False):
+ return utils.system_output(command, timeout, ignore_status,
+ retain_output)
def urlopen(url, data=None, proxies=None, timeout=300):
- return utils.urlopen(url, data=data, proxies=proxies, timeout=timeout)
+ return utils.urlopen(url, data=data, proxies=proxies, timeout=timeout)
def urlretrieve(url, filename=None, reporthook=None, data=None, timeout=300):
- return utils.urlretrieve(url, filename=filename, reporthook=reporthook,
- data=data, timeout=timeout)
+ return utils.urlretrieve(url, filename=filename, reporthook=reporthook,
+ data=data, timeout=timeout)
def read_keyval(path):
- return utils.read_keyval(path)
+ return utils.read_keyval(path)
def write_keyval(path, dictionary):
- return utils.write_keyval(path, dictionary)
+ return utils.write_keyval(path, dictionary)
####################################################################
def sh_escape(command):
- """
- Escape special characters from a command so that it can be passed
- as a double quoted (" ") string in a (ba)sh command.
+ """
+ Escape special characters from a command so that it can be passed
+ as a double quoted (" ") string in a (ba)sh command.
- Args:
- command: the command string to escape.
+ Args:
+ command: the command string to escape.
- Returns:
- The escaped command string. The required englobing double
- quotes are NOT added and so should be added at some point by
- the caller.
+ Returns:
+ The escaped command string. The required englobing double
+ quotes are NOT added and so should be added at some point by
+ the caller.
- See also: http://www.tldp.org/LDP/abs/html/escapingsection.html
- """
- command = command.replace("\\", "\\\\")
- command = command.replace("$", r'\$')
- command = command.replace('"', r'\"')
- command = command.replace('`', r'\`')
- return command
+ See also: http://www.tldp.org/LDP/abs/html/escapingsection.html
+ """
+ command = command.replace("\\", "\\\\")
+ command = command.replace("$", r'\$')
+ command = command.replace('"', r'\"')
+ command = command.replace('`', r'\`')
+ return command
def scp_remote_escape(filename):
- """
- Escape special characters from a filename so that it can be passed
- to scp (within double quotes) as a remote file.
+ """
+ Escape special characters from a filename so that it can be passed
+ to scp (within double quotes) as a remote file.
- Bis-quoting has to be used with scp for remote files, "bis-quoting"
- as in quoting x 2
- scp does not support a newline in the filename
+ Bis-quoting has to be used with scp for remote files, "bis-quoting"
+ as in quoting x 2
+ scp does not support a newline in the filename
- Args:
- filename: the filename string to escape.
+ Args:
+ filename: the filename string to escape.
- Returns:
- The escaped filename string. The required englobing double
- quotes are NOT added and so should be added at some point by
- the caller.
- """
- escape_chars= r' !"$&' "'" r'()*,:;<=>?[\]^`{|}'
+ Returns:
+ The escaped filename string. The required englobing double
+ quotes are NOT added and so should be added at some point by
+ the caller.
+ """
+ escape_chars= r' !"$&' "'" r'()*,:;<=>?[\]^`{|}'
- new_name= []
- for char in filename:
- if char in escape_chars:
- new_name.append("\\%s" % (char,))
- else:
- new_name.append(char)
+ new_name= []
+ for char in filename:
+ if char in escape_chars:
+ new_name.append("\\%s" % (char,))
+ else:
+ new_name.append(char)
- return sh_escape("".join(new_name))
+ return sh_escape("".join(new_name))
def get(location, local_copy = False):
- """Get a file or directory to a local temporary directory.
+ """Get a file or directory to a local temporary directory.
- Args:
- location: the source of the material to get. This source may
- be one of:
- * a local file or directory
- * a URL (http or ftp)
- * a python file-like object
+ Args:
+ location: the source of the material to get. This source may
+ be one of:
+ * a local file or directory
+ * a URL (http or ftp)
+ * a python file-like object
- Returns:
- The location of the file or directory where the requested
- content was saved. This will be contained in a temporary
- directory on the local host. If the material to get was a
- directory, the location will contain a trailing '/'
- """
- tmpdir = get_tmp_dir()
+ Returns:
+ The location of the file or directory where the requested
+ content was saved. This will be contained in a temporary
+ directory on the local host. If the material to get was a
+ directory, the location will contain a trailing '/'
+ """
+ tmpdir = get_tmp_dir()
- # location is a file-like object
- if hasattr(location, "read"):
- tmpfile = os.path.join(tmpdir, "file")
- tmpfileobj = file(tmpfile, 'w')
- shutil.copyfileobj(location, tmpfileobj)
- tmpfileobj.close()
- return tmpfile
+ # location is a file-like object
+ if hasattr(location, "read"):
+ tmpfile = os.path.join(tmpdir, "file")
+ tmpfileobj = file(tmpfile, 'w')
+ shutil.copyfileobj(location, tmpfileobj)
+ tmpfileobj.close()
+ return tmpfile
- if isinstance(location, types.StringTypes):
- # location is a URL
- if location.startswith('http') or location.startswith('ftp'):
- tmpfile = os.path.join(tmpdir, os.path.basename(location))
- utils.urlretrieve(location, tmpfile)
- return tmpfile
- # location is a local path
- elif os.path.exists(os.path.abspath(location)):
- if not local_copy:
- if os.path.isdir(location):
- return location.rstrip('/') + '/'
- else:
- return location
- tmpfile = os.path.join(tmpdir, os.path.basename(location))
- if os.path.isdir(location):
- tmpfile += '/'
- shutil.copytree(location, tmpfile, symlinks=True)
- return tmpfile
- shutil.copyfile(location, tmpfile)
- return tmpfile
- # location is just a string, dump it to a file
- else:
- tmpfd, tmpfile = tempfile.mkstemp(dir=tmpdir)
- tmpfileobj = os.fdopen(tmpfd, 'w')
- tmpfileobj.write(location)
- tmpfileobj.close()
- return tmpfile
+ if isinstance(location, types.StringTypes):
+ # location is a URL
+ if location.startswith('http') or location.startswith('ftp'):
+ tmpfile = os.path.join(tmpdir, os.path.basename(location))
+ utils.urlretrieve(location, tmpfile)
+ return tmpfile
+ # location is a local path
+ elif os.path.exists(os.path.abspath(location)):
+ if not local_copy:
+ if os.path.isdir(location):
+ return location.rstrip('/') + '/'
+ else:
+ return location
+ tmpfile = os.path.join(tmpdir, os.path.basename(location))
+ if os.path.isdir(location):
+ tmpfile += '/'
+ shutil.copytree(location, tmpfile, symlinks=True)
+ return tmpfile
+ shutil.copyfile(location, tmpfile)
+ return tmpfile
+ # location is just a string, dump it to a file
+ else:
+ tmpfd, tmpfile = tempfile.mkstemp(dir=tmpdir)
+ tmpfileobj = os.fdopen(tmpfd, 'w')
+ tmpfileobj.write(location)
+ tmpfileobj.close()
+ return tmpfile
def get_tmp_dir():
- """Return the pathname of a directory on the host suitable
- for temporary file storage.
+ """Return the pathname of a directory on the host suitable
+ for temporary file storage.
- The directory and its content will be deleted automatically
- at the end of the program execution if they are still present.
- """
- global __tmp_dirs
+ The directory and its content will be deleted automatically
+ at the end of the program execution if they are still present.
+ """
+ global __tmp_dirs
- dir_name= tempfile.mkdtemp(prefix="autoserv-")
- pid = os.getpid()
- if not pid in __tmp_dirs:
- __tmp_dirs[pid] = []
- __tmp_dirs[pid].append(dir_name)
- return dir_name
+ dir_name= tempfile.mkdtemp(prefix="autoserv-")
+ pid = os.getpid()
+ if not pid in __tmp_dirs:
+ __tmp_dirs[pid] = []
+ __tmp_dirs[pid].append(dir_name)
+ return dir_name
@atexit.register
def __clean_tmp_dirs():
- """Erase temporary directories that were created by the get_tmp_dir()
- function and that are still present.
- """
- global __tmp_dirs
+ """Erase temporary directories that were created by the get_tmp_dir()
+ function and that are still present.
+ """
+ global __tmp_dirs
- pid = os.getpid()
- if pid not in __tmp_dirs:
- return
- for dir in __tmp_dirs[pid]:
- try:
- shutil.rmtree(dir)
- except OSError, e:
- if e.errno == 2:
- pass
- __tmp_dirs[pid] = []
+ pid = os.getpid()
+ if pid not in __tmp_dirs:
+ return
+ for dir in __tmp_dirs[pid]:
+ try:
+ shutil.rmtree(dir)
+ except OSError, e:
+ if e.errno == 2:
+ pass
+ __tmp_dirs[pid] = []
def unarchive(host, source_material):
- """Uncompress and untar an archive on a host.
+ """Uncompress and untar an archive on a host.
- If the "source_material" is compresses (according to the file
- extension) it will be uncompressed. Supported compression formats
- are gzip and bzip2. Afterwards, if the source_material is a tar
- archive, it will be untarred.
+ If the "source_material" is compresses (according to the file
+ extension) it will be uncompressed. Supported compression formats
+ are gzip and bzip2. Afterwards, if the source_material is a tar
+ archive, it will be untarred.
- Args:
- host: the host object on which the archive is located
- source_material: the path of the archive on the host
+ Args:
+ host: the host object on which the archive is located
+ source_material: the path of the archive on the host
- Returns:
- The file or directory name of the unarchived source material.
- If the material is a tar archive, it will be extracted in the
- directory where it is and the path returned will be the first
- entry in the archive, assuming it is the topmost directory.
- If the material is not an archive, nothing will be done so this
- function is "harmless" when it is "useless".
- """
- # uncompress
- if (source_material.endswith(".gz") or
- source_material.endswith(".gzip")):
- host.run('gunzip "%s"' % (sh_escape(source_material)))
- source_material= ".".join(source_material.split(".")[:-1])
- elif source_material.endswith("bz2"):
- host.run('bunzip2 "%s"' % (sh_escape(source_material)))
- source_material= ".".join(source_material.split(".")[:-1])
+ Returns:
+ The file or directory name of the unarchived source material.
+ If the material is a tar archive, it will be extracted in the
+ directory where it is and the path returned will be the first
+ entry in the archive, assuming it is the topmost directory.
+ If the material is not an archive, nothing will be done so this
+ function is "harmless" when it is "useless".
+ """
+ # uncompress
+ if (source_material.endswith(".gz") or
+ source_material.endswith(".gzip")):
+ host.run('gunzip "%s"' % (sh_escape(source_material)))
+ source_material= ".".join(source_material.split(".")[:-1])
+ elif source_material.endswith("bz2"):
+ host.run('bunzip2 "%s"' % (sh_escape(source_material)))
+ source_material= ".".join(source_material.split(".")[:-1])
- # untar
- if source_material.endswith(".tar"):
- retval= host.run('tar -C "%s" -xvf "%s"' % (
- sh_escape(os.path.dirname(source_material)),
- sh_escape(source_material),))
- source_material= os.path.join(os.path.dirname(source_material),
- retval.stdout.split()[0])
+ # untar
+ if source_material.endswith(".tar"):
+ retval= host.run('tar -C "%s" -xvf "%s"' % (
+ sh_escape(os.path.dirname(source_material)),
+ sh_escape(source_material),))
+ source_material= os.path.join(os.path.dirname(source_material),
+ retval.stdout.split()[0])
- return source_material
+ return source_material
def get_server_dir():
- path = os.path.dirname(sys.modules['autotest_lib.server.utils'].__file__)
- return os.path.abspath(path)
+ path = os.path.dirname(sys.modules['autotest_lib.server.utils'].__file__)
+ return os.path.abspath(path)
def find_pid(command):
- for line in utils.system_output('ps -eo pid,cmd').rstrip().split('\n'):
- (pid, cmd) = line.split(None, 1)
- if re.search(command, cmd):
- return int(pid)
- return None
+ for line in utils.system_output('ps -eo pid,cmd').rstrip().split('\n'):
+ (pid, cmd) = line.split(None, 1)
+ if re.search(command, cmd):
+ return int(pid)
+ return None
def nohup(command, stdout='/dev/null', stderr='/dev/null', background=True,
- env = {}):
- cmd = ' '.join(key+'='+val for key, val in env.iteritems())
- cmd += ' nohup ' + command
- cmd += ' > %s' % stdout
- if stdout == stderr:
- cmd += ' 2>&1'
- else:
- cmd += ' 2> %s' % stderr
- if background:
- cmd += ' &'
- utils.system(cmd)
+ env = {}):
+ cmd = ' '.join(key+'='+val for key, val in env.iteritems())
+ cmd += ' nohup ' + command
+ cmd += ' > %s' % stdout
+ if stdout == stderr:
+ cmd += ' 2>&1'
+ else:
+ cmd += ' 2> %s' % stderr
+ if background:
+ cmd += ' &'
+ utils.system(cmd)
def default_mappings(machines):
- """
- Returns a simple mapping in which all machines are assigned to the
- same key. Provides the default behavior for
- form_ntuples_from_machines. """
- mappings = {}
- failures = []
-
- mach = machines[0]
- mappings['ident'] = [mach]
- if len(machines) > 1:
- machines = machines[1:]
- for machine in machines:
- mappings['ident'].append(machine)
-
- return (mappings, failures)
+ """
+ Returns a simple mapping in which all machines are assigned to the
+ same key. Provides the default behavior for
+ form_ntuples_from_machines. """
+ mappings = {}
+ failures = []
+
+ mach = machines[0]
+ mappings['ident'] = [mach]
+ if len(machines) > 1:
+ machines = machines[1:]
+ for machine in machines:
+ mappings['ident'].append(machine)
+
+ return (mappings, failures)
def form_ntuples_from_machines(machines, n=2, mapping_func=default_mappings):
- """Returns a set of ntuples from machines where the machines in an
- ntuple are in the same mapping, and a set of failures which are
- (machine name, reason) tuples."""
- ntuples = []
- (mappings, failures) = mapping_func(machines)
-
- # now run through the mappings and create n-tuples.
- # throw out the odd guys out
- for key in mappings:
- key_machines = mappings[key]
- total_machines = len(key_machines)
+ """Returns a set of ntuples from machines where the machines in an
+ ntuple are in the same mapping, and a set of failures which are
+ (machine name, reason) tuples."""
+ ntuples = []
+ (mappings, failures) = mapping_func(machines)
- # form n-tuples
- while len(key_machines) >= n:
- ntuples.append(key_machines[0:n])
- key_machines = key_machines[n:]
+ # now run through the mappings and create n-tuples.
+ # throw out the odd guys out
+ for key in mappings:
+ key_machines = mappings[key]
+ total_machines = len(key_machines)
- for mach in key_machines:
- failures.append((mach, "machine can not be tupled"))
+ # form n-tuples
+ while len(key_machines) >= n:
+ ntuples.append(key_machines[0:n])
+ key_machines = key_machines[n:]
- return (ntuples, failures)
+ for mach in key_machines:
+ failures.append((mach, "machine can not be tupled"))
+
+ return (ntuples, failures)
def parse_machine(machine, user = 'root', port = 22, password = ''):
- """
- Parse the machine string user:pass@host:port and return it separately,
- if the machine string is not complete, use the default parameters
- when appropriate.
- """
+ """
+ Parse the machine string user:pass@host:port and return it separately,
+ if the machine string is not complete, use the default parameters
+ when appropriate.
+ """
- user = user
- port = port
- password = password
+ user = user
+ port = port
+ password = password
- if re.search('@', machine):
- machine = machine.split('@')
+ if re.search('@', machine):
+ machine = machine.split('@')
- if re.search(':', machine[0]):
- machine[0] = machine[0].split(':')
- user = machine[0][0]
- password = machine[0][1]
+ if re.search(':', machine[0]):
+ machine[0] = machine[0].split(':')
+ user = machine[0][0]
+ password = machine[0][1]
- else:
- user = machine[0]
+ else:
+ user = machine[0]
- if re.search(':', machine[1]):
- machine[1] = machine[1].split(':')
- hostname = machine[1][0]
- port = int(machine[1][1])
+ if re.search(':', machine[1]):
+ machine[1] = machine[1].split(':')
+ hostname = machine[1][0]
+ port = int(machine[1][1])
- else:
- hostname = machine[1]
+ else:
+ hostname = machine[1]
- elif re.search(':', machine):
- machine = machine.split(':')
- hostname = machine[0]
- port = int(machine[1])
+ elif re.search(':', machine):
+ machine = machine.split(':')
+ hostname = machine[0]
+ port = int(machine[1])
- else:
- hostname = machine
+ else:
+ hostname = machine
- return hostname, user, password, port
+ return hostname, user, password, port
def get_public_key():
- """
- Return a valid string ssh public key for the user executing autoserv or
- autotest. If there's no DSA or RSA public key, create a DSA keypair with
- ssh-keygen and return it.
- """
+ """
+ Return a valid string ssh public key for the user executing autoserv or
+ autotest. If there's no DSA or RSA public key, create a DSA keypair with
+ ssh-keygen and return it.
+ """
- ssh_conf_path = os.path.join(os.environ['HOME'], '.ssh')
+ ssh_conf_path = os.path.join(os.environ['HOME'], '.ssh')
- dsa_public_key_path = os.path.join(ssh_conf_path, 'id_dsa.pub')
- dsa_private_key_path = os.path.join(ssh_conf_path, 'id_dsa')
+ dsa_public_key_path = os.path.join(ssh_conf_path, 'id_dsa.pub')
+ dsa_private_key_path = os.path.join(ssh_conf_path, 'id_dsa')
- rsa_public_key_path = os.path.join(ssh_conf_path, 'id_rsa.pub')
- rsa_private_key_path = os.path.join(ssh_conf_path, 'id_rsa')
+ rsa_public_key_path = os.path.join(ssh_conf_path, 'id_rsa.pub')
+ rsa_private_key_path = os.path.join(ssh_conf_path, 'id_rsa')
- has_dsa_keypair = os.path.isfile(dsa_public_key_path) and \
- os.path.isfile(dsa_private_key_path)
- has_rsa_keypair = os.path.isfile(rsa_public_key_path) and \
- os.path.isfile(rsa_private_key_path)
+ has_dsa_keypair = os.path.isfile(dsa_public_key_path) and \
+ os.path.isfile(dsa_private_key_path)
+ has_rsa_keypair = os.path.isfile(rsa_public_key_path) and \
+ os.path.isfile(rsa_private_key_path)
- if has_dsa_keypair:
- print 'DSA keypair found, using it'
- public_key_path = dsa_public_key_path
+ if has_dsa_keypair:
+ print 'DSA keypair found, using it'
+ public_key_path = dsa_public_key_path
- elif has_rsa_keypair:
- print 'RSA keypair found, using it'
- public_key_path = rsa_public_key_path
+ elif has_rsa_keypair:
+ print 'RSA keypair found, using it'
+ public_key_path = rsa_public_key_path
- else:
- print 'Neither RSA nor DSA keypair found, creating DSA ssh key pair'
- system('ssh-keygen -t dsa -q -N "" -f %s' % dsa_private_key_path)
- public_key_path = dsa_public_key_path
+ else:
+ print 'Neither RSA nor DSA keypair found, creating DSA ssh key pair'
+ system('ssh-keygen -t dsa -q -N "" -f %s' % dsa_private_key_path)
+ public_key_path = dsa_public_key_path
- public_key = open(public_key_path, 'r')
- public_key_str = public_key.read()
- public_key.close()
+ public_key = open(public_key_path, 'r')
+ public_key_str = public_key.read()
+ public_key.close()
- return public_key_str
+ return public_key_str
diff --git a/server/utils_unittest.py b/server/utils_unittest.py
index d28ec16..002164e 100644
--- a/server/utils_unittest.py
+++ b/server/utils_unittest.py
@@ -8,23 +8,23 @@
class UtilsTest(unittest.TestCase):
-
- def setUp(self):
- # define out machines here
- self.machines = ['mach1', 'mach2', 'mach3', 'mach4', 'mach5',
- 'mach6', 'mach7']
-
- self.ntuples = [['mach1', 'mach2'], ['mach3', 'mach4'],
- ['mach5', 'mach6']]
- self.failures = []
- self.failures.append(('mach7', "machine can not be tupled"))
+
+ def setUp(self):
+ # define out machines here
+ self.machines = ['mach1', 'mach2', 'mach3', 'mach4', 'mach5',
+ 'mach6', 'mach7']
+
+ self.ntuples = [['mach1', 'mach2'], ['mach3', 'mach4'],
+ ['mach5', 'mach6']]
+ self.failures = []
+ self.failures.append(('mach7', "machine can not be tupled"))
- def test_form_cell_mappings(self):
- (ntuples, failures) = utils.form_ntuples_from_machines(self.machines)
- self.assertEquals(self.ntuples, ntuples)
- self.assertEquals(self.failures, failures)
+ def test_form_cell_mappings(self):
+ (ntuples, failures) = utils.form_ntuples_from_machines(self.machines)
+ self.assertEquals(self.ntuples, ntuples)
+ self.assertEquals(self.failures, failures)
if __name__ == "__main__":
- unittest.main()
+ unittest.main()
diff --git a/server/warning_monitor.py b/server/warning_monitor.py
index 6cd15c7..3b216b9 100644
--- a/server/warning_monitor.py
+++ b/server/warning_monitor.py
@@ -8,11 +8,11 @@
# the format for a warning used here is:
# <timestamp (integer)> <tab> <status (string)> <newline>
def make_alert(msg):
- def alert(*params):
- formatted_msg = msg % params
- timestamped_msg = "%d\t%s" % (time.time(), formatted_msg)
- print >> warnfile, timestamped_msg
- return alert
+ def alert(*params):
+ formatted_msg = msg % params
+ timestamped_msg = "%d\t%s" % (time.time(), formatted_msg)
+ print >> warnfile, timestamped_msg
+ return alert
pattern_file = os.path.join(os.path.dirname(__file__), 'warning_patterns')
@@ -28,15 +28,15 @@
# assert that the patterns are separated by empty lines
if sum(len(line.strip()) for line in pattern_lines[2::3]) > 0:
- raise ValueError('warning patterns are not separated by blank lines')
+ raise ValueError('warning patterns are not separated by blank lines')
hooks = [(re.compile(regex.rstrip('\n')), make_alert(alert.rstrip('\n')))
- for regex, alert in patterns]
+ for regex, alert in patterns]
while True:
- line = sys.stdin.readline()
- logfile.write(line)
- for regex, callback in hooks:
- match = re.match(regex, line.strip())
- if match:
- callback(*match.groups())
+ line = sys.stdin.readline()
+ logfile.write(line)
+ for regex, callback in hooks:
+ match = re.match(regex, line.strip())
+ if match:
+ callback(*match.groups())
diff --git a/tko/db.py b/tko/db.py
index 235b485..175869b 100644
--- a/tko/db.py
+++ b/tko/db.py
@@ -5,488 +5,488 @@
class MySQLTooManyRows(Exception):
- pass
+ pass
class db_sql:
- def __init__(self, debug=False, autocommit=True, host=None,
- database=None, user=None, password=None):
- self.debug = debug
- self.autocommit = autocommit
- self._load_config(host, database, user, password)
+ def __init__(self, debug=False, autocommit=True, host=None,
+ database=None, user=None, password=None):
+ self.debug = debug
+ self.autocommit = autocommit
+ self._load_config(host, database, user, password)
- self.con = None
- self._init_db()
+ self.con = None
+ self._init_db()
- # if not present, insert statuses
- self.status_idx = {}
- self.status_word = {}
- status_rows = self.select('status_idx, word', 'status', None)
- for s in status_rows:
- self.status_idx[s[1]] = s[0]
- self.status_word[s[0]] = s[1]
+ # if not present, insert statuses
+ self.status_idx = {}
+ self.status_word = {}
+ status_rows = self.select('status_idx, word', 'status', None)
+ for s in status_rows:
+ self.status_idx[s[1]] = s[0]
+ self.status_word[s[0]] = s[1]
- machine_map = os.path.join(os.path.dirname(__file__),
- 'machines')
- if os.path.exists(machine_map):
- self.machine_map = machine_map
- else:
- self.machine_map = None
- self.machine_group = {}
+ machine_map = os.path.join(os.path.dirname(__file__),
+ 'machines')
+ if os.path.exists(machine_map):
+ self.machine_map = machine_map
+ else:
+ self.machine_map = None
+ self.machine_group = {}
- def _load_config(self, host, database, user, password):
- # grab the global config
- get_value = global_config.global_config.get_config_value
+ def _load_config(self, host, database, user, password):
+ # grab the global config
+ get_value = global_config.global_config.get_config_value
- # grab the host, database
- if host:
- self.host = host
- else:
- self.host = get_value("TKO", "host")
- if database:
- self.database = database
- else:
- self.database = get_value("TKO", "database")
+ # grab the host, database
+ if host:
+ self.host = host
+ else:
+ self.host = get_value("TKO", "host")
+ if database:
+ self.database = database
+ else:
+ self.database = get_value("TKO", "database")
- # grab the user and password
- if user:
- self.user = user
- else:
- self.user = get_value("TKO", "user")
- if password:
- self.password = password
- else:
- self.password = get_value("TKO", "password")
+ # grab the user and password
+ if user:
+ self.user = user
+ else:
+ self.user = get_value("TKO", "user")
+ if password:
+ self.password = password
+ else:
+ self.password = get_value("TKO", "password")
- # grab the timeout configuration
- self.query_timeout = get_value("TKO", "query_timeout",
- type=int, default=3600)
- self.min_delay = get_value("TKO", "min_retry_delay", type=int,
- default=20)
- self.max_delay = get_value("TKO", "max_retry_delay", type=int,
- default=60)
+ # grab the timeout configuration
+ self.query_timeout = get_value("TKO", "query_timeout",
+ type=int, default=3600)
+ self.min_delay = get_value("TKO", "min_retry_delay", type=int,
+ default=20)
+ self.max_delay = get_value("TKO", "max_retry_delay", type=int,
+ default=60)
- def _init_db(self):
- # make sure we clean up any existing connection
- if self.con:
- self.con.close()
- self.con = None
+ def _init_db(self):
+ # make sure we clean up any existing connection
+ if self.con:
+ self.con.close()
+ self.con = None
- # create the db connection and cursor
- self.con = self.connect(self.host, self.database,
- self.user, self.password)
- self.cur = self.con.cursor()
+ # create the db connection and cursor
+ self.con = self.connect(self.host, self.database,
+ self.user, self.password)
+ self.cur = self.con.cursor()
- def _random_delay(self):
- delay = random.randint(self.min_delay, self.max_delay)
- time.sleep(delay)
+ def _random_delay(self):
+ delay = random.randint(self.min_delay, self.max_delay)
+ time.sleep(delay)
- def run_with_retry(self, function, *args, **dargs):
- """Call function(*args, **dargs) until either it passes
- without an operational error, or a timeout is reached.
- This will re-connect to the database, so it is NOT safe
- to use this inside of a database transaction.
+ def run_with_retry(self, function, *args, **dargs):
+ """Call function(*args, **dargs) until either it passes
+ without an operational error, or a timeout is reached.
+ This will re-connect to the database, so it is NOT safe
+ to use this inside of a database transaction.
- It can be safely used with transactions, but the
- transaction start & end must be completely contained
- within the call to 'function'."""
- OperationalError = _get_error_class("OperationalError")
+ It can be safely used with transactions, but the
+ transaction start & end must be completely contained
+ within the call to 'function'."""
+ OperationalError = _get_error_class("OperationalError")
- success = False
- start_time = time.time()
- while not success:
- try:
- result = function(*args, **dargs)
- except OperationalError, e:
- self._log_operational_error(e)
- stop_time = time.time()
- elapsed_time = stop_time - start_time
- if elapsed_time > self.query_timeout:
- raise
- else:
- try:
- self._random_delay()
- self._init_db()
- except OperationalError, e:
- self._log_operational_error(e)
- else:
- success = True
- return result
+ success = False
+ start_time = time.time()
+ while not success:
+ try:
+ result = function(*args, **dargs)
+ except OperationalError, e:
+ self._log_operational_error(e)
+ stop_time = time.time()
+ elapsed_time = stop_time - start_time
+ if elapsed_time > self.query_timeout:
+ raise
+ else:
+ try:
+ self._random_delay()
+ self._init_db()
+ except OperationalError, e:
+ self._log_operational_error(e)
+ else:
+ success = True
+ return result
- def _log_operational_error(self, e):
- msg = ("An operational error occured during a database "
- "operation: %s" % str(e))
- print >> sys.stderr, msg
- sys.stderr.flush() # we want these msgs to show up immediately
+ def _log_operational_error(self, e):
+ msg = ("An operational error occured during a database "
+ "operation: %s" % str(e))
+ print >> sys.stderr, msg
+ sys.stderr.flush() # we want these msgs to show up immediately
- def dprint(self, value):
- if self.debug:
- sys.stdout.write('SQL: ' + str(value) + '\n')
+ def dprint(self, value):
+ if self.debug:
+ sys.stdout.write('SQL: ' + str(value) + '\n')
- def commit(self):
- self.con.commit()
+ def commit(self):
+ self.con.commit()
- def get_last_autonumber_value(self):
- self.cur.execute('SELECT LAST_INSERT_ID()', [])
- return self.cur.fetchall()[0][0]
+ def get_last_autonumber_value(self):
+ self.cur.execute('SELECT LAST_INSERT_ID()', [])
+ return self.cur.fetchall()[0][0]
- def select(self, fields, table, where, wherein={},
- distinct = False, group_by = None, max_rows = None):
- """\
- This selects all the fields requested from a
- specific table with a particular where clause.
- The where clause can either be a dictionary of
- field=value pairs, a string, or a tuple of (string,
- a list of values). The last option is what you
- should use when accepting user input as it'll
- protect you against sql injection attacks (if
- all user data is placed in the array rather than
- the raw SQL).
+ def select(self, fields, table, where, wherein={},
+ distinct = False, group_by = None, max_rows = None):
+ """\
+ This selects all the fields requested from a
+ specific table with a particular where clause.
+ The where clause can either be a dictionary of
+ field=value pairs, a string, or a tuple of (string,
+ a list of values). The last option is what you
+ should use when accepting user input as it'll
+ protect you against sql injection attacks (if
+ all user data is placed in the array rather than
+ the raw SQL).
- For example:
- where = ("a = %s AND b = %s", ['val', 'val'])
- is better than
- where = "a = 'val' AND b = 'val'"
- """
- cmd = ['select']
- if distinct:
- cmd.append('distinct')
- cmd += [fields, 'from', table]
+ For example:
+ where = ("a = %s AND b = %s", ['val', 'val'])
+ is better than
+ where = "a = 'val' AND b = 'val'"
+ """
+ cmd = ['select']
+ if distinct:
+ cmd.append('distinct')
+ cmd += [fields, 'from', table]
- values = []
- if where and isinstance(where, types.DictionaryType):
- # key/value pairs (which should be equal)
- keys = [field + '=%s' for field in where.keys()]
- values = [where[field] for field in where.keys()]
+ values = []
+ if where and isinstance(where, types.DictionaryType):
+ # key/value pairs (which should be equal)
+ keys = [field + '=%s' for field in where.keys()]
+ values = [where[field] for field in where.keys()]
- cmd.append(' where ' + ' and '.join(keys))
- elif where and isinstance(where, types.StringTypes):
- # the exact string
- cmd.append(' where ' + where)
- elif where and isinstance(where, types.TupleType):
- # preformatted where clause + values
- (sql, vals) = where
- values = vals
- cmd.append(' where (%s) ' % sql)
+ cmd.append(' where ' + ' and '.join(keys))
+ elif where and isinstance(where, types.StringTypes):
+ # the exact string
+ cmd.append(' where ' + where)
+ elif where and isinstance(where, types.TupleType):
+ # preformatted where clause + values
+ (sql, vals) = where
+ values = vals
+ cmd.append(' where (%s) ' % sql)
- # TODO: this assumes there's a where clause...bad
- if wherein and isinstance(wherein, types.DictionaryType):
- keys_in = ["%s in (%s) " % (field, ','.join(where))
- for field, where in wherein.iteritems()]
- cmd.append(' and '+' and '.join(keys_in))
+ # TODO: this assumes there's a where clause...bad
+ if wherein and isinstance(wherein, types.DictionaryType):
+ keys_in = ["%s in (%s) " % (field, ','.join(where))
+ for field, where in wherein.iteritems()]
+ cmd.append(' and '+' and '.join(keys_in))
- if group_by:
- cmd.append(' GROUP BY ' + group_by)
+ if group_by:
+ cmd.append(' GROUP BY ' + group_by)
- self.dprint('%s %s' % (' '.join(cmd), values))
+ self.dprint('%s %s' % (' '.join(cmd), values))
- # create a re-runable function for executing the query
- def exec_sql():
- sql = ' '.join(cmd)
- numRec = self.cur.execute(sql, values)
- if max_rows != None and numRec > max_rows:
- msg = 'Exceeded allowed number of records'
- raise MySQLTooManyRows(msg)
- return self.cur.fetchall()
+ # create a re-runable function for executing the query
+ def exec_sql():
+ sql = ' '.join(cmd)
+ numRec = self.cur.execute(sql, values)
+ if max_rows != None and numRec > max_rows:
+ msg = 'Exceeded allowed number of records'
+ raise MySQLTooManyRows(msg)
+ return self.cur.fetchall()
- # run the query, re-trying after operational errors
- if self.autocommit:
- return self.run_with_retry(exec_sql)
- else:
- return exec_sql()
+ # run the query, re-trying after operational errors
+ if self.autocommit:
+ return self.run_with_retry(exec_sql)
+ else:
+ return exec_sql()
- def select_sql(self, fields, table, sql, values):
- """\
- select fields from table "sql"
- """
- cmd = 'select %s from %s %s' % (fields, table, sql)
- self.dprint(cmd)
-
- # create a -re-runable function for executing the query
- def exec_sql():
- self.cur.execute(cmd, values)
- return self.cur.fetchall()
-
- # run the query, re-trying after operational errors
- if self.autocommit:
- return self.run_with_retry(exec_sql)
- else:
- return exec_sql()
-
-
- def _exec_sql_with_commit(self, sql, values, commit):
- if self.autocommit:
- # re-run the query until it succeeds
- def exec_sql():
- self.cur.execute(sql, values)
- self.con.commit()
- self.run_with_retry(exec_sql)
- else:
- # take one shot at running the query
- self.cur.execute(sql, values)
- if commit:
- self.con.commit()
-
-
- def insert(self, table, data, commit = None):
- """\
- 'insert into table (keys) values (%s ... %s)', values
-
- data:
- dictionary of fields and data
- """
- fields = data.keys()
- refs = ['%s' for field in fields]
- values = [data[field] for field in fields]
- cmd = 'insert into %s (%s) values (%s)' % \
- (table, ','.join(fields), ','.join(refs))
- self.dprint('%s %s' % (cmd, values))
-
- self._exec_sql_with_commit(cmd, values, commit)
-
-
- def delete(self, table, where, commit = None):
- cmd = ['delete from', table]
- if commit == None:
- commit = self.autocommit
- if where and isinstance(where, types.DictionaryType):
- keys = [field + '=%s' for field in where.keys()]
- values = [where[field] for field in where.keys()]
- cmd += ['where', ' and '.join(keys)]
- sql = ' '.join(cmd)
- self.dprint('%s %s' % (sql, values))
-
- self._exec_sql_with_commit(sql, values, commit)
-
-
- def update(self, table, data, where, commit = None):
- """\
- 'update table set data values (%s ... %s) where ...'
-
- data:
- dictionary of fields and data
- """
- if commit == None:
- commit = self.autocommit
- cmd = 'update %s ' % table
- fields = data.keys()
- data_refs = [field + '=%s' for field in fields]
- data_values = [data[field] for field in fields]
- cmd += ' set ' + ' and '.join(data_refs)
-
- where_keys = [field + '=%s' for field in where.keys()]
- where_values = [where[field] for field in where.keys()]
- cmd += ' where ' + ' and '.join(where_keys)
-
- values = data_values + where_values
- print '%s %s' % (cmd, values)
-
- self._exec_sql_with_commit(cmd, values, commit)
-
+ def select_sql(self, fields, table, sql, values):
+ """\
+ select fields from table "sql"
+ """
+ cmd = 'select %s from %s %s' % (fields, table, sql)
+ self.dprint(cmd)
- def delete_job(self, tag, commit = None):
- job_idx = self.find_job(tag)
- for test_idx in self.find_tests(job_idx):
- where = {'test_idx' : test_idx}
- self.delete('iteration_result', where)
- self.delete('test_attributes', where)
- where = {'job_idx' : job_idx}
- self.delete('tests', where)
- self.delete('jobs', where)
+ # create a -re-runable function for executing the query
+ def exec_sql():
+ self.cur.execute(cmd, values)
+ return self.cur.fetchall()
+ # run the query, re-trying after operational errors
+ if self.autocommit:
+ return self.run_with_retry(exec_sql)
+ else:
+ return exec_sql()
- def insert_job(self, tag, job, commit = None):
- job.machine_idx = self.lookup_machine(job.machine)
- if not job.machine_idx:
- job.machine_idx = self.insert_machine(job,
- commit=commit)
- self.insert('jobs', {'tag':tag,
- 'label': job.label,
- 'username': job.user,
- 'machine_idx': job.machine_idx,
- 'queued_time': job.queued_time,
- 'started_time': job.started_time,
- 'finished_time': job.finished_time},
- commit=commit)
- job.index = self.get_last_autonumber_value()
- for test in job.tests:
- self.insert_test(job, test, commit=commit)
+ def _exec_sql_with_commit(self, sql, values, commit):
+ if self.autocommit:
+ # re-run the query until it succeeds
+ def exec_sql():
+ self.cur.execute(sql, values)
+ self.con.commit()
+ self.run_with_retry(exec_sql)
+ else:
+ # take one shot at running the query
+ self.cur.execute(sql, values)
+ if commit:
+ self.con.commit()
- def insert_test(self, job, test, commit = None):
- kver = self.insert_kernel(test.kernel, commit=commit)
- data = {'job_idx':job.index, 'test':test.testname,
- 'subdir':test.subdir, 'kernel_idx':kver,
- 'status':self.status_idx[test.status],
- 'reason':test.reason, 'machine_idx':job.machine_idx,
- 'started_time': test.started_time,
- 'finished_time':test.finished_time}
- self.insert('tests', data, commit=commit)
- test_idx = self.get_last_autonumber_value()
- data = { 'test_idx':test_idx }
- for i in test.iterations:
- data['iteration'] = i.index
- for key, value in i.attr_keyval.iteritems():
- data['attribute'] = key
- data['value'] = value
- self.insert('iteration_attributes', data,
- commit=commit)
- for key, value in i.perf_keyval.iteritems():
- data['attribute'] = key
- data['value'] = value
- self.insert('iteration_result', data,
- commit=commit)
+ def insert(self, table, data, commit = None):
+ """\
+ 'insert into table (keys) values (%s ... %s)', values
- for key, value in test.attributes.iteritems():
- data = {'test_idx': test_idx, 'attribute': key,
- 'value': value}
- self.insert('test_attributes', data, commit=commit)
+ data:
+ dictionary of fields and data
+ """
+ fields = data.keys()
+ refs = ['%s' for field in fields]
+ values = [data[field] for field in fields]
+ cmd = 'insert into %s (%s) values (%s)' % \
+ (table, ','.join(fields), ','.join(refs))
+ self.dprint('%s %s' % (cmd, values))
+ self._exec_sql_with_commit(cmd, values, commit)
- def read_machine_map(self):
- self.machine_group = {}
- for line in open(self.machine_map, 'r').readlines():
- (machine, group) = line.split()
- self.machine_group[machine] = group
+ def delete(self, table, where, commit = None):
+ cmd = ['delete from', table]
+ if commit == None:
+ commit = self.autocommit
+ if where and isinstance(where, types.DictionaryType):
+ keys = [field + '=%s' for field in where.keys()]
+ values = [where[field] for field in where.keys()]
+ cmd += ['where', ' and '.join(keys)]
+ sql = ' '.join(cmd)
+ self.dprint('%s %s' % (sql, values))
- def insert_machine(self, job, group = None, commit = None):
- hostname = job.machine
- if self.machine_map and not self.machine_group:
- self.read_machine_map()
+ self._exec_sql_with_commit(sql, values, commit)
- if not group:
- group = self.machine_group.get(hostname, hostname)
- if group == hostname and job.machine_owner:
- group = job.machine_owner + '/' + hostname
- self.insert('machines',
- { 'hostname' : hostname ,
- 'machine_group' : group ,
- 'owner' : job.machine_owner },
- commit=commit)
- return self.get_last_autonumber_value()
+ def update(self, table, data, where, commit = None):
+ """\
+ 'update table set data values (%s ... %s) where ...'
+ data:
+ dictionary of fields and data
+ """
+ if commit == None:
+ commit = self.autocommit
+ cmd = 'update %s ' % table
+ fields = data.keys()
+ data_refs = [field + '=%s' for field in fields]
+ data_values = [data[field] for field in fields]
+ cmd += ' set ' + ' and '.join(data_refs)
- def lookup_machine(self, hostname):
- where = { 'hostname' : hostname }
- rows = self.select('machine_idx', 'machines', where)
- if rows:
- return rows[0][0]
- else:
- return None
+ where_keys = [field + '=%s' for field in where.keys()]
+ where_values = [where[field] for field in where.keys()]
+ cmd += ' where ' + ' and '.join(where_keys)
+ values = data_values + where_values
+ print '%s %s' % (cmd, values)
- def lookup_kernel(self, kernel):
- rows = self.select('kernel_idx', 'kernels',
- {'kernel_hash':kernel.kernel_hash})
- if rows:
- return rows[0][0]
- else:
- return None
+ self._exec_sql_with_commit(cmd, values, commit)
- def insert_kernel(self, kernel, commit = None):
- kver = self.lookup_kernel(kernel)
- if kver:
- return kver
-
- # If this kernel has any significant patches, append their hash
- # as diferentiator.
- printable = kernel.base
- patch_count = 0
- for patch in kernel.patches:
- match = re.match(r'.*(-mm[0-9]+|-git[0-9]+)\.(bz2|gz)$',
- patch.reference)
- if not match:
- patch_count += 1
+ def delete_job(self, tag, commit = None):
+ job_idx = self.find_job(tag)
+ for test_idx in self.find_tests(job_idx):
+ where = {'test_idx' : test_idx}
+ self.delete('iteration_result', where)
+ self.delete('test_attributes', where)
+ where = {'job_idx' : job_idx}
+ self.delete('tests', where)
+ self.delete('jobs', where)
- self.insert('kernels',
- {'base':kernel.base,
- 'kernel_hash':kernel.kernel_hash,
- 'printable':printable},
- commit=commit)
- kver = self.get_last_autonumber_value()
- if patch_count > 0:
- printable += ' p%d' % (kver)
- self.update('kernels',
- {'printable':printable},
- {'kernel_idx':kver})
+ def insert_job(self, tag, job, commit = None):
+ job.machine_idx = self.lookup_machine(job.machine)
+ if not job.machine_idx:
+ job.machine_idx = self.insert_machine(job,
+ commit=commit)
+ self.insert('jobs', {'tag':tag,
+ 'label': job.label,
+ 'username': job.user,
+ 'machine_idx': job.machine_idx,
+ 'queued_time': job.queued_time,
+ 'started_time': job.started_time,
+ 'finished_time': job.finished_time},
+ commit=commit)
+ job.index = self.get_last_autonumber_value()
+ for test in job.tests:
+ self.insert_test(job, test, commit=commit)
- for patch in kernel.patches:
- self.insert_patch(kver, patch, commit=commit)
- return kver
+ def insert_test(self, job, test, commit = None):
+ kver = self.insert_kernel(test.kernel, commit=commit)
+ data = {'job_idx':job.index, 'test':test.testname,
+ 'subdir':test.subdir, 'kernel_idx':kver,
+ 'status':self.status_idx[test.status],
+ 'reason':test.reason, 'machine_idx':job.machine_idx,
+ 'started_time': test.started_time,
+ 'finished_time':test.finished_time}
+ self.insert('tests', data, commit=commit)
+ test_idx = self.get_last_autonumber_value()
+ data = { 'test_idx':test_idx }
- def insert_patch(self, kver, patch, commit = None):
- print patch.reference
- name = os.path.basename(patch.reference)[:80]
- self.insert('patches',
- {'kernel_idx': kver,
- 'name':name,
- 'url':patch.reference,
- 'hash':patch.hash},
+ for i in test.iterations:
+ data['iteration'] = i.index
+ for key, value in i.attr_keyval.iteritems():
+ data['attribute'] = key
+ data['value'] = value
+ self.insert('iteration_attributes', data,
+ commit=commit)
+ for key, value in i.perf_keyval.iteritems():
+ data['attribute'] = key
+ data['value'] = value
+ self.insert('iteration_result', data,
commit=commit)
-
- def find_test(self, job_idx, subdir):
- where = { 'job_idx':job_idx , 'subdir':subdir }
- rows = self.select('test_idx', 'tests', where)
- if rows:
- return rows[0][0]
- else:
- return None
+ for key, value in test.attributes.iteritems():
+ data = {'test_idx': test_idx, 'attribute': key,
+ 'value': value}
+ self.insert('test_attributes', data, commit=commit)
- def find_tests(self, job_idx):
- where = { 'job_idx':job_idx }
- rows = self.select('test_idx', 'tests', where)
- if rows:
- return [row[0] for row in rows]
- else:
- return []
+ def read_machine_map(self):
+ self.machine_group = {}
+ for line in open(self.machine_map, 'r').readlines():
+ (machine, group) = line.split()
+ self.machine_group[machine] = group
- def find_job(self, tag):
- rows = self.select('job_idx', 'jobs', {'tag': tag})
- if rows:
- return rows[0][0]
- else:
- return None
+ def insert_machine(self, job, group = None, commit = None):
+ hostname = job.machine
+ if self.machine_map and not self.machine_group:
+ self.read_machine_map()
+
+ if not group:
+ group = self.machine_group.get(hostname, hostname)
+ if group == hostname and job.machine_owner:
+ group = job.machine_owner + '/' + hostname
+
+ self.insert('machines',
+ { 'hostname' : hostname ,
+ 'machine_group' : group ,
+ 'owner' : job.machine_owner },
+ commit=commit)
+ return self.get_last_autonumber_value()
+
+
+ def lookup_machine(self, hostname):
+ where = { 'hostname' : hostname }
+ rows = self.select('machine_idx', 'machines', where)
+ if rows:
+ return rows[0][0]
+ else:
+ return None
+
+
+ def lookup_kernel(self, kernel):
+ rows = self.select('kernel_idx', 'kernels',
+ {'kernel_hash':kernel.kernel_hash})
+ if rows:
+ return rows[0][0]
+ else:
+ return None
+
+
+ def insert_kernel(self, kernel, commit = None):
+ kver = self.lookup_kernel(kernel)
+ if kver:
+ return kver
+
+ # If this kernel has any significant patches, append their hash
+ # as diferentiator.
+ printable = kernel.base
+ patch_count = 0
+ for patch in kernel.patches:
+ match = re.match(r'.*(-mm[0-9]+|-git[0-9]+)\.(bz2|gz)$',
+ patch.reference)
+ if not match:
+ patch_count += 1
+
+ self.insert('kernels',
+ {'base':kernel.base,
+ 'kernel_hash':kernel.kernel_hash,
+ 'printable':printable},
+ commit=commit)
+ kver = self.get_last_autonumber_value()
+
+ if patch_count > 0:
+ printable += ' p%d' % (kver)
+ self.update('kernels',
+ {'printable':printable},
+ {'kernel_idx':kver})
+
+ for patch in kernel.patches:
+ self.insert_patch(kver, patch, commit=commit)
+ return kver
+
+
+ def insert_patch(self, kver, patch, commit = None):
+ print patch.reference
+ name = os.path.basename(patch.reference)[:80]
+ self.insert('patches',
+ {'kernel_idx': kver,
+ 'name':name,
+ 'url':patch.reference,
+ 'hash':patch.hash},
+ commit=commit)
+
+
+ def find_test(self, job_idx, subdir):
+ where = { 'job_idx':job_idx , 'subdir':subdir }
+ rows = self.select('test_idx', 'tests', where)
+ if rows:
+ return rows[0][0]
+ else:
+ return None
+
+
+ def find_tests(self, job_idx):
+ where = { 'job_idx':job_idx }
+ rows = self.select('test_idx', 'tests', where)
+ if rows:
+ return [row[0] for row in rows]
+ else:
+ return []
+
+
+ def find_job(self, tag):
+ rows = self.select('job_idx', 'jobs', {'tag': tag})
+ if rows:
+ return rows[0][0]
+ else:
+ return None
def _get_db_type():
- """Get the database type name to use from the global config."""
- get_value = global_config.global_config.get_config_value
- return "db_" + get_value("TKO", "db_type", default="mysql")
+ """Get the database type name to use from the global config."""
+ get_value = global_config.global_config.get_config_value
+ return "db_" + get_value("TKO", "db_type", default="mysql")
def _get_error_class(class_name):
- """Retrieves the appropriate error class by name from the database
- module."""
- db_module = __import__("autotest_lib.tko." + _get_db_type(),
- globals(), locals(), ["driver"])
- return getattr(db_module.driver, class_name)
+ """Retrieves the appropriate error class by name from the database
+ module."""
+ db_module = __import__("autotest_lib.tko." + _get_db_type(),
+ globals(), locals(), ["driver"])
+ return getattr(db_module.driver, class_name)
def db(*args, **dargs):
- """Creates an instance of the database class with the arguments
- provided in args and dargs, using the database type specified by
- the global configuration (defaulting to mysql)."""
- db_type = _get_db_type()
- db_module = __import__("autotest_lib.tko." + db_type, globals(),
- locals(), [db_type])
- db = getattr(db_module, db_type)(*args, **dargs)
- return db
+ """Creates an instance of the database class with the arguments
+ provided in args and dargs, using the database type specified by
+ the global configuration (defaulting to mysql)."""
+ db_type = _get_db_type()
+ db_module = __import__("autotest_lib.tko." + db_type, globals(),
+ locals(), [db_type])
+ db = getattr(db_module, db_type)(*args, **dargs)
+ return db
diff --git a/tko/db_mysql.py b/tko/db_mysql.py
index da59ecd..13c2037 100644
--- a/tko/db_mysql.py
+++ b/tko/db_mysql.py
@@ -2,6 +2,6 @@
import db
class db_mysql(db.db_sql):
- def connect(self, host, database, user, password):
- return driver.connect(host=host, user=user,
- passwd=password, db=database)
+ def connect(self, host, database, user, password):
+ return driver.connect(host=host, user=user,
+ passwd=password, db=database)
diff --git a/tko/db_postgres.py b/tko/db_postgres.py
index 166ee02..31834fa 100644
--- a/tko/db_postgres.py
+++ b/tko/db_postgres.py
@@ -2,6 +2,6 @@
import db
class db_postgres(db.db_sql):
- def connect(self, host, database, user, password):
- return driver.connect("dbname=%s user=%s password=%s" % \
- (database, user, password))
+ def connect(self, host, database, user, password):
+ return driver.connect("dbname=%s user=%s password=%s" % \
+ (database, user, password))
diff --git a/tko/delete_job_results b/tko/delete_job_results
index 4977988..f747510 100644
--- a/tko/delete_job_results
+++ b/tko/delete_job_results
@@ -7,15 +7,14 @@
usage = "usage: delete_job_results <job tag>"
if len(sys.argv) < 2:
- print usage
- sys.exit(2)
+ print usage
+ sys.exit(2)
tag = sys.argv[1]
resultsdir = os.path.abspath(os.path.join(thisdir, '../results', tag))
db = db.db()
if not db.find_job(tag):
- raise "Job tag %s does not exist in database" % tag
+ raise "Job tag %s does not exist in database" % tag
db.delete_job(tag)
shutil.rmtree(resultsdir)
-
diff --git a/tko/display.py b/tko/display.py
index ccf1923..f233fb8 100755
--- a/tko/display.py
+++ b/tko/display.py
@@ -2,23 +2,23 @@
import frontend, reason_qualifier
color_map = {
- 'header' : '#e5e5c0', # greyish yellow
- 'blank' : '#ffffff', # white
- 'plain_text' : '#e5e5c0', # greyish yellow
- 'borders' : '#bbbbbb', # grey
- 'white' : '#ffffff', # white
- 'green' : '#66ff66', # green
- 'yellow' : '#fffc00', # yellow
- 'red' : '#ff6666', # red
+ 'header' : '#e5e5c0', # greyish yellow
+ 'blank' : '#ffffff', # white
+ 'plain_text' : '#e5e5c0', # greyish yellow
+ 'borders' : '#bbbbbb', # grey
+ 'white' : '#ffffff', # white
+ 'green' : '#66ff66', # green
+ 'yellow' : '#fffc00', # yellow
+ 'red' : '#ff6666', # red
- #### additional keys for shaded color of a box
- #### depending on stats of GOOD/FAIL
- '100pct' : '#32CD32', # green, 94% to 100% of success
- '95pct' : '#c0ff80', # step twrds yellow, 88% to 94% of success
- '90pct' : '#ffff00', # yellow, 82% to 88%
- '85pct' : '#ffc040', # 76% to 82%
- '75pct' : '#ff4040', # red, 1% to 76%
- '0pct' : '#d080d0', # violet, <1% of success
+ #### additional keys for shaded color of a box
+ #### depending on stats of GOOD/FAIL
+ '100pct' : '#32CD32', # green, 94% to 100% of success
+ '95pct' : '#c0ff80', # step twrds yellow, 88% to 94% of success
+ '90pct' : '#ffff00', # yellow, 82% to 88%
+ '85pct' : '#ffc040', # 76% to 82%
+ '75pct' : '#ff4040', # red, 1% to 76%
+ '0pct' : '#d080d0', # violet, <1% of success
}
@@ -26,316 +26,315 @@
def set_brief_mode():
- global _brief_mode
- _brief_mode = True
+ global _brief_mode
+ _brief_mode = True
def is_brief_mode():
- return _brief_mode
+ return _brief_mode
def color_keys_row():
- """ Returns one row table with samples of 'NNpct' colors
- defined in the color_map
- and numbers of corresponding %%
- """
- ### This function does not require maintenance in case of
- ### color_map augmenting - as long as
- ### color keys for box shading have names that end with 'pct'
- keys = filter(lambda key: key.endswith('pct'), color_map.keys())
- def num_pct(key):
- return int(key.replace('pct',''))
- keys.sort(key=num_pct)
- html = ''
- for key in keys:
- html+= "\t\t\t<td bgcolor =%s> </td>\n"\
- % color_map[key]
- hint = key.replace('pct',' %')
- if hint[0]<>'0': ## anything but 0 %
- hint = 'to ' + hint
- html+= "\t\t\t<td> %s </td>\n" % hint
+ """ Returns one row table with samples of 'NNpct' colors
+ defined in the color_map
+ and numbers of corresponding %%
+ """
+ ### This function does not require maintenance in case of
+ ### color_map augmenting - as long as
+ ### color keys for box shading have names that end with 'pct'
+ keys = filter(lambda key: key.endswith('pct'), color_map.keys())
+ def num_pct(key):
+ return int(key.replace('pct',''))
+ keys.sort(key=num_pct)
+ html = ''
+ for key in keys:
+ html+= "\t\t\t<td bgcolor =%s> </td>\n"\
+ % color_map[key]
+ hint = key.replace('pct',' %')
+ if hint[0]<>'0': ## anything but 0 %
+ hint = 'to ' + hint
+ html+= "\t\t\t<td> %s </td>\n" % hint
- html = """
+ html = """
<table width = "500" border="0" cellpadding="2" cellspacing="2">\n
- <tbody>\n
- <tr>\n
+ <tbody>\n
+ <tr>\n
%s
- </tr>\n
- </tbody>
+ </tr>\n
+ </tbody>
</table><br>
""" % html
- return html
+ return html
def calculate_html(link, data, tooltip=None, row_label=None, column_label=None):
- if not is_brief_mode():
- hover_text = '%s:%s' % (row_label, column_label)
- if data: ## cell is not empty
- hover_text += '<br>%s' % tooltip
- else:
- ## avoid "None" printed in empty cells
- data = ' '
- html = ('<center><a class="info" href="%s">'
- '%s<span>%s</span></a></center>' %
- (link, data, hover_text))
- return html
- # no hover if embedded into AFE but links shall redirect to new window
- if data: ## cell is non empty
- html = '<a href="%s" target=NEW>%s</a>' % (link, data)
- return html
- else: ## cell is empty
- return ' '
+ if not is_brief_mode():
+ hover_text = '%s:%s' % (row_label, column_label)
+ if data: ## cell is not empty
+ hover_text += '<br>%s' % tooltip
+ else:
+ ## avoid "None" printed in empty cells
+ data = ' '
+ html = ('<center><a class="info" href="%s">'
+ '%s<span>%s</span></a></center>' %
+ (link, data, hover_text))
+ return html
+ # no hover if embedded into AFE but links shall redirect to new window
+ if data: ## cell is non empty
+ html = '<a href="%s" target=NEW>%s</a>' % (link, data)
+ return html
+ else: ## cell is empty
+ return ' '
class box:
- def __init__(self, data, color_key = None, header = False, link = None,
- tooltip = None, row_label = None, column_label = None):
-
- ## in brief mode we display grid table only and nothing more
- ## - mouse hovering feature is stubbed in brief mode
- ## - any link opens new window or tab
+ def __init__(self, data, color_key = None, header = False, link = None,
+ tooltip = None, row_label = None, column_label = None):
- redirect = ""
- if is_brief_mode():
- ## we are acting under AFE
- ## any link shall open new window
- redirect = " target=NEW"
-
- if data:
- data = "<tt>%s</tt>" % data
-
- if link and not tooltip:
- ## FlipAxis corner, column and row headers
- self.data = ('<a href="%s"%s>%s</a>' %
- (link, redirect, data))
- else:
- self.data = calculate_html(link, data, tooltip,
- row_label, column_label)
+ ## in brief mode we display grid table only and nothing more
+ ## - mouse hovering feature is stubbed in brief mode
+ ## - any link opens new window or tab
- if color_map.has_key(color_key):
- self.color = color_map[color_key]
- elif header:
- self.color = color_map['header']
- elif data:
- self.color = color_map['plain_text']
- else:
- self.color = color_map['blank']
- self.header = header
+ redirect = ""
+ if is_brief_mode():
+ ## we are acting under AFE
+ ## any link shall open new window
+ redirect = " target=NEW"
+
+ if data:
+ data = "<tt>%s</tt>" % data
+
+ if link and not tooltip:
+ ## FlipAxis corner, column and row headers
+ self.data = ('<a href="%s"%s>%s</a>' %
+ (link, redirect, data))
+ else:
+ self.data = calculate_html(link, data, tooltip,
+ row_label, column_label)
+
+ if color_map.has_key(color_key):
+ self.color = color_map[color_key]
+ elif header:
+ self.color = color_map['header']
+ elif data:
+ self.color = color_map['plain_text']
+ else:
+ self.color = color_map['blank']
+ self.header = header
- def html(self):
- if self.data:
- data = self.data
- else:
- data = ' '
+ def html(self):
+ if self.data:
+ data = self.data
+ else:
+ data = ' '
- if self.header:
- box_html = 'th'
- else:
- box_html = 'td'
+ if self.header:
+ box_html = 'th'
+ else:
+ box_html = 'td'
- return "<%s bgcolor=%s>%s</%s>" % \
- (box_html, self.color, data, box_html)
+ return "<%s bgcolor=%s>%s</%s>" % \
+ (box_html, self.color, data, box_html)
def grade_from_status(status):
- # % of goodness
- # GOOD (6) -> 1
- # TEST_NA (8) is not counted
- # ## If the test doesn't PASS, it FAILS
- # else -> 0
+ # % of goodness
+ # GOOD (6) -> 1
+ # TEST_NA (8) is not counted
+ # ## If the test doesn't PASS, it FAILS
+ # else -> 0
- if status == 6:
- return 1.0
- else:
- return 0.0
+ if status == 6:
+ return 1.0
+ else:
+ return 0.0
def average_grade_from_status_count(status_count):
- average_grade = 0
- total_count = 0
- for key in status_count.keys():
- if key != 8: # TEST_NA status
- average_grade += (grade_from_status(key)
- * status_count[key])
- total_count += status_count[key]
- if total_count != 0:
- average_grade = average_grade / total_count
- else:
- average_grade = 0.0
- return average_grade
+ average_grade = 0
+ total_count = 0
+ for key in status_count.keys():
+ if key != 8: # TEST_NA status
+ average_grade += (grade_from_status(key)
+ * status_count[key])
+ total_count += status_count[key]
+ if total_count != 0:
+ average_grade = average_grade / total_count
+ else:
+ average_grade = 0.0
+ return average_grade
def shade_from_status_count(status_count):
- if not status_count:
- return None
-
- ## average_grade defines a shade of the box
- ## 0 -> violet
- ## 0.76 -> red
- ## 0.88-> yellow
- ## 1.0 -> green
- average_grade = average_grade_from_status_count(status_count)
-
- ## find appropiate keyword from color_map
- if average_grade<0.01:
- shade = '0pct'
- elif average_grade<0.75:
- shade = '75pct'
- elif average_grade<0.85:
- shade = '85pct'
- elif average_grade<0.90:
- shade = '90pct'
- elif average_grade<0.95:
- shade = '95pct'
- else:
- shade = '100pct'
-
- return shade
+ if not status_count:
+ return None
+
+ ## average_grade defines a shade of the box
+ ## 0 -> violet
+ ## 0.76 -> red
+ ## 0.88-> yellow
+ ## 1.0 -> green
+ average_grade = average_grade_from_status_count(status_count)
+
+ ## find appropiate keyword from color_map
+ if average_grade<0.01:
+ shade = '0pct'
+ elif average_grade<0.75:
+ shade = '75pct'
+ elif average_grade<0.85:
+ shade = '85pct'
+ elif average_grade<0.90:
+ shade = '90pct'
+ elif average_grade<0.95:
+ shade = '95pct'
+ else:
+ shade = '100pct'
+
+ return shade
def status_html(db, box_data, shade):
- """
- status_count: dict mapping from status (integer key) to count
- eg. { 'GOOD' : 4, 'FAIL' : 1 }
- """
- status_count = box_data.status_count
- if 6 in status_count.keys():
- html = "%d / %d " \
- %(status_count[6],sum(status_count.values()))
- else:
- html = "%d / %d " % \
- (0, sum(status_count.values()))
+ """
+ status_count: dict mapping from status (integer key) to count
+ eg. { 'GOOD' : 4, 'FAIL' : 1 }
+ """
+ status_count = box_data.status_count
+ if 6 in status_count.keys():
+ html = "%d / %d " \
+ %(status_count[6],sum(status_count.values()))
+ else:
+ html = "%d / %d " % \
+ (0, sum(status_count.values()))
- if box_data.reasons_list:
- reasons_list = box_data.reasons_list
- aggregated_reasons_list = \
- reason_qualifier.aggregate_reason_fields(reasons_list)
- for reason in aggregated_reasons_list:
- ## a bit of more postprocessing
- ## to look nicer in a cell
- ## in future: to do subtable within the cell
- reason = reason.replace('<br>','\n')
- reason = reason.replace('<','[').replace('>',']')
- reason = reason.replace('|','\n').replace('&',' AND ')
- reason = reason.replace('\n','<br>')
- html += '<br>' + reason
+ if box_data.reasons_list:
+ reasons_list = box_data.reasons_list
+ aggregated_reasons_list = \
+ reason_qualifier.aggregate_reason_fields(reasons_list)
+ for reason in aggregated_reasons_list:
+ ## a bit of more postprocessing
+ ## to look nicer in a cell
+ ## in future: to do subtable within the cell
+ reason = reason.replace('<br>','\n')
+ reason = reason.replace('<','[').replace('>',']')
+ reason = reason.replace('|','\n').replace('&',' AND ')
+ reason = reason.replace('\n','<br>')
+ html += '<br>' + reason
- tooltip = ""
- for status in sorted(status_count.keys(), reverse = True):
- status_word = db.status_word[status]
- tooltip += "%d %s " % (status_count[status], status_word)
- return (html,tooltip)
+ tooltip = ""
+ for status in sorted(status_count.keys(), reverse = True):
+ status_word = db.status_word[status]
+ tooltip += "%d %s " % (status_count[status], status_word)
+ return (html,tooltip)
def status_count_box(db, tests, link = None):
- """
- Display a ratio of total number of GOOD tests
- to total number of all tests in the group of tests.
- More info (e.g. 10 GOOD, 2 WARN, 3 FAIL) is in tooltips
- """
- if not tests:
- return box(None, None)
+ """
+ Display a ratio of total number of GOOD tests
+ to total number of all tests in the group of tests.
+ More info (e.g. 10 GOOD, 2 WARN, 3 FAIL) is in tooltips
+ """
+ if not tests:
+ return box(None, None)
- status_count = {}
- for test in tests:
- count = status_count.get(test.status_num, 0)
- status_count[test.status_num] = count + 1
- return status_precounted_box(db, status_count, link)
+ status_count = {}
+ for test in tests:
+ count = status_count.get(test.status_num, 0)
+ status_count[test.status_num] = count + 1
+ return status_precounted_box(db, status_count, link)
def status_precounted_box(db, box_data, link = None,
- x_label = None, y_label = None):
- """
- Display a ratio of total number of GOOD tests
- to total number of all tests in the group of tests.
- More info (e.g. 10 GOOD, 2 WARN, 3 FAIL) is in tooltips
- """
- status_count = box_data.status_count
- if not status_count:
- return box(None, None)
-
- shade = shade_from_status_count(status_count)
- html,tooltip = status_html(db, box_data, shade)
- precounted_box = box(html, shade, False, link, tooltip,
- x_label, y_label)
- return precounted_box
+ x_label = None, y_label = None):
+ """
+ Display a ratio of total number of GOOD tests
+ to total number of all tests in the group of tests.
+ More info (e.g. 10 GOOD, 2 WARN, 3 FAIL) is in tooltips
+ """
+ status_count = box_data.status_count
+ if not status_count:
+ return box(None, None)
+
+ shade = shade_from_status_count(status_count)
+ html,tooltip = status_html(db, box_data, shade)
+ precounted_box = box(html, shade, False, link, tooltip,
+ x_label, y_label)
+ return precounted_box
def print_table(matrix):
- """
- matrix: list of lists of boxes, giving a matrix of data
- Each of the inner lists is a row, not a column.
+ """
+ matrix: list of lists of boxes, giving a matrix of data
+ Each of the inner lists is a row, not a column.
- Display the given matrix of data as a table.
- """
+ Display the given matrix of data as a table.
+ """
- print ('<table bgcolor="%s" cellspacing="1" cellpadding="5" '
- 'style="margin-right: 200px;">') % (
- color_map['borders'])
- for row in matrix:
- print '<tr>'
- for element in row:
- print element.html()
- print '</tr>'
- print '</table>'
+ print ('<table bgcolor="%s" cellspacing="1" cellpadding="5" '
+ 'style="margin-right: 200px;">') % (
+ color_map['borders'])
+ for row in matrix:
+ print '<tr>'
+ for element in row:
+ print element.html()
+ print '</tr>'
+ print '</table>'
def sort_tests(tests):
- kernel_order = ['patch', 'config', 'build', 'mkinitrd', 'install']
+ kernel_order = ['patch', 'config', 'build', 'mkinitrd', 'install']
- results = []
- for kernel_op in kernel_order:
- test = 'kernel.' + kernel_op
- if tests.count(test):
- results.append(test)
- tests.remove(test)
- if tests.count('boot'):
- results.append('boot')
- tests.remove('boot')
- return results + sorted(tests)
+ results = []
+ for kernel_op in kernel_order:
+ test = 'kernel.' + kernel_op
+ if tests.count(test):
+ results.append(test)
+ tests.remove(test)
+ if tests.count('boot'):
+ results.append('boot')
+ tests.remove('boot')
+ return results + sorted(tests)
def print_main_header():
- hover_css="""\
+ hover_css="""\
a.info{
- position:relative; /*this is the key*/
- z-index:1
- color:#000;
- text-decoration:none}
+position:relative; /*this is the key*/
+z-index:1
+color:#000;
+text-decoration:none}
a.info:hover{z-index:25;}
a.info span{display: none}
a.info:hover span{ /*the span will display just on :hover state*/
- display:block;
- position:absolute;
- top:1em; left:1em;
- min-width: 100px;
- overflow: visible;
- border:1px solid #036;
- background-color:#fff; color:#000;
- text-align: left
+display:block;
+position:absolute;
+top:1em; left:1em;
+min-width: 100px;
+overflow: visible;
+border:1px solid #036;
+background-color:#fff; color:#000;
+text-align: left
}
"""
- print '<head><style type="text/css">'
- print 'a { text-decoration: none }'
- print hover_css
- print '</style></head>'
- print '<h2>'
- print '<a href="compose_query.cgi">Functional</a>'
- print '   '
- print '<a href="machine_benchmark.cgi">Performance</a>'
- print '   '
- print '<a href="http://test.kernel.org/autotest">[about Autotest]</a>'
- print '</h2><p>'
+ print '<head><style type="text/css">'
+ print 'a { text-decoration: none }'
+ print hover_css
+ print '</style></head>'
+ print '<h2>'
+ print '<a href="compose_query.cgi">Functional</a>'
+ print '   '
+ print '<a href="machine_benchmark.cgi">Performance</a>'
+ print '   '
+ print '<a href="http://test.kernel.org/autotest">[about Autotest]</a>'
+ print '</h2><p>'
def group_name(group):
- name = re.sub('_', '<br>', group.name)
- if re.search('/', name):
- (owner, machine) = name.split('/', 1)
- name = owner + '<br>' + machine
- return name
-
+ name = re.sub('_', '<br>', group.name)
+ if re.search('/', name):
+ (owner, machine) = name.split('/', 1)
+ name = owner + '<br>' + machine
+ return name
diff --git a/tko/frontend.py b/tko/frontend.py
index 00e7fed..45de298 100755
--- a/tko/frontend.py
+++ b/tko/frontend.py
@@ -10,299 +10,299 @@
root_url_file = os.path.join(tko, '.root_url')
if os.path.exists(root_url_file):
- html_root = open(root_url_file, 'r').readline().rstrip()
+ html_root = open(root_url_file, 'r').readline().rstrip()
else:
- html_root = '/results/'
+ html_root = '/results/'
class status_cell:
- # One cell in the matrix of status data.
- def __init__(self):
- # Count is a dictionary: status -> count of tests with status
- self.status_count = {}
- self.reasons_list = []
- self.job_tag = None
- self.job_tag_count = 0
+ # One cell in the matrix of status data.
+ def __init__(self):
+ # Count is a dictionary: status -> count of tests with status
+ self.status_count = {}
+ self.reasons_list = []
+ self.job_tag = None
+ self.job_tag_count = 0
- def add(self, status, count, job_tags, reasons = None):
- assert count > 0
+ def add(self, status, count, job_tags, reasons = None):
+ assert count > 0
- self.job_tag = job_tags
- self.job_tag_count += count
- if self.job_tag_count > 1:
- self.job_tag = None
-
- self.status_count[status] = count
- ### status == 6 means 'GOOD'
- if status != 6:
- ## None implies sorting problems and extra CRs in a cell
- if reasons:
- self.reasons_list.append(reasons)
+ self.job_tag = job_tags
+ self.job_tag_count += count
+ if self.job_tag_count > 1:
+ self.job_tag = None
+
+ self.status_count[status] = count
+ ### status == 6 means 'GOOD'
+ if status != 6:
+ ## None implies sorting problems and extra CRs in a cell
+ if reasons:
+ self.reasons_list.append(reasons)
class status_data:
- def __init__(self, sql_rows, x_field, y_field, query_reasons = False):
- data = {}
- y_values = set()
+ def __init__(self, sql_rows, x_field, y_field, query_reasons = False):
+ data = {}
+ y_values = set()
- # Walk through the query, filing all results by x, y info
- for row in sql_rows:
- if query_reasons:
- (x,y, status, count, job_tags, reasons) = row
- else:
- (x,y, status, count, job_tags) = row
- reasons = None
- if not data.has_key(x):
- data[x] = {}
- if not data[x].has_key(y):
- y_values.add(y)
- data[x][y] = status_cell()
- data[x][y].add(status, count, job_tags, reasons)
+ # Walk through the query, filing all results by x, y info
+ for row in sql_rows:
+ if query_reasons:
+ (x,y, status, count, job_tags, reasons) = row
+ else:
+ (x,y, status, count, job_tags) = row
+ reasons = None
+ if not data.has_key(x):
+ data[x] = {}
+ if not data[x].has_key(y):
+ y_values.add(y)
+ data[x][y] = status_cell()
+ data[x][y].add(status, count, job_tags, reasons)
- # 2-d hash of data - [x-value][y-value]
- self.data = data
- # List of possible columns (x-values)
- self.x_values = smart_sort(data.keys(), x_field)
- # List of rows columns (y-values)
- self.y_values = smart_sort(list(y_values), y_field)
- nCells = len(self.y_values)*len(self.x_values)
- if nCells > MAX_CELLS:
- msg = 'Exceeded allowed number of cells in a table'
- raise db.MySQLTooManyRows(msg)
-
+ # 2-d hash of data - [x-value][y-value]
+ self.data = data
+ # List of possible columns (x-values)
+ self.x_values = smart_sort(data.keys(), x_field)
+ # List of rows columns (y-values)
+ self.y_values = smart_sort(list(y_values), y_field)
+ nCells = len(self.y_values)*len(self.x_values)
+ if nCells > MAX_CELLS:
+ msg = 'Exceeded allowed number of cells in a table'
+ raise db.MySQLTooManyRows(msg)
+
def get_matrix_data(db_obj, x_axis, y_axis, where = None,
- query_reasons = False):
- # Searches on the test_view table - x_axis and y_axis must both be
- # column names in that table.
- x_field = test_view_field_dict[x_axis]
- y_field = test_view_field_dict[y_axis]
- query_fields_list = [x_field, y_field, 'status','COUNT(status)']
- query_fields_list.append("LEFT(GROUP_CONCAT(job_tag),100)")
- if query_reasons:
- query_fields_list.append(
- "LEFT(GROUP_CONCAT(DISTINCT reason SEPARATOR '|'),500)"
- )
- fields = ','.join(query_fields_list)
+ query_reasons = False):
+ # Searches on the test_view table - x_axis and y_axis must both be
+ # column names in that table.
+ x_field = test_view_field_dict[x_axis]
+ y_field = test_view_field_dict[y_axis]
+ query_fields_list = [x_field, y_field, 'status','COUNT(status)']
+ query_fields_list.append("LEFT(GROUP_CONCAT(job_tag),100)")
+ if query_reasons:
+ query_fields_list.append(
+ "LEFT(GROUP_CONCAT(DISTINCT reason SEPARATOR '|'),500)"
+ )
+ fields = ','.join(query_fields_list)
- group_by = '%s, %s, status' % (x_field, y_field)
- rows = db_obj.select(fields, 'test_view',
- where=where, group_by=group_by, max_rows = MAX_RECORDS)
- return status_data(rows, x_field, y_field, query_reasons)
+ group_by = '%s, %s, status' % (x_field, y_field)
+ rows = db_obj.select(fields, 'test_view',
+ where=where, group_by=group_by, max_rows = MAX_RECORDS)
+ return status_data(rows, x_field, y_field, query_reasons)
# Dictionary used simply for fast lookups from short reference names for users
# to fieldnames in test_view
test_view_field_dict = {
- 'kernel' : 'kernel_printable',
- 'hostname' : 'machine_hostname',
- 'test' : 'test',
- 'label' : 'job_label',
- 'machine_group' : 'machine_group',
- 'reason' : 'reason',
- 'tag' : 'job_tag',
- 'user' : 'job_username',
- 'status' : 'status_word',
- 'time' : 'test_finished_time',
- 'time_daily' : 'DATE(test_finished_time)'
+ 'kernel' : 'kernel_printable',
+ 'hostname' : 'machine_hostname',
+ 'test' : 'test',
+ 'label' : 'job_label',
+ 'machine_group' : 'machine_group',
+ 'reason' : 'reason',
+ 'tag' : 'job_tag',
+ 'user' : 'job_username',
+ 'status' : 'status_word',
+ 'time' : 'test_finished_time',
+ 'time_daily' : 'DATE(test_finished_time)'
}
def smart_sort(list, field):
- if field == 'kernel_printable':
- def kernel_encode(kernel):
- return kernel_versions.version_encode(kernel)
- list.sort(key = kernel_encode, reverse = True)
- return list
- ## old records may contain time=None
- ## make None comparable with timestamp datetime or date
- elif field == 'test_finished_time':
- def convert_None_to_datetime(date_time):
- if not date_time:
- return datetime.datetime(1970, 1, 1, 0, 0, 0)
- else:
- return date_time
- list = map(convert_None_to_datetime, list)
- elif field == 'DATE(test_finished_time)':
- def convert_None_to_date(date):
- if not date:
- return datetime.date(1970, 1, 1)
- else:
- return date
- list = map(convert_None_to_date, list)
- list.sort()
- return list
+ if field == 'kernel_printable':
+ def kernel_encode(kernel):
+ return kernel_versions.version_encode(kernel)
+ list.sort(key = kernel_encode, reverse = True)
+ return list
+ ## old records may contain time=None
+ ## make None comparable with timestamp datetime or date
+ elif field == 'test_finished_time':
+ def convert_None_to_datetime(date_time):
+ if not date_time:
+ return datetime.datetime(1970, 1, 1, 0, 0, 0)
+ else:
+ return date_time
+ list = map(convert_None_to_datetime, list)
+ elif field == 'DATE(test_finished_time)':
+ def convert_None_to_date(date):
+ if not date:
+ return datetime.date(1970, 1, 1)
+ else:
+ return date
+ list = map(convert_None_to_date, list)
+ list.sort()
+ return list
class group:
- @classmethod
- def select(klass, db):
- """Return all possible machine groups"""
- rows = db.select('distinct machine_group', 'machines',
- 'machine_group is not null')
- groupnames = sorted([row[0] for row in rows])
- return [klass(db, groupname) for groupname in groupnames]
+ @classmethod
+ def select(klass, db):
+ """Return all possible machine groups"""
+ rows = db.select('distinct machine_group', 'machines',
+ 'machine_group is not null')
+ groupnames = sorted([row[0] for row in rows])
+ return [klass(db, groupname) for groupname in groupnames]
- def __init__(self, db, name):
- self.name = name
- self.db = db
+ def __init__(self, db, name):
+ self.name = name
+ self.db = db
- def machines(self):
- return machine.select(self.db, { 'machine_group' : self.name })
+ def machines(self):
+ return machine.select(self.db, { 'machine_group' : self.name })
- def tests(self, where = {}):
- values = [self.name]
- sql = 't inner join machines m on m.machine_idx=t.machine_idx'
- sql += ' where m.machine_group=%s'
- for key in where.keys():
- sql += ' and %s=%%s' % key
- values.append(where[key])
- return test.select_sql(self.db, sql, values)
+ def tests(self, where = {}):
+ values = [self.name]
+ sql = 't inner join machines m on m.machine_idx=t.machine_idx'
+ sql += ' where m.machine_group=%s'
+ for key in where.keys():
+ sql += ' and %s=%%s' % key
+ values.append(where[key])
+ return test.select_sql(self.db, sql, values)
class machine:
- @classmethod
- def select(klass, db, where = {}):
- fields = ['machine_idx', 'hostname', 'machine_group', 'owner']
- machines = []
- for row in db.select(','.join(fields), 'machines', where):
- machines.append(klass(db, *row))
- return machines
+ @classmethod
+ def select(klass, db, where = {}):
+ fields = ['machine_idx', 'hostname', 'machine_group', 'owner']
+ machines = []
+ for row in db.select(','.join(fields), 'machines', where):
+ machines.append(klass(db, *row))
+ return machines
- def __init__(self, db, idx, hostname, group, owner):
- self.db = db
- self.idx = idx
- self.hostname = hostname
- self.group = group
- self.owner = owner
+ def __init__(self, db, idx, hostname, group, owner):
+ self.db = db
+ self.idx = idx
+ self.hostname = hostname
+ self.group = group
+ self.owner = owner
class kernel:
- @classmethod
- def select(klass, db, where = {}):
- fields = ['kernel_idx', 'kernel_hash', 'base', 'printable']
- rows = db.select(','.join(fields), 'kernels', where)
- return [klass(db, *row) for row in rows]
+ @classmethod
+ def select(klass, db, where = {}):
+ fields = ['kernel_idx', 'kernel_hash', 'base', 'printable']
+ rows = db.select(','.join(fields), 'kernels', where)
+ return [klass(db, *row) for row in rows]
- def __init__(self, db, idx, hash, base, printable):
- self.db = db
- self.idx = idx
- self.hash = hash
- self.base = base
- self.printable = printable
- self.patches = [] # THIS SHOULD PULL IN PATCHES!
+ def __init__(self, db, idx, hash, base, printable):
+ self.db = db
+ self.idx = idx
+ self.hash = hash
+ self.base = base
+ self.printable = printable
+ self.patches = [] # THIS SHOULD PULL IN PATCHES!
class test:
- @classmethod
- def select(klass, db, where = {}, wherein = {}, distinct = False):
- fields = ['test_idx', 'job_idx', 'test', 'subdir',
- 'kernel_idx', 'status', 'reason', 'machine_idx']
- tests = []
- for row in db.select(','.join(fields), 'tests', where,
- wherein,distinct):
- tests.append(klass(db, *row))
- return tests
+ @classmethod
+ def select(klass, db, where = {}, wherein = {}, distinct = False):
+ fields = ['test_idx', 'job_idx', 'test', 'subdir',
+ 'kernel_idx', 'status', 'reason', 'machine_idx']
+ tests = []
+ for row in db.select(','.join(fields), 'tests', where,
+ wherein,distinct):
+ tests.append(klass(db, *row))
+ return tests
- @classmethod
- def select_sql(klass, db, sql, values):
- fields = ['test_idx', 'job_idx', 'test', 'subdir',
- 'kernel_idx', 'status', 'reason', 'machine_idx']
- fields = ['t.'+field for field in fields]
- rows = db.select_sql(','.join(fields), 'tests', sql, values)
- return [klass(db, *row) for row in rows]
-
-
- def __init__(self, db, test_idx, job_idx, testname, subdir, kernel_idx,
- status_num, reason, machine_idx):
- self.idx = test_idx
- self.job = job(db, job_idx)
- self.testname = testname
- self.subdir = subdir
- self.kernel_idx = kernel_idx
- self.__kernel = None
- self.__iterations = None
- self.machine_idx = machine_idx
- self.__machine = None
- self.status_num = status_num
- self.status_word = db.status_word[status_num]
- self.reason = reason
- self.db = db
- if self.subdir:
- self.url = html_root + self.job.tag + '/' + self.subdir
- else:
- self.url = None
+ @classmethod
+ def select_sql(klass, db, sql, values):
+ fields = ['test_idx', 'job_idx', 'test', 'subdir',
+ 'kernel_idx', 'status', 'reason', 'machine_idx']
+ fields = ['t.'+field for field in fields]
+ rows = db.select_sql(','.join(fields), 'tests', sql, values)
+ return [klass(db, *row) for row in rows]
- def iterations(self):
- """
- Caching function for iterations
- """
- if not self.__iterations:
- self.__iterations = {}
- # A dictionary - dict{key} = [value1, value2, ....]
- where = {'test_idx' : self.idx}
- for i in iteration.select(self.db, where):
- if self.__iterations.has_key(i.key):
- self.__iterations[i.key].append(i.value)
- else:
- self.__iterations[i.key] = [i.value]
- return self.__iterations
-
-
- def kernel(self):
- """
- Caching function for kernels
- """
- if not self.__kernel:
- where = {'kernel_idx' : self.kernel_idx}
- self.__kernel = kernel.select(self.db, where)[0]
- return self.__kernel
+ def __init__(self, db, test_idx, job_idx, testname, subdir, kernel_idx,
+ status_num, reason, machine_idx):
+ self.idx = test_idx
+ self.job = job(db, job_idx)
+ self.testname = testname
+ self.subdir = subdir
+ self.kernel_idx = kernel_idx
+ self.__kernel = None
+ self.__iterations = None
+ self.machine_idx = machine_idx
+ self.__machine = None
+ self.status_num = status_num
+ self.status_word = db.status_word[status_num]
+ self.reason = reason
+ self.db = db
+ if self.subdir:
+ self.url = html_root + self.job.tag + '/' + self.subdir
+ else:
+ self.url = None
- def machine(self):
- """
- Caching function for kernels
- """
- if not self.__machine:
- where = {'machine_idx' : self.machine_idx}
- self.__machine = machine.select(self.db, where)[0]
- return self.__machine
+ def iterations(self):
+ """
+ Caching function for iterations
+ """
+ if not self.__iterations:
+ self.__iterations = {}
+ # A dictionary - dict{key} = [value1, value2, ....]
+ where = {'test_idx' : self.idx}
+ for i in iteration.select(self.db, where):
+ if self.__iterations.has_key(i.key):
+ self.__iterations[i.key].append(i.value)
+ else:
+ self.__iterations[i.key] = [i.value]
+ return self.__iterations
+
+
+ def kernel(self):
+ """
+ Caching function for kernels
+ """
+ if not self.__kernel:
+ where = {'kernel_idx' : self.kernel_idx}
+ self.__kernel = kernel.select(self.db, where)[0]
+ return self.__kernel
+
+
+ def machine(self):
+ """
+ Caching function for kernels
+ """
+ if not self.__machine:
+ where = {'machine_idx' : self.machine_idx}
+ self.__machine = machine.select(self.db, where)[0]
+ return self.__machine
class job:
- def __init__(self, db, job_idx):
- where = {'job_idx' : job_idx}
- rows = db.select('tag, machine_idx', 'jobs', where)
- if not rows:
- return None
- (self.tag, self.machine_idx) = rows[0]
- self.job_idx = job_idx
+ def __init__(self, db, job_idx):
+ where = {'job_idx' : job_idx}
+ rows = db.select('tag, machine_idx', 'jobs', where)
+ if not rows:
+ return None
+ (self.tag, self.machine_idx) = rows[0]
+ self.job_idx = job_idx
-
+
class iteration:
- @classmethod
- def select(klass, db, where):
- fields = ['iteration', 'attribute', 'value']
- iterations = []
- rows = db.select(','.join(fields), 'iteration_result', where)
- for row in rows:
- iterations.append(klass(*row))
- return iterations
+ @classmethod
+ def select(klass, db, where):
+ fields = ['iteration', 'attribute', 'value']
+ iterations = []
+ rows = db.select(','.join(fields), 'iteration_result', where)
+ for row in rows:
+ iterations.append(klass(*row))
+ return iterations
- def __init__(self, iteration, key, value):
- self.iteration = iteration
- self.key = key
- self.value = value
+ def __init__(self, iteration, key, value):
+ self.iteration = iteration
+ self.key = key
+ self.value = value
# class patch:
-# def __init__(self):
-# self.spec = None
+# def __init__(self):
+# self.spec = None
diff --git a/tko/machine_load b/tko/machine_load
index 98b0fa9..b8c0472 100755
--- a/tko/machine_load
+++ b/tko/machine_load
@@ -4,8 +4,8 @@
db = db.db()
for line in open('machines', 'r').readlines():
- (machine, group) = line.rstrip().split()
- print 'X %s Y %s' % (machine, group)
- set = { 'machine_group' : group }
- where = { 'hostname' : machine }
- db.update('machines', set, where)
+ (machine, group) = line.rstrip().split()
+ print 'X %s Y %s' % (machine, group)
+ set = { 'machine_group' : group }
+ where = { 'hostname' : machine }
+ db.update('machines', set, where)
diff --git a/tko/migrations/001_initial_db.py b/tko/migrations/001_initial_db.py
index 2c19abc..065e5c5 100755
--- a/tko/migrations/001_initial_db.py
+++ b/tko/migrations/001_initial_db.py
@@ -1,31 +1,31 @@
import os
required_tables = ('machines', 'jobs', 'patches', 'tests', 'test_attributes',
- 'iteration_result')
+ 'iteration_result')
def migrate_up(manager):
- manager.execute("SHOW TABLES")
- tables = [row[0] for row in manager.cursor.fetchall()]
- db_initialized = True
- for table in required_tables:
- if table not in tables:
- db_initialized = False
- break
- if not db_initialized:
- response = raw_input(
- 'Your tko database does not appear to be initialized. Do '
- 'you want to recreate it (this will result in loss of any '
- 'existing data) (yes/No)? ')
- if response != 'yes':
- raise Exception('User has chosen to abort migration')
+ manager.execute("SHOW TABLES")
+ tables = [row[0] for row in manager.cursor.fetchall()]
+ db_initialized = True
+ for table in required_tables:
+ if table not in tables:
+ db_initialized = False
+ break
+ if not db_initialized:
+ response = raw_input(
+ 'Your tko database does not appear to be initialized. Do '
+ 'you want to recreate it (this will result in loss of any '
+ 'existing data) (yes/No)? ')
+ if response != 'yes':
+ raise Exception('User has chosen to abort migration')
- manager.execute_script(CREATE_DB_SQL)
+ manager.execute_script(CREATE_DB_SQL)
- manager.create_migrate_table()
+ manager.create_migrate_table()
def migrate_down(manager):
- manager.execute_script(DROP_DB_SQL)
+ manager.execute_script(DROP_DB_SQL)
DROP_DB_SQL = """\
@@ -48,44 +48,44 @@
CREATE_DB_SQL = DROP_DB_SQL + """\
-- status key
CREATE TABLE status (
-status_idx int(10) unsigned NOT NULL auto_increment PRIMARY KEY , -- numerical status
-word VARCHAR(10) -- status word
+status_idx int(10) unsigned NOT NULL auto_increment PRIMARY KEY , -- numerical status
+word VARCHAR(10) -- status word
) TYPE=InnoDB;
-- kernel versions
CREATE TABLE kernels (
kernel_idx int(10) unsigned NOT NULL auto_increment PRIMARY KEY,
-kernel_hash VARCHAR(35), -- Hash of base + all patches
-base VARCHAR(30), -- Base version without patches
-printable VARCHAR(100) -- Full version with patches
+kernel_hash VARCHAR(35), -- Hash of base + all patches
+base VARCHAR(30), -- Base version without patches
+printable VARCHAR(100) -- Full version with patches
) TYPE=InnoDB;
-- machines/hosts table
CREATE TABLE machines (
machine_idx int(10) unsigned NOT NULL auto_increment PRIMARY KEY,
-hostname VARCHAR(100) unique KEY, -- hostname
-machine_group VARCHAR(80), -- group name
-owner VARCHAR(80) -- owner name
+hostname VARCHAR(100) unique KEY, -- hostname
+machine_group VARCHAR(80), -- group name
+owner VARCHAR(80) -- owner name
) TYPE=InnoDB;
-- main jobs table
CREATE TABLE jobs (
-job_idx int(10) unsigned NOT NULL auto_increment PRIMARY KEY, -- index number
-tag VARCHAR(100) unique KEY, -- job key
+job_idx int(10) unsigned NOT NULL auto_increment PRIMARY KEY, -- index number
+tag VARCHAR(100) unique KEY, -- job key
label VARCHAR(100), -- job label assigned by user
KEY (label),
-username VARCHAR(80), -- user name
+username VARCHAR(80), -- user name
KEY (username),
-machine_idx INT(10) unsigned NOT NULL, -- reference to machine table
+machine_idx INT(10) unsigned NOT NULL, -- reference to machine table
KEY (machine_idx),
FOREIGN KEY (machine_idx) REFERENCES machines(machine_idx) ON DELETE CASCADE
) TYPE=InnoDB;
-- One entry per patch used, anywhere
CREATE TABLE patches (
-kernel_idx INT(10) unsigned NOT NULL, -- index number
-name VARCHAR(80), -- short name
-url VARCHAR(300), -- full URL
+kernel_idx INT(10) unsigned NOT NULL, -- index number
+name VARCHAR(80), -- short name
+url VARCHAR(300), -- full URL
hash VARCHAR(35),
KEY (kernel_idx),
FOREIGN KEY (kernel_idx) REFERENCES kernels(kernel_idx) ON DELETE CASCADE
@@ -93,18 +93,18 @@
-- test functional results
CREATE TABLE tests (
-test_idx int(10) unsigned NOT NULL auto_increment PRIMARY KEY, -- index number
-job_idx INTEGER, -- ref to job table
-test VARCHAR(30), -- name of test
-subdir VARCHAR(60), -- subdirectory name
-kernel_idx INT(10) unsigned NOT NULL, -- kernel test was AGAINST
+test_idx int(10) unsigned NOT NULL auto_increment PRIMARY KEY, -- index number
+job_idx INTEGER, -- ref to job table
+test VARCHAR(30), -- name of test
+subdir VARCHAR(60), -- subdirectory name
+kernel_idx INT(10) unsigned NOT NULL, -- kernel test was AGAINST
KEY (kernel_idx),
FOREIGN KEY (kernel_idx) REFERENCES kernels(kernel_idx) ON DELETE CASCADE,
-status int(10) unsigned NOT NULL, -- test status
+status int(10) unsigned NOT NULL, -- test status
KEY (status),
FOREIGN KEY (status) REFERENCES status(status_idx) ON DELETE CASCADE,
-reason VARCHAR(100), -- reason for test status
-machine_idx INT(10) unsigned NOT NULL, -- reference to machine table
+reason VARCHAR(100), -- reason for test status
+machine_idx INT(10) unsigned NOT NULL, -- reference to machine table
KEY (machine_idx),
FOREIGN KEY (machine_idx) REFERENCES machines(machine_idx) ON DELETE CASCADE,
invalid BOOL NOT NULL
@@ -112,49 +112,49 @@
-- test attributes (key value pairs at a test level)
CREATE TABLE test_attributes (
-test_idx int(10) unsigned NOT NULL, -- ref to test table
+test_idx int(10) unsigned NOT NULL, -- ref to test table
FOREIGN KEY (test_idx) REFERENCES tests(test_idx) ON DELETE CASCADE,
-attribute VARCHAR(30), -- attribute name (e.g. 'version')
-value VARCHAR(100), -- attribute value
+attribute VARCHAR(30), -- attribute name (e.g. 'version')
+value VARCHAR(100), -- attribute value
KEY `test_idx` (`test_idx`)
) TYPE=InnoDB;
-- test performance results
CREATE TABLE iteration_result(
-test_idx int(10) unsigned NOT NULL, -- ref to test table
+test_idx int(10) unsigned NOT NULL, -- ref to test table
FOREIGN KEY (test_idx) REFERENCES tests(test_idx) ON DELETE CASCADE,
-iteration INTEGER, -- integer
-attribute VARCHAR(30), -- attribute name (e.g. 'throughput')
-value FLOAT, -- attribute value (eg 700.1)
+iteration INTEGER, -- integer
+attribute VARCHAR(30), -- attribute name (e.g. 'throughput')
+value FLOAT, -- attribute value (eg 700.1)
KEY `test_idx` (`test_idx`)
) TYPE=InnoDB;
-- BRRD syncronization
CREATE TABLE brrd_sync (
-test_idx int(10) unsigned NOT NULL, -- ref to test table
+test_idx int(10) unsigned NOT NULL, -- ref to test table
FOREIGN KEY (test_idx) REFERENCES tests(test_idx) ON DELETE CASCADE
) TYPE=InnoDB;
-- test_view (to make life easier for people trying to mine data)
CREATE VIEW test_view AS
-SELECT tests.test_idx,
- tests.job_idx,
- tests.test,
- tests.subdir,
- tests.kernel_idx,
- tests.status,
- tests.reason,
- tests.machine_idx,
- jobs.tag AS job_tag,
- jobs.label AS job_label,
- jobs.username AS job_username,
- machines.hostname AS machine_hostname,
- machines.machine_group,
- machines.owner AS machine_owner,
- kernels.kernel_hash,
- kernels.base AS kernel_base,
- kernels.printable AS kernel_printable,
- status.word AS status_word
+SELECT tests.test_idx,
+ tests.job_idx,
+ tests.test,
+ tests.subdir,
+ tests.kernel_idx,
+ tests.status,
+ tests.reason,
+ tests.machine_idx,
+ jobs.tag AS job_tag,
+ jobs.label AS job_label,
+ jobs.username AS job_username,
+ machines.hostname AS machine_hostname,
+ machines.machine_group,
+ machines.owner AS machine_owner,
+ kernels.kernel_hash,
+ kernels.base AS kernel_base,
+ kernels.printable AS kernel_printable,
+ status.word AS status_word
FROM tests
INNER JOIN jobs ON jobs.job_idx = tests.job_idx
INNER JOIN machines ON machines.machine_idx = jobs.machine_idx
@@ -163,27 +163,27 @@
-- perf_view (to make life easier for people trying to mine performance data)
CREATE VIEW perf_view AS
-SELECT tests.test_idx,
- tests.job_idx,
- tests.test,
- tests.subdir,
- tests.kernel_idx,
- tests.status,
- tests.reason,
- tests.machine_idx,
- jobs.tag AS job_tag,
- jobs.label AS job_label,
- jobs.username AS job_username,
- machines.hostname AS machine_hostname,
- machines.machine_group,
- machines.owner AS machine_owner,
- kernels.kernel_hash,
- kernels.base AS kernel_base,
- kernels.printable AS kernel_printable,
- status.word AS status_word,
- iteration_result.iteration,
- iteration_result.attribute AS iteration_key,
- iteration_result.value AS iteration_value
+SELECT tests.test_idx,
+ tests.job_idx,
+ tests.test,
+ tests.subdir,
+ tests.kernel_idx,
+ tests.status,
+ tests.reason,
+ tests.machine_idx,
+ jobs.tag AS job_tag,
+ jobs.label AS job_label,
+ jobs.username AS job_username,
+ machines.hostname AS machine_hostname,
+ machines.machine_group,
+ machines.owner AS machine_owner,
+ kernels.kernel_hash,
+ kernels.base AS kernel_base,
+ kernels.printable AS kernel_printable,
+ status.word AS status_word,
+ iteration_result.iteration,
+ iteration_result.attribute AS iteration_key,
+ iteration_result.value AS iteration_value
FROM tests
INNER JOIN jobs ON jobs.job_idx = tests.job_idx
INNER JOIN machines ON machines.machine_idx = jobs.machine_idx
diff --git a/tko/migrations/002_add_job_timestamps.py b/tko/migrations/002_add_job_timestamps.py
index 8bbb83a..7cd5004 100644
--- a/tko/migrations/002_add_job_timestamps.py
+++ b/tko/migrations/002_add_job_timestamps.py
@@ -1,11 +1,11 @@
def migrate_up(manager):
- manager.execute_script(ADD_COLUMNS_SQL)
- manager.execute_script(ALTER_VIEWS_UP_SQL)
+ manager.execute_script(ADD_COLUMNS_SQL)
+ manager.execute_script(ALTER_VIEWS_UP_SQL)
def migrate_down(manager):
- manager.execute_script(DROP_COLUMNS_SQL)
- manager.execute_script(ALTER_VIEWS_DOWN_SQL)
+ manager.execute_script(DROP_COLUMNS_SQL)
+ manager.execute_script(ALTER_VIEWS_DOWN_SQL)
ADD_COLUMNS_SQL = """\
@@ -22,27 +22,27 @@
ALTER_VIEWS_UP_SQL = """\
ALTER VIEW test_view AS
-SELECT tests.test_idx,
- tests.job_idx,
- tests.test,
- tests.subdir,
- tests.kernel_idx,
- tests.status,
- tests.reason,
- tests.machine_idx,
- jobs.tag AS job_tag,
- jobs.label AS job_label,
- jobs.username AS job_username,
- jobs.queued_time AS job_queued_time,
- jobs.started_time AS job_started_time,
- jobs.finished_time AS job_finished_time,
- machines.hostname AS machine_hostname,
- machines.machine_group,
- machines.owner AS machine_owner,
- kernels.kernel_hash,
- kernels.base AS kernel_base,
- kernels.printable AS kernel_printable,
- status.word AS status_word
+SELECT tests.test_idx,
+ tests.job_idx,
+ tests.test,
+ tests.subdir,
+ tests.kernel_idx,
+ tests.status,
+ tests.reason,
+ tests.machine_idx,
+ jobs.tag AS job_tag,
+ jobs.label AS job_label,
+ jobs.username AS job_username,
+ jobs.queued_time AS job_queued_time,
+ jobs.started_time AS job_started_time,
+ jobs.finished_time AS job_finished_time,
+ machines.hostname AS machine_hostname,
+ machines.machine_group,
+ machines.owner AS machine_owner,
+ kernels.kernel_hash,
+ kernels.base AS kernel_base,
+ kernels.printable AS kernel_printable,
+ status.word AS status_word
FROM tests
INNER JOIN jobs ON jobs.job_idx = tests.job_idx
INNER JOIN machines ON machines.machine_idx = jobs.machine_idx
@@ -51,30 +51,30 @@
-- perf_view (to make life easier for people trying to mine performance data)
ALTER VIEW perf_view AS
-SELECT tests.test_idx,
- tests.job_idx,
- tests.test,
- tests.subdir,
- tests.kernel_idx,
- tests.status,
- tests.reason,
- tests.machine_idx,
- jobs.tag AS job_tag,
- jobs.label AS job_label,
- jobs.username AS job_username,
- jobs.queued_time AS job_queued_time,
- jobs.started_time AS job_started_time,
- jobs.finished_time AS job_finished_time,
- machines.hostname AS machine_hostname,
- machines.machine_group,
- machines.owner AS machine_owner,
- kernels.kernel_hash,
- kernels.base AS kernel_base,
- kernels.printable AS kernel_printable,
- status.word AS status_word,
- iteration_result.iteration,
- iteration_result.attribute AS iteration_key,
- iteration_result.value AS iteration_value
+SELECT tests.test_idx,
+ tests.job_idx,
+ tests.test,
+ tests.subdir,
+ tests.kernel_idx,
+ tests.status,
+ tests.reason,
+ tests.machine_idx,
+ jobs.tag AS job_tag,
+ jobs.label AS job_label,
+ jobs.username AS job_username,
+ jobs.queued_time AS job_queued_time,
+ jobs.started_time AS job_started_time,
+ jobs.finished_time AS job_finished_time,
+ machines.hostname AS machine_hostname,
+ machines.machine_group,
+ machines.owner AS machine_owner,
+ kernels.kernel_hash,
+ kernels.base AS kernel_base,
+ kernels.printable AS kernel_printable,
+ status.word AS status_word,
+ iteration_result.iteration,
+ iteration_result.attribute AS iteration_key,
+ iteration_result.value AS iteration_value
FROM tests
INNER JOIN jobs ON jobs.job_idx = tests.job_idx
INNER JOIN machines ON machines.machine_idx = jobs.machine_idx
@@ -86,24 +86,24 @@
ALTER_VIEWS_DOWN_SQL = """\
ALTER VIEW test_view AS
-SELECT tests.test_idx,
- tests.job_idx,
- tests.test,
- tests.subdir,
- tests.kernel_idx,
- tests.status,
- tests.reason,
- tests.machine_idx,
- jobs.tag AS job_tag,
- jobs.label AS job_label,
- jobs.username AS job_username,
- machines.hostname AS machine_hostname,
- machines.machine_group,
- machines.owner AS machine_owner,
- kernels.kernel_hash,
- kernels.base AS kernel_base,
- kernels.printable AS kernel_printable,
- status.word AS status_word
+SELECT tests.test_idx,
+ tests.job_idx,
+ tests.test,
+ tests.subdir,
+ tests.kernel_idx,
+ tests.status,
+ tests.reason,
+ tests.machine_idx,
+ jobs.tag AS job_tag,
+ jobs.label AS job_label,
+ jobs.username AS job_username,
+ machines.hostname AS machine_hostname,
+ machines.machine_group,
+ machines.owner AS machine_owner,
+ kernels.kernel_hash,
+ kernels.base AS kernel_base,
+ kernels.printable AS kernel_printable,
+ status.word AS status_word
FROM tests
INNER JOIN jobs ON jobs.job_idx = tests.job_idx
INNER JOIN machines ON machines.machine_idx = jobs.machine_idx
@@ -112,27 +112,27 @@
-- perf_view (to make life easier for people trying to mine performance data)
ALTER VIEW perf_view AS
-SELECT tests.test_idx,
- tests.job_idx,
- tests.test,
- tests.subdir,
- tests.kernel_idx,
- tests.status,
- tests.reason,
- tests.machine_idx,
- jobs.tag AS job_tag,
- jobs.label AS job_label,
- jobs.username AS job_username,
- machines.hostname AS machine_hostname,
- machines.machine_group,
- machines.owner AS machine_owner,
- kernels.kernel_hash,
- kernels.base AS kernel_base,
- kernels.printable AS kernel_printable,
- status.word AS status_word,
- iteration_result.iteration,
- iteration_result.attribute AS iteration_key,
- iteration_result.value AS iteration_value
+SELECT tests.test_idx,
+ tests.job_idx,
+ tests.test,
+ tests.subdir,
+ tests.kernel_idx,
+ tests.status,
+ tests.reason,
+ tests.machine_idx,
+ jobs.tag AS job_tag,
+ jobs.label AS job_label,
+ jobs.username AS job_username,
+ machines.hostname AS machine_hostname,
+ machines.machine_group,
+ machines.owner AS machine_owner,
+ kernels.kernel_hash,
+ kernels.base AS kernel_base,
+ kernels.printable AS kernel_printable,
+ status.word AS status_word,
+ iteration_result.iteration,
+ iteration_result.attribute AS iteration_key,
+ iteration_result.value AS iteration_value
FROM tests
INNER JOIN jobs ON jobs.job_idx = tests.job_idx
INNER JOIN machines ON machines.machine_idx = jobs.machine_idx
diff --git a/tko/migrations/003_add_test_timestamps.py b/tko/migrations/003_add_test_timestamps.py
index e9148cb..fee4cab 100644
--- a/tko/migrations/003_add_test_timestamps.py
+++ b/tko/migrations/003_add_test_timestamps.py
@@ -1,11 +1,11 @@
def migrate_up(manager):
- manager.execute_script(ADD_COLUMN_SQL)
- manager.execute_script(ALTER_VIEWS_UP_SQL)
+ manager.execute_script(ADD_COLUMN_SQL)
+ manager.execute_script(ALTER_VIEWS_UP_SQL)
def migrate_down(manager):
- manager.execute_script(DROP_COLUMN_SQL)
- manager.execute_script(ALTER_VIEWS_DOWN_SQL)
+ manager.execute_script(DROP_COLUMN_SQL)
+ manager.execute_script(ALTER_VIEWS_DOWN_SQL)
ADD_COLUMN_SQL = """\
@@ -18,28 +18,28 @@
ALTER_VIEWS_UP_SQL = """\
ALTER VIEW test_view AS
-SELECT tests.test_idx,
- tests.job_idx,
- tests.test,
- tests.subdir,
- tests.kernel_idx,
- tests.status,
- tests.reason,
- tests.machine_idx,
- tests.finished_time AS test_finished_time,
- jobs.tag AS job_tag,
- jobs.label AS job_label,
- jobs.username AS job_username,
- jobs.queued_time AS job_queued_time,
- jobs.started_time AS job_started_time,
- jobs.finished_time AS job_finished_time,
- machines.hostname AS machine_hostname,
- machines.machine_group,
- machines.owner AS machine_owner,
- kernels.kernel_hash,
- kernels.base AS kernel_base,
- kernels.printable AS kernel_printable,
- status.word AS status_word
+SELECT tests.test_idx,
+ tests.job_idx,
+ tests.test,
+ tests.subdir,
+ tests.kernel_idx,
+ tests.status,
+ tests.reason,
+ tests.machine_idx,
+ tests.finished_time AS test_finished_time,
+ jobs.tag AS job_tag,
+ jobs.label AS job_label,
+ jobs.username AS job_username,
+ jobs.queued_time AS job_queued_time,
+ jobs.started_time AS job_started_time,
+ jobs.finished_time AS job_finished_time,
+ machines.hostname AS machine_hostname,
+ machines.machine_group,
+ machines.owner AS machine_owner,
+ kernels.kernel_hash,
+ kernels.base AS kernel_base,
+ kernels.printable AS kernel_printable,
+ status.word AS status_word
FROM tests
INNER JOIN jobs ON jobs.job_idx = tests.job_idx
INNER JOIN machines ON machines.machine_idx = jobs.machine_idx
@@ -48,31 +48,31 @@
-- perf_view (to make life easier for people trying to mine performance data)
ALTER VIEW perf_view AS
-SELECT tests.test_idx,
- tests.job_idx,
- tests.test,
- tests.subdir,
- tests.kernel_idx,
- tests.status,
- tests.reason,
- tests.machine_idx,
- tests.finished_time AS test_finished_time,
- jobs.tag AS job_tag,
- jobs.label AS job_label,
- jobs.username AS job_username,
- jobs.queued_time AS job_queued_time,
- jobs.started_time AS job_started_time,
- jobs.finished_time AS job_finished_time,
- machines.hostname AS machine_hostname,
- machines.machine_group,
- machines.owner AS machine_owner,
- kernels.kernel_hash,
- kernels.base AS kernel_base,
- kernels.printable AS kernel_printable,
- status.word AS status_word,
- iteration_result.iteration,
- iteration_result.attribute AS iteration_key,
- iteration_result.value AS iteration_value
+SELECT tests.test_idx,
+ tests.job_idx,
+ tests.test,
+ tests.subdir,
+ tests.kernel_idx,
+ tests.status,
+ tests.reason,
+ tests.machine_idx,
+ tests.finished_time AS test_finished_time,
+ jobs.tag AS job_tag,
+ jobs.label AS job_label,
+ jobs.username AS job_username,
+ jobs.queued_time AS job_queued_time,
+ jobs.started_time AS job_started_time,
+ jobs.finished_time AS job_finished_time,
+ machines.hostname AS machine_hostname,
+ machines.machine_group,
+ machines.owner AS machine_owner,
+ kernels.kernel_hash,
+ kernels.base AS kernel_base,
+ kernels.printable AS kernel_printable,
+ status.word AS status_word,
+ iteration_result.iteration,
+ iteration_result.attribute AS iteration_key,
+ iteration_result.value AS iteration_value
FROM tests
INNER JOIN jobs ON jobs.job_idx = tests.job_idx
INNER JOIN machines ON machines.machine_idx = jobs.machine_idx
@@ -84,27 +84,27 @@
ALTER_VIEWS_DOWN_SQL = """\
ALTER VIEW test_view AS
-SELECT tests.test_idx,
- tests.job_idx,
- tests.test,
- tests.subdir,
- tests.kernel_idx,
- tests.status,
- tests.reason,
- tests.machine_idx,
- jobs.tag AS job_tag,
- jobs.label AS job_label,
- jobs.username AS job_username,
- jobs.queued_time AS job_queued_time,
- jobs.started_time AS job_started_time,
- jobs.finished_time AS job_finished_time,
- machines.hostname AS machine_hostname,
- machines.machine_group,
- machines.owner AS machine_owner,
- kernels.kernel_hash,
- kernels.base AS kernel_base,
- kernels.printable AS kernel_printable,
- status.word AS status_word
+SELECT tests.test_idx,
+ tests.job_idx,
+ tests.test,
+ tests.subdir,
+ tests.kernel_idx,
+ tests.status,
+ tests.reason,
+ tests.machine_idx,
+ jobs.tag AS job_tag,
+ jobs.label AS job_label,
+ jobs.username AS job_username,
+ jobs.queued_time AS job_queued_time,
+ jobs.started_time AS job_started_time,
+ jobs.finished_time AS job_finished_time,
+ machines.hostname AS machine_hostname,
+ machines.machine_group,
+ machines.owner AS machine_owner,
+ kernels.kernel_hash,
+ kernels.base AS kernel_base,
+ kernels.printable AS kernel_printable,
+ status.word AS status_word
FROM tests
INNER JOIN jobs ON jobs.job_idx = tests.job_idx
INNER JOIN machines ON machines.machine_idx = jobs.machine_idx
@@ -113,30 +113,30 @@
-- perf_view (to make life easier for people trying to mine performance data)
ALTER VIEW perf_view AS
-SELECT tests.test_idx,
- tests.job_idx,
- tests.test,
- tests.subdir,
- tests.kernel_idx,
- tests.status,
- tests.reason,
- tests.machine_idx,
- jobs.tag AS job_tag,
- jobs.label AS job_label,
- jobs.username AS job_username,
- jobs.queued_time AS job_queued_time,
- jobs.started_time AS job_started_time,
- jobs.finished_time AS job_finished_time,
- machines.hostname AS machine_hostname,
- machines.machine_group,
- machines.owner AS machine_owner,
- kernels.kernel_hash,
- kernels.base AS kernel_base,
- kernels.printable AS kernel_printable,
- status.word AS status_word,
- iteration_result.iteration,
- iteration_result.attribute AS iteration_key,
- iteration_result.value AS iteration_value
+SELECT tests.test_idx,
+ tests.job_idx,
+ tests.test,
+ tests.subdir,
+ tests.kernel_idx,
+ tests.status,
+ tests.reason,
+ tests.machine_idx,
+ jobs.tag AS job_tag,
+ jobs.label AS job_label,
+ jobs.username AS job_username,
+ jobs.queued_time AS job_queued_time,
+ jobs.started_time AS job_started_time,
+ jobs.finished_time AS job_finished_time,
+ machines.hostname AS machine_hostname,
+ machines.machine_group,
+ machines.owner AS machine_owner,
+ kernels.kernel_hash,
+ kernels.base AS kernel_base,
+ kernels.printable AS kernel_printable,
+ status.word AS status_word,
+ iteration_result.iteration,
+ iteration_result.attribute AS iteration_key,
+ iteration_result.value AS iteration_value
FROM tests
INNER JOIN jobs ON jobs.job_idx = tests.job_idx
INNER JOIN machines ON machines.machine_idx = jobs.machine_idx
diff --git a/tko/migrations/004_add_test_started.py b/tko/migrations/004_add_test_started.py
index f3d9f3d..cf91064 100644
--- a/tko/migrations/004_add_test_started.py
+++ b/tko/migrations/004_add_test_started.py
@@ -1,11 +1,11 @@
def migrate_up(manager):
- manager.execute_script(ADD_COLUMN_SQL)
- manager.execute_script(ALTER_VIEWS_UP_SQL)
+ manager.execute_script(ADD_COLUMN_SQL)
+ manager.execute_script(ALTER_VIEWS_UP_SQL)
def migrate_down(manager):
- manager.execute_script(DROP_COLUMN_SQL)
- manager.execute_script(ALTER_VIEWS_DOWN_SQL)
+ manager.execute_script(DROP_COLUMN_SQL)
+ manager.execute_script(ALTER_VIEWS_DOWN_SQL)
ADD_COLUMN_SQL = """\
@@ -18,29 +18,29 @@
ALTER_VIEWS_UP_SQL = """\
ALTER VIEW test_view AS
-SELECT tests.test_idx,
- tests.job_idx,
- tests.test,
- tests.subdir,
- tests.kernel_idx,
- tests.status,
- tests.reason,
- tests.machine_idx,
+SELECT tests.test_idx,
+ tests.job_idx,
+ tests.test,
+ tests.subdir,
+ tests.kernel_idx,
+ tests.status,
+ tests.reason,
+ tests.machine_idx,
tests.started_time AS test_started_time,
- tests.finished_time AS test_finished_time,
- jobs.tag AS job_tag,
- jobs.label AS job_label,
- jobs.username AS job_username,
- jobs.queued_time AS job_queued_time,
- jobs.started_time AS job_started_time,
- jobs.finished_time AS job_finished_time,
- machines.hostname AS machine_hostname,
- machines.machine_group,
- machines.owner AS machine_owner,
- kernels.kernel_hash,
- kernels.base AS kernel_base,
- kernels.printable AS kernel_printable,
- status.word AS status_word
+ tests.finished_time AS test_finished_time,
+ jobs.tag AS job_tag,
+ jobs.label AS job_label,
+ jobs.username AS job_username,
+ jobs.queued_time AS job_queued_time,
+ jobs.started_time AS job_started_time,
+ jobs.finished_time AS job_finished_time,
+ machines.hostname AS machine_hostname,
+ machines.machine_group,
+ machines.owner AS machine_owner,
+ kernels.kernel_hash,
+ kernels.base AS kernel_base,
+ kernels.printable AS kernel_printable,
+ status.word AS status_word
FROM tests
INNER JOIN jobs ON jobs.job_idx = tests.job_idx
INNER JOIN machines ON machines.machine_idx = jobs.machine_idx
@@ -49,32 +49,32 @@
-- perf_view (to make life easier for people trying to mine performance data)
ALTER VIEW perf_view AS
-SELECT tests.test_idx,
- tests.job_idx,
- tests.test,
- tests.subdir,
- tests.kernel_idx,
- tests.status,
- tests.reason,
- tests.machine_idx,
+SELECT tests.test_idx,
+ tests.job_idx,
+ tests.test,
+ tests.subdir,
+ tests.kernel_idx,
+ tests.status,
+ tests.reason,
+ tests.machine_idx,
tests.started_time AS test_started_time,
- tests.finished_time AS test_finished_time,
- jobs.tag AS job_tag,
- jobs.label AS job_label,
- jobs.username AS job_username,
- jobs.queued_time AS job_queued_time,
- jobs.started_time AS job_started_time,
- jobs.finished_time AS job_finished_time,
- machines.hostname AS machine_hostname,
- machines.machine_group,
- machines.owner AS machine_owner,
- kernels.kernel_hash,
- kernels.base AS kernel_base,
- kernels.printable AS kernel_printable,
- status.word AS status_word,
- iteration_result.iteration,
- iteration_result.attribute AS iteration_key,
- iteration_result.value AS iteration_value
+ tests.finished_time AS test_finished_time,
+ jobs.tag AS job_tag,
+ jobs.label AS job_label,
+ jobs.username AS job_username,
+ jobs.queued_time AS job_queued_time,
+ jobs.started_time AS job_started_time,
+ jobs.finished_time AS job_finished_time,
+ machines.hostname AS machine_hostname,
+ machines.machine_group,
+ machines.owner AS machine_owner,
+ kernels.kernel_hash,
+ kernels.base AS kernel_base,
+ kernels.printable AS kernel_printable,
+ status.word AS status_word,
+ iteration_result.iteration,
+ iteration_result.attribute AS iteration_key,
+ iteration_result.value AS iteration_value
FROM tests
INNER JOIN jobs ON jobs.job_idx = tests.job_idx
INNER JOIN machines ON machines.machine_idx = jobs.machine_idx
@@ -85,28 +85,28 @@
ALTER_VIEWS_DOWN_SQL = """\
ALTER VIEW test_view AS
-SELECT tests.test_idx,
- tests.job_idx,
- tests.test,
- tests.subdir,
- tests.kernel_idx,
- tests.status,
- tests.reason,
- tests.machine_idx,
- tests.finished_time AS test_finished_time,
- jobs.tag AS job_tag,
- jobs.label AS job_label,
- jobs.username AS job_username,
- jobs.queued_time AS job_queued_time,
- jobs.started_time AS job_started_time,
- jobs.finished_time AS job_finished_time,
- machines.hostname AS machine_hostname,
- machines.machine_group,
- machines.owner AS machine_owner,
- kernels.kernel_hash,
- kernels.base AS kernel_base,
- kernels.printable AS kernel_printable,
- status.word AS status_word
+SELECT tests.test_idx,
+ tests.job_idx,
+ tests.test,
+ tests.subdir,
+ tests.kernel_idx,
+ tests.status,
+ tests.reason,
+ tests.machine_idx,
+ tests.finished_time AS test_finished_time,
+ jobs.tag AS job_tag,
+ jobs.label AS job_label,
+ jobs.username AS job_username,
+ jobs.queued_time AS job_queued_time,
+ jobs.started_time AS job_started_time,
+ jobs.finished_time AS job_finished_time,
+ machines.hostname AS machine_hostname,
+ machines.machine_group,
+ machines.owner AS machine_owner,
+ kernels.kernel_hash,
+ kernels.base AS kernel_base,
+ kernels.printable AS kernel_printable,
+ status.word AS status_word
FROM tests
INNER JOIN jobs ON jobs.job_idx = tests.job_idx
INNER JOIN machines ON machines.machine_idx = jobs.machine_idx
@@ -115,31 +115,31 @@
-- perf_view (to make life easier for people trying to mine performance data)
ALTER VIEW perf_view AS
-SELECT tests.test_idx,
- tests.job_idx,
- tests.test,
- tests.subdir,
- tests.kernel_idx,
- tests.status,
- tests.reason,
- tests.machine_idx,
- tests.finished_time AS test_finished_time,
- jobs.tag AS job_tag,
- jobs.label AS job_label,
- jobs.username AS job_username,
- jobs.queued_time AS job_queued_time,
- jobs.started_time AS job_started_time,
- jobs.finished_time AS job_finished_time,
- machines.hostname AS machine_hostname,
- machines.machine_group,
- machines.owner AS machine_owner,
- kernels.kernel_hash,
- kernels.base AS kernel_base,
- kernels.printable AS kernel_printable,
- status.word AS status_word,
- iteration_result.iteration,
- iteration_result.attribute AS iteration_key,
- iteration_result.value AS iteration_value
+SELECT tests.test_idx,
+ tests.job_idx,
+ tests.test,
+ tests.subdir,
+ tests.kernel_idx,
+ tests.status,
+ tests.reason,
+ tests.machine_idx,
+ tests.finished_time AS test_finished_time,
+ jobs.tag AS job_tag,
+ jobs.label AS job_label,
+ jobs.username AS job_username,
+ jobs.queued_time AS job_queued_time,
+ jobs.started_time AS job_started_time,
+ jobs.finished_time AS job_finished_time,
+ machines.hostname AS machine_hostname,
+ machines.machine_group,
+ machines.owner AS machine_owner,
+ kernels.kernel_hash,
+ kernels.base AS kernel_base,
+ kernels.printable AS kernel_printable,
+ status.word AS status_word,
+ iteration_result.iteration,
+ iteration_result.attribute AS iteration_key,
+ iteration_result.value AS iteration_value
FROM tests
INNER JOIN jobs ON jobs.job_idx = tests.job_idx
INNER JOIN machines ON machines.machine_idx = jobs.machine_idx
diff --git a/tko/migrations/005_add_testna_status.py b/tko/migrations/005_add_testna_status.py
index e22abe5..4e4e480 100644
--- a/tko/migrations/005_add_testna_status.py
+++ b/tko/migrations/005_add_testna_status.py
@@ -1,6 +1,6 @@
def migrate_up(monger):
- monger.execute("INSERT INTO status (word) values ('TEST_NA')")
+ monger.execute("INSERT INTO status (word) values ('TEST_NA')")
def migrate_down(monger):
- monger.execute("DELETE FROM status where word = 'TEST_NA'")
+ monger.execute("DELETE FROM status where word = 'TEST_NA'")
diff --git a/tko/migrations/006_add_table_query_history.py b/tko/migrations/006_add_table_query_history.py
index 41c5e29..891836f 100644
--- a/tko/migrations/006_add_table_query_history.py
+++ b/tko/migrations/006_add_table_query_history.py
@@ -1,9 +1,9 @@
def migrate_up(manager):
- manager.execute_script(ADD_TABLE_QUERY_HISTORY)
+ manager.execute_script(ADD_TABLE_QUERY_HISTORY)
def migrate_down(manager):
- manager.execute_script(DROP_TABLE_QUERY_HISTORY)
+ manager.execute_script(DROP_TABLE_QUERY_HISTORY)
ADD_TABLE_QUERY_HISTORY = """
@@ -15,4 +15,3 @@
DROP_TABLE_QUERY_HISTORY = """
DROP TABLE query_history;
"""
-
diff --git a/tko/migrations/007_widen_reason_field.py b/tko/migrations/007_widen_reason_field.py
index 7df6bc8..aae2a44 100644
--- a/tko/migrations/007_widen_reason_field.py
+++ b/tko/migrations/007_widen_reason_field.py
@@ -1,5 +1,5 @@
def migrate_up(mgr):
- mgr.execute("alter table tests modify column reason varchar(1024);")
+ mgr.execute("alter table tests modify column reason varchar(1024);")
def migrate_down(mgr):
- mgr.execute("alter table tests modify column reason varchar(100);")
+ mgr.execute("alter table tests modify column reason varchar(100);")
diff --git a/tko/migrations/008_add_iteration_attributes.py b/tko/migrations/008_add_iteration_attributes.py
index 4ef176e..87cd56b 100644
--- a/tko/migrations/008_add_iteration_attributes.py
+++ b/tko/migrations/008_add_iteration_attributes.py
@@ -1,18 +1,18 @@
def migrate_up(manager):
- manager.execute_script(CREATE_TABLE_SQL)
+ manager.execute_script(CREATE_TABLE_SQL)
def migrate_down(manager):
- manager.execute_script(DROP_TABLE_SQL)
+ manager.execute_script(DROP_TABLE_SQL)
CREATE_TABLE_SQL = """
-- test iteration attributes (key value pairs at an iteration level)
CREATE TABLE iteration_attributes (
-test_idx int(10) unsigned NOT NULL, -- ref to test table
+test_idx int(10) unsigned NOT NULL, -- ref to test table
FOREIGN KEY (test_idx) REFERENCES tests(test_idx) ON DELETE CASCADE,
-iteration INTEGER, -- integer
-attribute VARCHAR(30), -- attribute name (e.g. 'run_id')
-value VARCHAR(100), -- attribute value
+iteration INTEGER, -- integer
+attribute VARCHAR(30), -- attribute name (e.g. 'run_id')
+value VARCHAR(100), -- attribute value
KEY `test_idx` (`test_idx`)
) TYPE=InnoDB;
"""
diff --git a/tko/models.py b/tko/models.py
index b1651f3..a5e230b 100644
--- a/tko/models.py
+++ b/tko/models.py
@@ -5,136 +5,136 @@
class job(object):
- def __init__(self, dir, user, label, machine, queued_time,
- started_time, finished_time, machine_owner):
- self.dir = dir
- self.tests = []
- self.user = user
- self.label = label
- self.machine = machine
- self.queued_time = queued_time
- self.started_time = started_time
- self.finished_time = finished_time
- self.machine_owner = machine_owner
+ def __init__(self, dir, user, label, machine, queued_time,
+ started_time, finished_time, machine_owner):
+ self.dir = dir
+ self.tests = []
+ self.user = user
+ self.label = label
+ self.machine = machine
+ self.queued_time = queued_time
+ self.started_time = started_time
+ self.finished_time = finished_time
+ self.machine_owner = machine_owner
class kernel(object):
- def __init__(self, base, patches, kernel_hash):
- self.base = base
- self.patches = patches
- self.kernel_hash = kernel_hash
+ def __init__(self, base, patches, kernel_hash):
+ self.base = base
+ self.patches = patches
+ self.kernel_hash = kernel_hash
- @staticmethod
- def compute_hash(base, hashes):
- key_string = ','.join([base] + hashes)
- return md5.new(key_string).hexdigest()
+ @staticmethod
+ def compute_hash(base, hashes):
+ key_string = ','.join([base] + hashes)
+ return md5.new(key_string).hexdigest()
class test(object):
- def __init__(self, subdir, testname, status, reason, test_kernel,
- machine, started_time, finished_time, iterations,
- attributes):
- self.subdir = subdir
- self.testname = testname
- self.status = status
- self.reason = reason
- self.kernel = test_kernel
- self.machine = machine
- self.started_time = started_time
- self.finished_time = finished_time
- self.iterations = iterations
- self.attributes = attributes
+ def __init__(self, subdir, testname, status, reason, test_kernel,
+ machine, started_time, finished_time, iterations,
+ attributes):
+ self.subdir = subdir
+ self.testname = testname
+ self.status = status
+ self.reason = reason
+ self.kernel = test_kernel
+ self.machine = machine
+ self.started_time = started_time
+ self.finished_time = finished_time
+ self.iterations = iterations
+ self.attributes = attributes
- @staticmethod
- def load_iterations(keyval_path):
- """Abstract method to load a list of iterations from a keyval
- file."""
- raise NotImplementedError
+ @staticmethod
+ def load_iterations(keyval_path):
+ """Abstract method to load a list of iterations from a keyval
+ file."""
+ raise NotImplementedError
- @classmethod
- def parse_test(cls, job, subdir, testname, status, reason, test_kernel,
- started_time, finished_time):
- """Given a job and the basic metadata about the test that
- can be extracted from the status logs, parse the test
- keyval files and use it to construct a complete test
- instance."""
- tko_utils.dprint("parsing test %s %s" % (subdir, testname))
+ @classmethod
+ def parse_test(cls, job, subdir, testname, status, reason, test_kernel,
+ started_time, finished_time):
+ """Given a job and the basic metadata about the test that
+ can be extracted from the status logs, parse the test
+ keyval files and use it to construct a complete test
+ instance."""
+ tko_utils.dprint("parsing test %s %s" % (subdir, testname))
- if subdir:
- # grab iterations from the results keyval
- iteration_keyval = os.path.join(job.dir, subdir,
- "results", "keyval")
- iterations = cls.load_iterations(iteration_keyval)
+ if subdir:
+ # grab iterations from the results keyval
+ iteration_keyval = os.path.join(job.dir, subdir,
+ "results", "keyval")
+ iterations = cls.load_iterations(iteration_keyval)
- # grab test attributes from the subdir keyval
- test_keyval = os.path.join(job.dir, subdir, "keyval")
- attributes = test.load_attributes(test_keyval)
- else:
- iterations = []
- attributes = {}
+ # grab test attributes from the subdir keyval
+ test_keyval = os.path.join(job.dir, subdir, "keyval")
+ attributes = test.load_attributes(test_keyval)
+ else:
+ iterations = []
+ attributes = {}
- return cls(subdir, testname, status, reason, test_kernel,
- job.machine, started_time, finished_time,
- iterations, attributes)
+ return cls(subdir, testname, status, reason, test_kernel,
+ job.machine, started_time, finished_time,
+ iterations, attributes)
- @staticmethod
- def load_attributes(keyval_path):
- """Load the test attributes into a dictionary from a test
- keyval path. Does not assume that the path actually exists."""
- if not os.path.exists(keyval_path):
- return {}
- return utils.read_keyval(keyval_path)
+ @staticmethod
+ def load_attributes(keyval_path):
+ """Load the test attributes into a dictionary from a test
+ keyval path. Does not assume that the path actually exists."""
+ if not os.path.exists(keyval_path):
+ return {}
+ return utils.read_keyval(keyval_path)
class patch(object):
- def __init__(self, spec, reference, hash):
- self.spec = spec
- self.reference = reference
- self.hash = hash
+ def __init__(self, spec, reference, hash):
+ self.spec = spec
+ self.reference = reference
+ self.hash = hash
class iteration(object):
- def __init__(self, index, attr_keyval, perf_keyval):
- self.index = index
- self.attr_keyval = attr_keyval
- self.perf_keyval = perf_keyval
+ def __init__(self, index, attr_keyval, perf_keyval):
+ self.index = index
+ self.attr_keyval = attr_keyval
+ self.perf_keyval = perf_keyval
- @staticmethod
- def parse_line_into_dicts(line, attr_dict, perf_dict):
- """Abstract method to parse a keyval line and insert it into
- the appropriate dictionary.
- attr_dict: generic iteration attributes
- perf_dict: iteration performance results
- """
- raise NotImplementedError
+ @staticmethod
+ def parse_line_into_dicts(line, attr_dict, perf_dict):
+ """Abstract method to parse a keyval line and insert it into
+ the appropriate dictionary.
+ attr_dict: generic iteration attributes
+ perf_dict: iteration performance results
+ """
+ raise NotImplementedError
- @classmethod
- def load_from_keyval(cls, keyval_path):
- """Load a list of iterations from an iteration keyval file.
- Keyval data from separate iterations is separated by blank
- lines. Makes use of the parse_line_into_dicts method to
- actually parse the individual lines."""
- if not os.path.exists(keyval_path):
- return []
+ @classmethod
+ def load_from_keyval(cls, keyval_path):
+ """Load a list of iterations from an iteration keyval file.
+ Keyval data from separate iterations is separated by blank
+ lines. Makes use of the parse_line_into_dicts method to
+ actually parse the individual lines."""
+ if not os.path.exists(keyval_path):
+ return []
- iterations = []
- index = 1
- attr, perf = {}, {}
- for line in file(keyval_path):
- line = line.strip()
- if line:
- cls.parse_line_into_dicts(line, attr, perf)
- else:
- iterations.append(cls(index, attr, perf))
- index += 1
- attr, perf = {}, {}
- if attr or perf:
- iterations.append(cls(index, attr, perf))
- return iterations
+ iterations = []
+ index = 1
+ attr, perf = {}, {}
+ for line in file(keyval_path):
+ line = line.strip()
+ if line:
+ cls.parse_line_into_dicts(line, attr, perf)
+ else:
+ iterations.append(cls(index, attr, perf))
+ index += 1
+ attr, perf = {}, {}
+ if attr or perf:
+ iterations.append(cls(index, attr, perf))
+ return iterations
diff --git a/tko/parse.py b/tko/parse.py
index 6d88d83..bdccdda 100755
--- a/tko/parse.py
+++ b/tko/parse.py
@@ -8,194 +8,194 @@
def parse_args():
- # build up our options parser and parse sys.argv
- parser = optparse.OptionParser()
- parser.add_option("-m", help="Send mail for FAILED tests",
- dest="mailit", action="store_true")
- parser.add_option("-r", help="Reparse the results of a job",
- dest="reparse", action="store_true")
- parser.add_option("-o", help="Parse a single results directory",
- dest="singledir", action="store_true")
- parser.add_option("-l", help=("Levels of subdirectories to include "
- "in the job name"),
- type="int", dest="level", default=1)
- parser.add_option("-n", help="No blocking on an existing parse",
- dest="noblock", action="store_true")
- parser.add_option("-s", help="Database server hostname",
- dest="db_host", action="store")
- parser.add_option("-u", help="Database username", dest="db_user",
- action="store")
- parser.add_option("-p", help="Database password", dest="db_pass",
- action="store")
- parser.add_option("-d", help="Database name", dest="db_name",
- action="store")
- options, args = parser.parse_args()
+ # build up our options parser and parse sys.argv
+ parser = optparse.OptionParser()
+ parser.add_option("-m", help="Send mail for FAILED tests",
+ dest="mailit", action="store_true")
+ parser.add_option("-r", help="Reparse the results of a job",
+ dest="reparse", action="store_true")
+ parser.add_option("-o", help="Parse a single results directory",
+ dest="singledir", action="store_true")
+ parser.add_option("-l", help=("Levels of subdirectories to include "
+ "in the job name"),
+ type="int", dest="level", default=1)
+ parser.add_option("-n", help="No blocking on an existing parse",
+ dest="noblock", action="store_true")
+ parser.add_option("-s", help="Database server hostname",
+ dest="db_host", action="store")
+ parser.add_option("-u", help="Database username", dest="db_user",
+ action="store")
+ parser.add_option("-p", help="Database password", dest="db_pass",
+ action="store")
+ parser.add_option("-d", help="Database name", dest="db_name",
+ action="store")
+ options, args = parser.parse_args()
- # we need a results directory
- if len(args) == 0:
- tko_utils.dprint("ERROR: at least one results directory must "
- "be provided")
- parser.print_help()
- sys.exit(1)
+ # we need a results directory
+ if len(args) == 0:
+ tko_utils.dprint("ERROR: at least one results directory must "
+ "be provided")
+ parser.print_help()
+ sys.exit(1)
- # pass the options back
- return options, args
+ # pass the options back
+ return options, args
def format_failure_message(jobname, kernel, testname, status, reason):
- format_string = "%-12s %-20s %-12s %-10s %s"
- return format_string % (jobname, kernel, testname, status, reason)
+ format_string = "%-12s %-20s %-12s %-10s %s"
+ return format_string % (jobname, kernel, testname, status, reason)
def mailfailure(jobname, job, message):
- message_lines = [""]
- message_lines.append("The following tests FAILED for this job")
- message_lines.append("http://%s/results/%s" %
- (socket.gethostname(), jobname))
- message_lines.append("")
- message_lines.append(format_failure_message("Job name", "Kernel",
- "Test name", "FAIL/WARN",
- "Failure reason"))
- message_lines.append(format_failure_message("=" * 8, "=" * 6, "=" * 8,
- "=" * 8, "=" * 14))
- message_header = "\n".join(message_lines)
+ message_lines = [""]
+ message_lines.append("The following tests FAILED for this job")
+ message_lines.append("http://%s/results/%s" %
+ (socket.gethostname(), jobname))
+ message_lines.append("")
+ message_lines.append(format_failure_message("Job name", "Kernel",
+ "Test name", "FAIL/WARN",
+ "Failure reason"))
+ message_lines.append(format_failure_message("=" * 8, "=" * 6, "=" * 8,
+ "=" * 8, "=" * 14))
+ message_header = "\n".join(message_lines)
- subject = "AUTOTEST: FAILED tests from job %s" % jobname
- mail.send("", job.user, "", subject, message_header + message)
+ subject = "AUTOTEST: FAILED tests from job %s" % jobname
+ mail.send("", job.user, "", subject, message_header + message)
def parse_one(db, jobname, path, reparse, mail_on_failure):
- """
- Parse a single job. Optionally send email on failure.
- """
- tko_utils.dprint("\nScanning %s (%s)" % (jobname, path))
- if reparse and db.find_job(jobname):
- tko_utils.dprint("! Deleting old copy of job results to "
- "reparse it")
- db.delete_job(jobname)
- if db.find_job(jobname):
- tko_utils.dprint("! Job is already parsed, done")
- return
+ """
+ Parse a single job. Optionally send email on failure.
+ """
+ tko_utils.dprint("\nScanning %s (%s)" % (jobname, path))
+ if reparse and db.find_job(jobname):
+ tko_utils.dprint("! Deleting old copy of job results to "
+ "reparse it")
+ db.delete_job(jobname)
+ if db.find_job(jobname):
+ tko_utils.dprint("! Job is already parsed, done")
+ return
- # look up the status version
- try:
- job_keyval = utils.read_keyval(path)
- except IOError, e:
- if e.errno == errno.ENOENT:
- status_version = 0
- else:
- raise
- else:
- status_version = job_keyval.get("status_version", 0)
+ # look up the status version
+ try:
+ job_keyval = utils.read_keyval(path)
+ except IOError, e:
+ if e.errno == errno.ENOENT:
+ status_version = 0
+ else:
+ raise
+ else:
+ status_version = job_keyval.get("status_version", 0)
- # parse out the job
- parser = status_lib.parser(status_version)
- job = parser.make_job(path)
- status_log = os.path.join(path, "status.log")
- if not os.path.exists(status_log):
- status_log = os.path.join(path, "status")
- if not os.path.exists(status_log):
- tko_utils.dprint("! Unable to parse job, no status file")
- return
+ # parse out the job
+ parser = status_lib.parser(status_version)
+ job = parser.make_job(path)
+ status_log = os.path.join(path, "status.log")
+ if not os.path.exists(status_log):
+ status_log = os.path.join(path, "status")
+ if not os.path.exists(status_log):
+ tko_utils.dprint("! Unable to parse job, no status file")
+ return
- # parse the status logs
- tko_utils.dprint("+ Parsing dir=%s, jobname=%s" % (path, jobname))
- status_lines = open(status_log).readlines()
- parser.start(job)
- tests = parser.end(status_lines)
- job.tests = tests
+ # parse the status logs
+ tko_utils.dprint("+ Parsing dir=%s, jobname=%s" % (path, jobname))
+ status_lines = open(status_log).readlines()
+ parser.start(job)
+ tests = parser.end(status_lines)
+ job.tests = tests
- # check for failures
- message_lines = [""]
- for test in job.tests:
- if not test.subdir:
- continue
- tko_utils.dprint("* testname, status, reason: %s %s %s"
- % (test.subdir, test.status, test.reason))
- if test.status in ("FAIL", "WARN"):
- message_lines.append(format_failure_message(
- jobname, test.kernel.base, test.subdir,
- test.status, test.reason))
- message = "\n".join(message_lines)
+ # check for failures
+ message_lines = [""]
+ for test in job.tests:
+ if not test.subdir:
+ continue
+ tko_utils.dprint("* testname, status, reason: %s %s %s"
+ % (test.subdir, test.status, test.reason))
+ if test.status in ("FAIL", "WARN"):
+ message_lines.append(format_failure_message(
+ jobname, test.kernel.base, test.subdir,
+ test.status, test.reason))
+ message = "\n".join(message_lines)
- # send out a email report of failure
- if len(message) > 2 and mail_on_failure:
- tko_utils.dprint("Sending email report of failure on %s to %s"
- % (jobname, job.user))
- mailfailure(jobname, job, message)
+ # send out a email report of failure
+ if len(message) > 2 and mail_on_failure:
+ tko_utils.dprint("Sending email report of failure on %s to %s"
+ % (jobname, job.user))
+ mailfailure(jobname, job, message)
- # write the job into the database
- db.insert_job(jobname, job)
- db.commit()
+ # write the job into the database
+ db.insert_job(jobname, job)
+ db.commit()
def parse_path(db, path, level, reparse, mail_on_failure):
- machine_list = os.path.join(path, ".machines")
- if os.path.exists(machine_list):
- # multi-machine job
- for m in file(machine_list):
- machine = m.rstrip()
- if not machine:
- continue
- jobpath = os.path.join(path, machine)
- jobname = "%s/%s" % (os.path.basename(path), machine)
- try:
- db.run_with_retry(parse_one, db, jobname,
- path, reparse,
- mail_on_failure)
- except Exception:
- traceback.print_exc()
- continue
- else:
- # single machine job
- job_elements = path.split("/")[-level:]
- jobname = "/".join(job_elements)
- try:
- db.run_with_retry(parse_one, db, jobname, path,
- reparse, mail_on_failure)
- except Exception:
- traceback.print_exc()
+ machine_list = os.path.join(path, ".machines")
+ if os.path.exists(machine_list):
+ # multi-machine job
+ for m in file(machine_list):
+ machine = m.rstrip()
+ if not machine:
+ continue
+ jobpath = os.path.join(path, machine)
+ jobname = "%s/%s" % (os.path.basename(path), machine)
+ try:
+ db.run_with_retry(parse_one, db, jobname,
+ path, reparse,
+ mail_on_failure)
+ except Exception:
+ traceback.print_exc()
+ continue
+ else:
+ # single machine job
+ job_elements = path.split("/")[-level:]
+ jobname = "/".join(job_elements)
+ try:
+ db.run_with_retry(parse_one, db, jobname, path,
+ reparse, mail_on_failure)
+ except Exception:
+ traceback.print_exc()
def main():
- options, args = parse_args()
- results_dir = os.path.abspath(args[0])
- assert os.path.exists(results_dir)
+ options, args = parse_args()
+ results_dir = os.path.abspath(args[0])
+ assert os.path.exists(results_dir)
- # build up the list of job dirs to parse
- if options.singledir:
- jobs_list = [results_dir]
- else:
- jobs_list = [os.path.join(results_dir, subdir)
- for subdir in os.listdir(results_dir)]
+ # build up the list of job dirs to parse
+ if options.singledir:
+ jobs_list = [results_dir]
+ else:
+ jobs_list = [os.path.join(results_dir, subdir)
+ for subdir in os.listdir(results_dir)]
- # build up the database
- db = tko_db.db(autocommit=False, host=options.db_host,
- user=options.db_user, password=options.db_pass,
- database=options.db_name)
+ # build up the database
+ db = tko_db.db(autocommit=False, host=options.db_host,
+ user=options.db_user, password=options.db_pass,
+ database=options.db_name)
- # parse all the jobs
- for path in jobs_list:
- lockfile = open(os.path.join(path, ".parse.lock"), "w")
- flags = fcntl.LOCK_EX
- if options.noblock:
- flags != fcntl.LOCK_NB
- try:
- fcntl.flock(lockfile, flags)
- except IOError, e:
- # was this because the lock is unavailable?
- if e.errno == errno.EWOULDBLOCK:
- lockfile.close()
- continue
- else:
- raise # something unexpected happened
- try:
- parse_path(db, path, options.level, options.reparse,
- options.mailit)
- finally:
- fcntl.flock(lockfile, fcntl.LOCK_UN)
- lockfile.close()
+ # parse all the jobs
+ for path in jobs_list:
+ lockfile = open(os.path.join(path, ".parse.lock"), "w")
+ flags = fcntl.LOCK_EX
+ if options.noblock:
+ flags != fcntl.LOCK_NB
+ try:
+ fcntl.flock(lockfile, flags)
+ except IOError, e:
+ # was this because the lock is unavailable?
+ if e.errno == errno.EWOULDBLOCK:
+ lockfile.close()
+ continue
+ else:
+ raise # something unexpected happened
+ try:
+ parse_path(db, path, options.level, options.reparse,
+ options.mailit)
+ finally:
+ fcntl.flock(lockfile, fcntl.LOCK_UN)
+ lockfile.close()
if __name__ == "__main__":
- main()
+ main()
diff --git a/tko/parsers/base.py b/tko/parsers/base.py
index c0517a8..37429b6 100644
--- a/tko/parsers/base.py
+++ b/tko/parsers/base.py
@@ -4,67 +4,67 @@
class parser(object):
- """
- Abstract parser base class. Provides a generic implementation of the
- standard parser interfaction functions. The derived classes must
- implement a state_iterator method for this class to be useful.
- """
- def start(self, job):
- """ Initialize the parser for processing the results of
- 'job'."""
- # initialize all the basic parser parameters
- self.job = job
- self.finished = False
- self.line_buffer = status_lib.line_buffer()
- # create and prime the parser state machine
- self.state = self.state_iterator(self.line_buffer)
- self.state.next()
+ """
+ Abstract parser base class. Provides a generic implementation of the
+ standard parser interfaction functions. The derived classes must
+ implement a state_iterator method for this class to be useful.
+ """
+ def start(self, job):
+ """ Initialize the parser for processing the results of
+ 'job'."""
+ # initialize all the basic parser parameters
+ self.job = job
+ self.finished = False
+ self.line_buffer = status_lib.line_buffer()
+ # create and prime the parser state machine
+ self.state = self.state_iterator(self.line_buffer)
+ self.state.next()
- def process_lines(self, lines):
- """ Feed 'lines' into the parser state machine, and return
- a list of all the new test results produced."""
- self.line_buffer.put_multiple(lines)
- try:
- return self.state.next()
- except StopIteration:
- msg = ("WARNING: parser was called to process status "
- "lines after it was end()ed\n"
- "Current traceback:\n" +
- traceback.format_exc() +
- "\nCurrent stack:\n" +
- "".join(traceback.format_stack()))
- tko_utils.dprint(msg)
- return []
+ def process_lines(self, lines):
+ """ Feed 'lines' into the parser state machine, and return
+ a list of all the new test results produced."""
+ self.line_buffer.put_multiple(lines)
+ try:
+ return self.state.next()
+ except StopIteration:
+ msg = ("WARNING: parser was called to process status "
+ "lines after it was end()ed\n"
+ "Current traceback:\n" +
+ traceback.format_exc() +
+ "\nCurrent stack:\n" +
+ "".join(traceback.format_stack()))
+ tko_utils.dprint(msg)
+ return []
- def end(self, lines=[]):
- """ Feed 'lines' into the parser state machine, signal to the
- state machine that no more lines are forthcoming, and then
- return a list of all the new test results produced."""
- self.line_buffer.put_multiple(lines)
- # run the state machine to clear out the buffer
- self.finished = True
- try:
- return self.state.next()
- except StopIteration:
- msg = ("WARNING: parser was end()ed multiple times\n"
- "Current traceback:\n" +
- traceback.format_exc() +
- "\nCurrent stack:\n" +
- "".join(traceback.format_stack()))
- tko_utils.dprint(msg)
- return []
+ def end(self, lines=[]):
+ """ Feed 'lines' into the parser state machine, signal to the
+ state machine that no more lines are forthcoming, and then
+ return a list of all the new test results produced."""
+ self.line_buffer.put_multiple(lines)
+ # run the state machine to clear out the buffer
+ self.finished = True
+ try:
+ return self.state.next()
+ except StopIteration:
+ msg = ("WARNING: parser was end()ed multiple times\n"
+ "Current traceback:\n" +
+ traceback.format_exc() +
+ "\nCurrent stack:\n" +
+ "".join(traceback.format_stack()))
+ tko_utils.dprint(msg)
+ return []
- @staticmethod
- def make_job(dir):
- """ Create a new instance of the job model used by the
- parser, given a results directory."""
- raise NotImplemented
+ @staticmethod
+ def make_job(dir):
+ """ Create a new instance of the job model used by the
+ parser, given a results directory."""
+ raise NotImplemented
- def state_iterator(self, buffer):
- """ A generator method that implements the actual parser
- state machine. """
- raise NotImplemented
+ def state_iterator(self, buffer):
+ """ A generator method that implements the actual parser
+ state machine. """
+ raise NotImplemented
diff --git a/tko/parsers/version_0.py b/tko/parsers/version_0.py
index 13d3859..91d0ad7 100644
--- a/tko/parsers/version_0.py
+++ b/tko/parsers/version_0.py
@@ -6,392 +6,392 @@
class job(models.job):
- def __init__(self, dir):
- job_dict = job.load_from_dir(dir)
- super(job, self).__init__(dir, **job_dict)
+ def __init__(self, dir):
+ job_dict = job.load_from_dir(dir)
+ super(job, self).__init__(dir, **job_dict)
- @staticmethod
- def load_from_dir(dir):
- try:
- keyval = common_utils.read_keyval(dir)
- tko_utils.dprint(str(keyval))
- except Exception:
- keyval = {}
+ @staticmethod
+ def load_from_dir(dir):
+ try:
+ keyval = common_utils.read_keyval(dir)
+ tko_utils.dprint(str(keyval))
+ except Exception:
+ keyval = {}
- user = keyval.get("user", None)
- label = keyval.get("label", None)
- machine = keyval.get("hostname", None)
- if machine:
- assert "," not in machine
- queued_time = tko_utils.get_timestamp(keyval, "job_queued")
- started_time = tko_utils.get_timestamp(keyval, "job_started")
- finished_time = tko_utils.get_timestamp(keyval, "job_finished")
- machine_owner = keyval.get("owner", None)
+ user = keyval.get("user", None)
+ label = keyval.get("label", None)
+ machine = keyval.get("hostname", None)
+ if machine:
+ assert "," not in machine
+ queued_time = tko_utils.get_timestamp(keyval, "job_queued")
+ started_time = tko_utils.get_timestamp(keyval, "job_started")
+ finished_time = tko_utils.get_timestamp(keyval, "job_finished")
+ machine_owner = keyval.get("owner", None)
- if not machine:
- machine = job.find_hostname(dir)
- tko_utils.dprint("MACHINE NAME: %s" % machine)
+ if not machine:
+ machine = job.find_hostname(dir)
+ tko_utils.dprint("MACHINE NAME: %s" % machine)
- return {"user": user, "label": label, "machine": machine,
- "queued_time": queued_time,
- "started_time": started_time,
- "finished_time": finished_time,
- "machine_owner": machine_owner}
+ return {"user": user, "label": label, "machine": machine,
+ "queued_time": queued_time,
+ "started_time": started_time,
+ "finished_time": finished_time,
+ "machine_owner": machine_owner}
- @staticmethod
- def find_hostname(path):
- hostname = os.path.join(path, "sysinfo", "hostname")
- try:
- machine = open(hostname).readline().rstrip()
- return machine
- except Exception:
- tko_utils.dprint("Could not read a hostname from "
- "sysinfo/hostname")
+ @staticmethod
+ def find_hostname(path):
+ hostname = os.path.join(path, "sysinfo", "hostname")
+ try:
+ machine = open(hostname).readline().rstrip()
+ return machine
+ except Exception:
+ tko_utils.dprint("Could not read a hostname from "
+ "sysinfo/hostname")
- uname = os.path.join(path, "sysinfo", "uname_-a")
- try:
- machine = open(uname).readline().split()[1]
- return
- except Exception:
- tko_utils.dprint("Could not read a hostname from "
- "sysinfo/uname_-a")
+ uname = os.path.join(path, "sysinfo", "uname_-a")
+ try:
+ machine = open(uname).readline().split()[1]
+ return
+ except Exception:
+ tko_utils.dprint("Could not read a hostname from "
+ "sysinfo/uname_-a")
- raise Exception("Unable to find a machine name")
+ raise Exception("Unable to find a machine name")
class kernel(models.kernel):
- def __init__(self, job, verify_ident=None):
- kernel_dict = kernel.load_from_dir(job.dir, verify_ident)
- super(kernel, self).__init__(**kernel_dict)
+ def __init__(self, job, verify_ident=None):
+ kernel_dict = kernel.load_from_dir(job.dir, verify_ident)
+ super(kernel, self).__init__(**kernel_dict)
- @staticmethod
- def load_from_dir(dir, verify_ident=None):
- # try and load the booted kernel version
- build_log = os.path.join(dir, "build", "debug", "build_log")
- attributes = kernel.load_from_build_log(build_log)
- if not attributes:
- if verify_ident:
- base = verify_ident
- else:
- base = kernel.load_from_sysinfo(dir)
- patches = []
- hashes = []
- else:
- base, patches, hashes = attributes
- tko_utils.dprint("kernel.__init__() found kernel version %s"
- % base)
+ @staticmethod
+ def load_from_dir(dir, verify_ident=None):
+ # try and load the booted kernel version
+ build_log = os.path.join(dir, "build", "debug", "build_log")
+ attributes = kernel.load_from_build_log(build_log)
+ if not attributes:
+ if verify_ident:
+ base = verify_ident
+ else:
+ base = kernel.load_from_sysinfo(dir)
+ patches = []
+ hashes = []
+ else:
+ base, patches, hashes = attributes
+ tko_utils.dprint("kernel.__init__() found kernel version %s"
+ % base)
- # compute the kernel hash
- if base == "UNKNOWN":
- kernel_hash = "UNKNOWN"
- else:
- kernel_hash = kernel.compute_hash(base, hashes)
+ # compute the kernel hash
+ if base == "UNKNOWN":
+ kernel_hash = "UNKNOWN"
+ else:
+ kernel_hash = kernel.compute_hash(base, hashes)
- return {"base": base, "patches": patches,
- "kernel_hash": kernel_hash}
+ return {"base": base, "patches": patches,
+ "kernel_hash": kernel_hash}
- @staticmethod
- def load_from_sysinfo(path):
- for subdir in ("reboot1", ""):
- uname_path = os.path.join(path, "sysinfo", subdir,
- "uname_-a")
- if not os.path.exists(uname_path):
- continue
- uname = open(uname_path).readline().split()
- return re.sub("-autotest$", "", uname[2])
- return "UNKNOWN"
+ @staticmethod
+ def load_from_sysinfo(path):
+ for subdir in ("reboot1", ""):
+ uname_path = os.path.join(path, "sysinfo", subdir,
+ "uname_-a")
+ if not os.path.exists(uname_path):
+ continue
+ uname = open(uname_path).readline().split()
+ return re.sub("-autotest$", "", uname[2])
+ return "UNKNOWN"
- @staticmethod
- def load_from_build_log(path):
- if not os.path.exists(path):
- return None
+ @staticmethod
+ def load_from_build_log(path):
+ if not os.path.exists(path):
+ return None
- base, patches, hashes = "UNKNOWN", [], []
- for line in file(path):
- head, rest = line.split(": ", 1)
- rest = rest.split()
- if head == "BASE":
- base = rest[0]
- elif head == "PATCH":
- patches.append(patch(*rest))
- hashes.append(rest[2])
- return base, patches, hashes
+ base, patches, hashes = "UNKNOWN", [], []
+ for line in file(path):
+ head, rest = line.split(": ", 1)
+ rest = rest.split()
+ if head == "BASE":
+ base = rest[0]
+ elif head == "PATCH":
+ patches.append(patch(*rest))
+ hashes.append(rest[2])
+ return base, patches, hashes
class test(models.test):
- def __init__(self, subdir, testname, status, reason, test_kernel,
- machine, started_time, finished_time, iterations,
- attributes):
- # for backwards compatibility with the original parser
- # implementation, if there is no test version we need a NULL
- # value to be used; also, if there is a version it should
- # be terminated by a newline
- if "version" in attributes:
- attributes["version"] = str(attributes["version"])
- else:
- attributes["version"] = None
+ def __init__(self, subdir, testname, status, reason, test_kernel,
+ machine, started_time, finished_time, iterations,
+ attributes):
+ # for backwards compatibility with the original parser
+ # implementation, if there is no test version we need a NULL
+ # value to be used; also, if there is a version it should
+ # be terminated by a newline
+ if "version" in attributes:
+ attributes["version"] = str(attributes["version"])
+ else:
+ attributes["version"] = None
- super(test, self).__init__(subdir, testname, status, reason,
- test_kernel, machine, started_time,
- finished_time, iterations,
- attributes)
+ super(test, self).__init__(subdir, testname, status, reason,
+ test_kernel, machine, started_time,
+ finished_time, iterations,
+ attributes)
- @staticmethod
- def load_iterations(keyval_path):
- return iteration.load_from_keyval(keyval_path)
+ @staticmethod
+ def load_iterations(keyval_path):
+ return iteration.load_from_keyval(keyval_path)
class patch(models.patch):
- def __init__(self, spec, reference, hash):
- tko_utils.dprint("PATCH::%s %s %s" % (spec, reference, hash))
- super(patch, self).__init__(spec, reference, hash)
- self.spec = spec
- self.reference = reference
- self.hash = hash
+ def __init__(self, spec, reference, hash):
+ tko_utils.dprint("PATCH::%s %s %s" % (spec, reference, hash))
+ super(patch, self).__init__(spec, reference, hash)
+ self.spec = spec
+ self.reference = reference
+ self.hash = hash
class iteration(models.iteration):
- @staticmethod
- def parse_line_into_dicts(line, attr_dict, perf_dict):
- key, value = line.split("=", 1)
- perf_dict[key] = value
+ @staticmethod
+ def parse_line_into_dicts(line, attr_dict, perf_dict):
+ key, value = line.split("=", 1)
+ perf_dict[key] = value
class status_line(object):
- def __init__(self, indent, status, subdir, testname, reason,
- optional_fields):
- # pull out the type & status of the line
- if status == "START":
- self.type = "START"
- self.status = None
- elif status.startswith("END "):
- self.type = "END"
- self.status = status[4:]
- else:
- self.type = "STATUS"
- self.status = status
- assert (self.status is None or
- self.status in status_lib.status_stack.statuses)
+ def __init__(self, indent, status, subdir, testname, reason,
+ optional_fields):
+ # pull out the type & status of the line
+ if status == "START":
+ self.type = "START"
+ self.status = None
+ elif status.startswith("END "):
+ self.type = "END"
+ self.status = status[4:]
+ else:
+ self.type = "STATUS"
+ self.status = status
+ assert (self.status is None or
+ self.status in status_lib.status_stack.statuses)
- # save all the other parameters
- self.indent = indent
- self.subdir = self.parse_name(subdir)
- self.testname = self.parse_name(testname)
- self.reason = reason
- self.optional_fields = optional_fields
+ # save all the other parameters
+ self.indent = indent
+ self.subdir = self.parse_name(subdir)
+ self.testname = self.parse_name(testname)
+ self.reason = reason
+ self.optional_fields = optional_fields
- @staticmethod
- def parse_name(name):
- if name == "----":
- return None
- return name
+ @staticmethod
+ def parse_name(name):
+ if name == "----":
+ return None
+ return name
- @staticmethod
- def is_status_line(line):
- return re.search(r"^\t*\S", line) is not None
+ @staticmethod
+ def is_status_line(line):
+ return re.search(r"^\t*\S", line) is not None
- @classmethod
- def parse_line(cls, line):
- if not status_line.is_status_line(line):
- return None
- indent, line = re.search(r"^(\t*)(.*)$", line).groups()
- indent = len(indent)
+ @classmethod
+ def parse_line(cls, line):
+ if not status_line.is_status_line(line):
+ return None
+ indent, line = re.search(r"^(\t*)(.*)$", line).groups()
+ indent = len(indent)
- # split the line into the fixed and optional fields
- parts = line.split("\t")
- status, subdir, testname = parts[0:3]
- reason = parts[-1]
- optional_parts = parts[3:-1]
+ # split the line into the fixed and optional fields
+ parts = line.split("\t")
+ status, subdir, testname = parts[0:3]
+ reason = parts[-1]
+ optional_parts = parts[3:-1]
- # all the optional parts should be of the form "key=value"
- assert sum('=' not in part for part in optional_parts) == 0
- optional_fields = dict(part.split("=", 1)
- for part in optional_parts)
+ # all the optional parts should be of the form "key=value"
+ assert sum('=' not in part for part in optional_parts) == 0
+ optional_fields = dict(part.split("=", 1)
+ for part in optional_parts)
- # build up a new status_line and return it
- return cls(indent, status, subdir, testname, reason,
- optional_fields)
+ # build up a new status_line and return it
+ return cls(indent, status, subdir, testname, reason,
+ optional_fields)
class parser(base.parser):
- @staticmethod
- def make_job(dir):
- return job(dir)
+ @staticmethod
+ def make_job(dir):
+ return job(dir)
- def state_iterator(self, buffer):
- new_tests = []
- boot_count = 0
- group_subdir = None
- sought_level = 0
- stack = status_lib.status_stack()
- current_kernel = kernel(self.job)
- boot_in_progress = False
- alert_pending = None
- started_time = None
+ def state_iterator(self, buffer):
+ new_tests = []
+ boot_count = 0
+ group_subdir = None
+ sought_level = 0
+ stack = status_lib.status_stack()
+ current_kernel = kernel(self.job)
+ boot_in_progress = False
+ alert_pending = None
+ started_time = None
- while not self.finished or buffer.size():
- # stop processing once the buffer is empty
- if buffer.size() == 0:
- yield new_tests
- new_tests = []
- continue
+ while not self.finished or buffer.size():
+ # stop processing once the buffer is empty
+ if buffer.size() == 0:
+ yield new_tests
+ new_tests = []
+ continue
- # parse the next line
- line = buffer.get()
- tko_utils.dprint('\nSTATUS: ' + line.strip())
- line = status_line.parse_line(line)
- if line is None:
- tko_utils.dprint('non-status line, ignoring')
- continue # ignore non-status lines
+ # parse the next line
+ line = buffer.get()
+ tko_utils.dprint('\nSTATUS: ' + line.strip())
+ line = status_line.parse_line(line)
+ if line is None:
+ tko_utils.dprint('non-status line, ignoring')
+ continue # ignore non-status lines
- # have we hit the job start line?
- if (line.type == "START" and not line.subdir and
- not line.testname):
- sought_level = 1
- tko_utils.dprint("found job level start "
- "marker, looking for level "
- "1 groups now")
- continue
+ # have we hit the job start line?
+ if (line.type == "START" and not line.subdir and
+ not line.testname):
+ sought_level = 1
+ tko_utils.dprint("found job level start "
+ "marker, looking for level "
+ "1 groups now")
+ continue
- # have we hit the job end line?
- if (line.type == "END" and not line.subdir and
- not line.testname):
- tko_utils.dprint("found job level end "
- "marker, looking for level "
- "0 lines now")
- sought_level = 0
+ # have we hit the job end line?
+ if (line.type == "END" and not line.subdir and
+ not line.testname):
+ tko_utils.dprint("found job level end "
+ "marker, looking for level "
+ "0 lines now")
+ sought_level = 0
- # START line, just push another layer on to the stack
- # and grab the start time if this is at the job level
- # we're currently seeking
- if line.type == "START":
- group_subdir = None
- stack.start()
- if line.indent == sought_level:
- started_time = \
- tko_utils.get_timestamp(
- line.optional_fields, "timestamp")
- tko_utils.dprint("start line, ignoring")
- continue
- # otherwise, update the status on the stack
- else:
- tko_utils.dprint("GROPE_STATUS: %s" %
- [stack.current_status(),
- line.status, line.subdir,
- line.testname, line.reason])
- stack.update(line.status)
+ # START line, just push another layer on to the stack
+ # and grab the start time if this is at the job level
+ # we're currently seeking
+ if line.type == "START":
+ group_subdir = None
+ stack.start()
+ if line.indent == sought_level:
+ started_time = \
+ tko_utils.get_timestamp(
+ line.optional_fields, "timestamp")
+ tko_utils.dprint("start line, ignoring")
+ continue
+ # otherwise, update the status on the stack
+ else:
+ tko_utils.dprint("GROPE_STATUS: %s" %
+ [stack.current_status(),
+ line.status, line.subdir,
+ line.testname, line.reason])
+ stack.update(line.status)
- if line.status == "ALERT":
- tko_utils.dprint("job level alert, recording")
- alert_pending = line.reason
- continue
+ if line.status == "ALERT":
+ tko_utils.dprint("job level alert, recording")
+ alert_pending = line.reason
+ continue
- # ignore Autotest.install => GOOD lines
- if (line.testname == "Autotest.install" and
- line.status == "GOOD"):
- tko_utils.dprint("Successful Autotest "
- "install, ignoring")
- continue
+ # ignore Autotest.install => GOOD lines
+ if (line.testname == "Autotest.install" and
+ line.status == "GOOD"):
+ tko_utils.dprint("Successful Autotest "
+ "install, ignoring")
+ continue
- # ignore END lines for a reboot group
- if (line.testname == "reboot" and line.type == "END"):
- tko_utils.dprint("reboot group, ignoring")
- continue
+ # ignore END lines for a reboot group
+ if (line.testname == "reboot" and line.type == "END"):
+ tko_utils.dprint("reboot group, ignoring")
+ continue
- # convert job-level ABORTs into a 'JOB' test, and
- # ignore other job-level events
- if line.testname is None:
- if (line.status == "ABORT" and
- line.type != "END"):
- line.testname = "JOB"
- else:
- tko_utils.dprint("job level event, "
- "ignoring")
- continue
+ # convert job-level ABORTs into a 'JOB' test, and
+ # ignore other job-level events
+ if line.testname is None:
+ if (line.status == "ABORT" and
+ line.type != "END"):
+ line.testname = "JOB"
+ else:
+ tko_utils.dprint("job level event, "
+ "ignoring")
+ continue
- # use the group subdir for END lines
- if line.type == "END":
- line.subdir = group_subdir
+ # use the group subdir for END lines
+ if line.type == "END":
+ line.subdir = group_subdir
- # are we inside a block group?
- if (line.indent != sought_level and
- line.status != "ABORT" and
- not line.testname.startswith('reboot.')):
- if line.subdir:
- tko_utils.dprint("set group_subdir: "
- + line.subdir)
- group_subdir = line.subdir
- tko_utils.dprint("ignoring incorrect indent "
- "level %d != %d," %
- (line.indent, sought_level))
- continue
+ # are we inside a block group?
+ if (line.indent != sought_level and
+ line.status != "ABORT" and
+ not line.testname.startswith('reboot.')):
+ if line.subdir:
+ tko_utils.dprint("set group_subdir: "
+ + line.subdir)
+ group_subdir = line.subdir
+ tko_utils.dprint("ignoring incorrect indent "
+ "level %d != %d," %
+ (line.indent, sought_level))
+ continue
- # use the subdir as the testname, except for
- # boot.* and kernel.* tests
- if (line.testname is None or
- not re.search(r"^(boot(\.\d+)?$|kernel\.)",
- line.testname)):
- if line.subdir and '.' in line.subdir:
- line.testname = line.subdir
+ # use the subdir as the testname, except for
+ # boot.* and kernel.* tests
+ if (line.testname is None or
+ not re.search(r"^(boot(\.\d+)?$|kernel\.)",
+ line.testname)):
+ if line.subdir and '.' in line.subdir:
+ line.testname = line.subdir
- # has a reboot started?
- if line.testname == "reboot.start":
- started_time = tko_utils.get_timestamp(
- line.optional_fields, "timestamp")
- tko_utils.dprint("reboot start event, "
- "ignoring")
- boot_in_progress = True
- continue
+ # has a reboot started?
+ if line.testname == "reboot.start":
+ started_time = tko_utils.get_timestamp(
+ line.optional_fields, "timestamp")
+ tko_utils.dprint("reboot start event, "
+ "ignoring")
+ boot_in_progress = True
+ continue
- # has a reboot finished?
- if line.testname == "reboot.verify":
- line.testname = "boot.%d" % boot_count
- tko_utils.dprint("reboot verified")
- boot_in_progress = False
- verify_ident = line.reason.strip()
- current_kernel = kernel(self.job, verify_ident)
- boot_count += 1
+ # has a reboot finished?
+ if line.testname == "reboot.verify":
+ line.testname = "boot.%d" % boot_count
+ tko_utils.dprint("reboot verified")
+ boot_in_progress = False
+ verify_ident = line.reason.strip()
+ current_kernel = kernel(self.job, verify_ident)
+ boot_count += 1
- if alert_pending:
- line.status = "ALERT"
- line.reason = alert_pending
- alert_pending = None
+ if alert_pending:
+ line.status = "ALERT"
+ line.reason = alert_pending
+ alert_pending = None
- # create the actual test object
- finished_time = tko_utils.get_timestamp(
- line.optional_fields, "timestamp")
- final_status = stack.end()
- tko_utils.dprint("Adding: "
- "%s\nSubdir:%s\nTestname:%s\n%s" %
- (final_status, line.subdir,
- line.testname, line.reason))
- new_test = test.parse_test(self.job, line.subdir,
- line.testname,
- final_status, line.reason,
- current_kernel,
- started_time,
- finished_time)
- started_time = None
- new_tests.append(new_test)
+ # create the actual test object
+ finished_time = tko_utils.get_timestamp(
+ line.optional_fields, "timestamp")
+ final_status = stack.end()
+ tko_utils.dprint("Adding: "
+ "%s\nSubdir:%s\nTestname:%s\n%s" %
+ (final_status, line.subdir,
+ line.testname, line.reason))
+ new_test = test.parse_test(self.job, line.subdir,
+ line.testname,
+ final_status, line.reason,
+ current_kernel,
+ started_time,
+ finished_time)
+ started_time = None
+ new_tests.append(new_test)
- # the job is finished, but we never came back from reboot
- if boot_in_progress:
- testname = "boot.%d" % boot_count
- reason = "machine did not return from reboot"
- tko_utils.dprint(("Adding: ABORT\nSubdir:----\n"
- "Testname:%s\n%s")
- % (testname, reason))
- new_test = test.parse_test(self.job, None, testname,
- "ABORT", reason,
- current_kernel, None, None)
- new_tests.append(new_test)
- yield new_tests
+ # the job is finished, but we never came back from reboot
+ if boot_in_progress:
+ testname = "boot.%d" % boot_count
+ reason = "machine did not return from reboot"
+ tko_utils.dprint(("Adding: ABORT\nSubdir:----\n"
+ "Testname:%s\n%s")
+ % (testname, reason))
+ new_test = test.parse_test(self.job, None, testname,
+ "ABORT", reason,
+ current_kernel, None, None)
+ new_tests.append(new_test)
+ yield new_tests
diff --git a/tko/parsers/version_0_unittest.py b/tko/parsers/version_0_unittest.py
index 947199b..3b879ae 100644
--- a/tko/parsers/version_0_unittest.py
+++ b/tko/parsers/version_0_unittest.py
@@ -7,118 +7,118 @@
class test_status_line(unittest.TestCase):
- statuses = ["GOOD", "WARN", "FAIL", "ABORT"]
+ statuses = ["GOOD", "WARN", "FAIL", "ABORT"]
- def test_handles_start(self):
- line = version_0.status_line(0, "START", "----", "test",
- "", {})
- self.assertEquals(line.type, "START")
- self.assertEquals(line.status, None)
+ def test_handles_start(self):
+ line = version_0.status_line(0, "START", "----", "test",
+ "", {})
+ self.assertEquals(line.type, "START")
+ self.assertEquals(line.status, None)
- def test_handles_status(self):
- for stat in self.statuses:
- line = version_0.status_line(0, stat, "----", "test",
- "", {})
- self.assertEquals(line.type, "STATUS")
- self.assertEquals(line.status, stat)
+ def test_handles_status(self):
+ for stat in self.statuses:
+ line = version_0.status_line(0, stat, "----", "test",
+ "", {})
+ self.assertEquals(line.type, "STATUS")
+ self.assertEquals(line.status, stat)
- def test_handles_endstatus(self):
- for stat in self.statuses:
- line = version_0.status_line(0, "END " + stat, "----",
- "test", "", {})
- self.assertEquals(line.type, "END")
- self.assertEquals(line.status, stat)
+ def test_handles_endstatus(self):
+ for stat in self.statuses:
+ line = version_0.status_line(0, "END " + stat, "----",
+ "test", "", {})
+ self.assertEquals(line.type, "END")
+ self.assertEquals(line.status, stat)
- def test_fails_on_bad_status(self):
- for stat in self.statuses:
- self.assertRaises(AssertionError,
- version_0.status_line, 0,
- "BAD " + stat, "----", "test",
- "", {})
+ def test_fails_on_bad_status(self):
+ for stat in self.statuses:
+ self.assertRaises(AssertionError,
+ version_0.status_line, 0,
+ "BAD " + stat, "----", "test",
+ "", {})
- def test_saves_all_fields(self):
- line = version_0.status_line(5, "GOOD", "subdir_name",
- "test_name", "my reason here",
- {"key1": "value",
- "key2": "another value",
- "key3": "value3"})
- self.assertEquals(line.indent, 5)
- self.assertEquals(line.status, "GOOD")
- self.assertEquals(line.subdir, "subdir_name")
- self.assertEquals(line.testname, "test_name")
- self.assertEquals(line.reason, "my reason here")
- self.assertEquals(line.optional_fields,
- {"key1": "value", "key2": "another value",
- "key3": "value3"})
+ def test_saves_all_fields(self):
+ line = version_0.status_line(5, "GOOD", "subdir_name",
+ "test_name", "my reason here",
+ {"key1": "value",
+ "key2": "another value",
+ "key3": "value3"})
+ self.assertEquals(line.indent, 5)
+ self.assertEquals(line.status, "GOOD")
+ self.assertEquals(line.subdir, "subdir_name")
+ self.assertEquals(line.testname, "test_name")
+ self.assertEquals(line.reason, "my reason here")
+ self.assertEquals(line.optional_fields,
+ {"key1": "value", "key2": "another value",
+ "key3": "value3"})
- def test_parses_blank_subdir(self):
- line = version_0.status_line(0, "GOOD", "----", "test",
- "", {})
- self.assertEquals(line.subdir, None)
+ def test_parses_blank_subdir(self):
+ line = version_0.status_line(0, "GOOD", "----", "test",
+ "", {})
+ self.assertEquals(line.subdir, None)
- def test_parses_blank_testname(self):
- line = version_0.status_line(0, "GOOD", "subdir", "----",
- "", {})
- self.assertEquals(line.testname, None)
+ def test_parses_blank_testname(self):
+ line = version_0.status_line(0, "GOOD", "subdir", "----",
+ "", {})
+ self.assertEquals(line.testname, None)
- def test_parse_line_smoketest(self):
- input_data = ("\t\t\tGOOD\t----\t----\t"
- "field1=val1\tfield2=val2\tTest Passed")
- line = version_0.status_line.parse_line(input_data)
- self.assertEquals(line.indent, 3)
- self.assertEquals(line.type, "STATUS")
- self.assertEquals(line.status, "GOOD")
- self.assertEquals(line.subdir, None)
- self.assertEquals(line.testname, None)
- self.assertEquals(line.reason, "Test Passed")
- self.assertEquals(line.optional_fields,
- {"field1": "val1", "field2": "val2"})
+ def test_parse_line_smoketest(self):
+ input_data = ("\t\t\tGOOD\t----\t----\t"
+ "field1=val1\tfield2=val2\tTest Passed")
+ line = version_0.status_line.parse_line(input_data)
+ self.assertEquals(line.indent, 3)
+ self.assertEquals(line.type, "STATUS")
+ self.assertEquals(line.status, "GOOD")
+ self.assertEquals(line.subdir, None)
+ self.assertEquals(line.testname, None)
+ self.assertEquals(line.reason, "Test Passed")
+ self.assertEquals(line.optional_fields,
+ {"field1": "val1", "field2": "val2"})
- def test_parse_line_handles_newline(self):
- input_data = ("\t\tGOOD\t----\t----\t"
- "field1=val1\tfield2=val2\tNo newline here!")
- for suffix in ("", "\n"):
- line = version_0.status_line.parse_line(input_data +
- suffix)
- self.assertEquals(line.indent, 2)
- self.assertEquals(line.type, "STATUS")
- self.assertEquals(line.status, "GOOD")
- self.assertEquals(line.subdir, None)
- self.assertEquals(line.testname, None)
- self.assertEquals(line.reason, "No newline here!")
- self.assertEquals(line.optional_fields,
- {"field1": "val1",
- "field2": "val2"})
+ def test_parse_line_handles_newline(self):
+ input_data = ("\t\tGOOD\t----\t----\t"
+ "field1=val1\tfield2=val2\tNo newline here!")
+ for suffix in ("", "\n"):
+ line = version_0.status_line.parse_line(input_data +
+ suffix)
+ self.assertEquals(line.indent, 2)
+ self.assertEquals(line.type, "STATUS")
+ self.assertEquals(line.status, "GOOD")
+ self.assertEquals(line.subdir, None)
+ self.assertEquals(line.testname, None)
+ self.assertEquals(line.reason, "No newline here!")
+ self.assertEquals(line.optional_fields,
+ {"field1": "val1",
+ "field2": "val2"})
- def test_parse_line_fails_on_untabbed_lines(self):
- input_data = " GOOD\trandom\tfields\tof text"
- line = version_0.status_line.parse_line(input_data)
- self.assertEquals(line, None)
- line = version_0.status_line.parse_line(input_data.lstrip())
- self.assertEquals(line.indent, 0)
- self.assertEquals(line.type, "STATUS")
- self.assertEquals(line.status, "GOOD")
- self.assertEquals(line.subdir, "random")
- self.assertEquals(line.testname, "fields")
- self.assertEquals(line.reason, "of text")
- self.assertEquals(line.optional_fields, {})
+ def test_parse_line_fails_on_untabbed_lines(self):
+ input_data = " GOOD\trandom\tfields\tof text"
+ line = version_0.status_line.parse_line(input_data)
+ self.assertEquals(line, None)
+ line = version_0.status_line.parse_line(input_data.lstrip())
+ self.assertEquals(line.indent, 0)
+ self.assertEquals(line.type, "STATUS")
+ self.assertEquals(line.status, "GOOD")
+ self.assertEquals(line.subdir, "random")
+ self.assertEquals(line.testname, "fields")
+ self.assertEquals(line.reason, "of text")
+ self.assertEquals(line.optional_fields, {})
- def test_parse_line_fails_on_bad_optional_fields(self):
- input_data = "GOOD\tfield1\tfield2\tfield3\tfield4"
- self.assertRaises(AssertionError,
- version_0.status_line.parse_line,
- input_data)
+ def test_parse_line_fails_on_bad_optional_fields(self):
+ input_data = "GOOD\tfield1\tfield2\tfield3\tfield4"
+ self.assertRaises(AssertionError,
+ version_0.status_line.parse_line,
+ input_data)
if __name__ == "__main__":
- unittest.main()
+ unittest.main()
diff --git a/tko/parsers/version_1.py b/tko/parsers/version_1.py
index b13e272..77c60c2 100644
--- a/tko/parsers/version_1.py
+++ b/tko/parsers/version_1.py
@@ -5,85 +5,85 @@
class kernel(models.kernel):
- def __init__(self, base, patches):
- if base:
- patches = [patch(*p.split()) for p in patches]
- hashes = [p.hash for p in patches]
- kernel_hash = self.compute_hash(base, hashes)
- else:
- base = "UNKNOWN"
- patches = []
- kernel_hash = "UNKNOWN"
- super(kernel, self).__init__(base, patches, kernel_hash)
+ def __init__(self, base, patches):
+ if base:
+ patches = [patch(*p.split()) for p in patches]
+ hashes = [p.hash for p in patches]
+ kernel_hash = self.compute_hash(base, hashes)
+ else:
+ base = "UNKNOWN"
+ patches = []
+ kernel_hash = "UNKNOWN"
+ super(kernel, self).__init__(base, patches, kernel_hash)
class test(models.test):
- @staticmethod
- def load_iterations(keyval_path):
- return iteration.load_from_keyval(keyval_path)
+ @staticmethod
+ def load_iterations(keyval_path):
+ return iteration.load_from_keyval(keyval_path)
class iteration(models.iteration):
- @staticmethod
- def parse_line_into_dicts(line, attr_dict, perf_dict):
- typed_match = re.search("^([^=]*)\{(\w*)\}=(.*)$", line)
- if typed_match:
- key, val_type, value = typed_match.groups()
- if val_type == "attr":
- attr_dict[key] = value
- elif val_type == "perf":
- perf_dict[key] = value
- else:
- msg = ("WARNING: line '%s' found in test "
- "iteration keyval could not be parsed")
- msg %= line
- tko_utils.dprint(msg)
- return # skip the line
- else:
- # old-fashioned untyped match, assume perf
- untyped_match = re.search("^([^=]*)=(.*)$", line)
- if not untyped_match:
- msg = ("WARNING: line '%s' found in test "
- "iteration keyval could not be parsed")
- msg %= line
- tko_utils.dprint(msg)
- return # skip this line
- key, value = untyped_match.groups()
- perf_dict[key] = value
+ @staticmethod
+ def parse_line_into_dicts(line, attr_dict, perf_dict):
+ typed_match = re.search("^([^=]*)\{(\w*)\}=(.*)$", line)
+ if typed_match:
+ key, val_type, value = typed_match.groups()
+ if val_type == "attr":
+ attr_dict[key] = value
+ elif val_type == "perf":
+ perf_dict[key] = value
+ else:
+ msg = ("WARNING: line '%s' found in test "
+ "iteration keyval could not be parsed")
+ msg %= line
+ tko_utils.dprint(msg)
+ return # skip the line
+ else:
+ # old-fashioned untyped match, assume perf
+ untyped_match = re.search("^([^=]*)=(.*)$", line)
+ if not untyped_match:
+ msg = ("WARNING: line '%s' found in test "
+ "iteration keyval could not be parsed")
+ msg %= line
+ tko_utils.dprint(msg)
+ return # skip this line
+ key, value = untyped_match.groups()
+ perf_dict[key] = value
class status_line(version_0.status_line):
- def is_successful_reboot(self, current_status):
- # make sure this is a reboot line
- if self.testname != "reboot":
- return False
+ def is_successful_reboot(self, current_status):
+ # make sure this is a reboot line
+ if self.testname != "reboot":
+ return False
- # make sure this was not a failure
- get_index = status_lib.status_stack.statuses.index
- if get_index(current_status) <= get_index("FAIL"):
- return False
+ # make sure this was not a failure
+ get_index = status_lib.status_stack.statuses.index
+ if get_index(current_status) <= get_index("FAIL"):
+ return False
- # it must have been a successful reboot
- return True
+ # it must have been a successful reboot
+ return True
- def get_kernel(self):
- # get the base kernel version
- fields = self.optional_fields
- base = fields.get("kernel", "")
- # get a list of patches
- patches = []
- patch_index = 0
- while ("patch%d" % patch_index) in fields:
- patches.append(fields["patch%d" % patch_index])
- patch_index += 1
- # create a new kernel instance
- return kernel(base, patches)
+ def get_kernel(self):
+ # get the base kernel version
+ fields = self.optional_fields
+ base = fields.get("kernel", "")
+ # get a list of patches
+ patches = []
+ patch_index = 0
+ while ("patch%d" % patch_index) in fields:
+ patches.append(fields["patch%d" % patch_index])
+ patch_index += 1
+ # create a new kernel instance
+ return kernel(base, patches)
- def get_timestamp(self):
- return tko_utils.get_timestamp(self.optional_fields,
- "timestamp")
+ def get_timestamp(self):
+ return tko_utils.get_timestamp(self.optional_fields,
+ "timestamp")
# the default implementations from version 0 will do for now
@@ -92,127 +92,127 @@
class parser(base.parser):
- @staticmethod
- def make_job(dir):
- return job(dir)
+ @staticmethod
+ def make_job(dir):
+ return job(dir)
- @staticmethod
- def make_dummy_abort(indent):
- indent = "\t" * indent
- return indent + "END ABORT\t----\t----\tUnexpected ABORT"
+ @staticmethod
+ def make_dummy_abort(indent):
+ indent = "\t" * indent
+ return indent + "END ABORT\t----\t----\tUnexpected ABORT"
- def state_iterator(self, buffer):
- new_tests = []
- boot_count = 0
- min_stack_size = 0
- stack = status_lib.status_stack()
- current_kernel = kernel("", []) # UNKNOWN
- started_time_stack = [None]
- subdir_stack = [None]
+ def state_iterator(self, buffer):
+ new_tests = []
+ boot_count = 0
+ min_stack_size = 0
+ stack = status_lib.status_stack()
+ current_kernel = kernel("", []) # UNKNOWN
+ started_time_stack = [None]
+ subdir_stack = [None]
- while True:
- # are we finished with parsing?
- if buffer.size() == 0 and self.finished:
- if stack.size() == 0:
- break
- # we have status lines left on the stack,
- # we need to implicitly abort them first
- for i in reversed(xrange(stack.size())):
- buffer.put(self.make_dummy_abort(i))
+ while True:
+ # are we finished with parsing?
+ if buffer.size() == 0 and self.finished:
+ if stack.size() == 0:
+ break
+ # we have status lines left on the stack,
+ # we need to implicitly abort them first
+ for i in reversed(xrange(stack.size())):
+ buffer.put(self.make_dummy_abort(i))
- # stop processing once the buffer is empty
- if buffer.size() == 0:
- yield new_tests
- new_tests = []
- continue
+ # stop processing once the buffer is empty
+ if buffer.size() == 0:
+ yield new_tests
+ new_tests = []
+ continue
- # reinitialize the per-iteration state
- started_time = None
- finished_time = None
+ # reinitialize the per-iteration state
+ started_time = None
+ finished_time = None
- # get the next line
- raw_line = buffer.get()
- tko_utils.dprint('\nSTATUS: ' + raw_line.strip())
- line = status_line.parse_line(raw_line)
- if line is None:
- tko_utils.dprint('non-status line, ignoring')
- continue
+ # get the next line
+ raw_line = buffer.get()
+ tko_utils.dprint('\nSTATUS: ' + raw_line.strip())
+ line = status_line.parse_line(raw_line)
+ if line is None:
+ tko_utils.dprint('non-status line, ignoring')
+ continue
- # initial line processing
- if line.type == "START":
- stack.start()
- if (line.testname, line.subdir) == (None,) * 2:
- min_stack_size = stack.size()
- started_time_stack.append(line.get_timestamp())
- subdir_stack.append(line.subdir)
- continue
- elif line.type == "STATUS":
- stack.update(line.status)
- indent = line.indent
- started_time = None
- finished_time = line.get_timestamp()
- if line.subdir:
- subdir_stack[-1] = line.subdir
- elif line.type == "END":
- if (line.testname, line.subdir) == (None,) * 2:
- min_stack_size = stack.size() - 1
- subdir_stack.pop()
- else:
- line.subdir = subdir_stack.pop()
- stack.update(line.status)
- indent = line.indent + 1
- started_time = started_time_stack.pop()
- finished_time = line.get_timestamp()
- else:
- assert False
+ # initial line processing
+ if line.type == "START":
+ stack.start()
+ if (line.testname, line.subdir) == (None,) * 2:
+ min_stack_size = stack.size()
+ started_time_stack.append(line.get_timestamp())
+ subdir_stack.append(line.subdir)
+ continue
+ elif line.type == "STATUS":
+ stack.update(line.status)
+ indent = line.indent
+ started_time = None
+ finished_time = line.get_timestamp()
+ if line.subdir:
+ subdir_stack[-1] = line.subdir
+ elif line.type == "END":
+ if (line.testname, line.subdir) == (None,) * 2:
+ min_stack_size = stack.size() - 1
+ subdir_stack.pop()
+ else:
+ line.subdir = subdir_stack.pop()
+ stack.update(line.status)
+ indent = line.indent + 1
+ started_time = started_time_stack.pop()
+ finished_time = line.get_timestamp()
+ else:
+ assert False
- # have we unexpectedly exited a group?
- if indent < stack.size():
- # yes, implicitly ABORT
- buffer.put_back(raw_line)
- abort = self.make_dummy_abort(stack.size() - 1)
- buffer.put_back(abort)
- continue
- else:
- # no, just update the group status
- current_status = line.status
- stack.update(current_status)
+ # have we unexpectedly exited a group?
+ if indent < stack.size():
+ # yes, implicitly ABORT
+ buffer.put_back(raw_line)
+ abort = self.make_dummy_abort(stack.size() - 1)
+ buffer.put_back(abort)
+ continue
+ else:
+ # no, just update the group status
+ current_status = line.status
+ stack.update(current_status)
- # do we need to pop the stack?
- if line.type == "END":
- current_status = stack.end()
- stack.update(current_status)
- if line.is_successful_reboot(current_status):
- current_kernel = line.get_kernel()
- # rename the reboot testname
- if line.testname == "reboot":
- line.testname = "boot.%d" % boot_count
- boot_count += 1
+ # do we need to pop the stack?
+ if line.type == "END":
+ current_status = stack.end()
+ stack.update(current_status)
+ if line.is_successful_reboot(current_status):
+ current_kernel = line.get_kernel()
+ # rename the reboot testname
+ if line.testname == "reboot":
+ line.testname = "boot.%d" % boot_count
+ boot_count += 1
- # have we just finished a test?
- if stack.size() <= min_stack_size:
- # if there was no testname, just use the subdir
- if line.testname is None:
- line.testname = line.subdir
- # if there was no testname or subdir, use 'JOB'
- if line.testname is None:
- line.testname = "JOB"
+ # have we just finished a test?
+ if stack.size() <= min_stack_size:
+ # if there was no testname, just use the subdir
+ if line.testname is None:
+ line.testname = line.subdir
+ # if there was no testname or subdir, use 'JOB'
+ if line.testname is None:
+ line.testname = "JOB"
- new_test = test.parse_test(self.job,
- line.subdir,
- line.testname,
- current_status,
- line.reason,
- current_kernel,
- started_time,
- finished_time)
- msg = "ADD: %s\nSubdir: %s\nTestname: %s\n%s"
- msg %= (new_test.status, new_test.subdir,
- new_test.testname, new_test.reason)
- tko_utils.dprint(msg)
- new_tests.append(new_test)
+ new_test = test.parse_test(self.job,
+ line.subdir,
+ line.testname,
+ current_status,
+ line.reason,
+ current_kernel,
+ started_time,
+ finished_time)
+ msg = "ADD: %s\nSubdir: %s\nTestname: %s\n%s"
+ msg %= (new_test.status, new_test.subdir,
+ new_test.testname, new_test.reason)
+ tko_utils.dprint(msg)
+ new_tests.append(new_test)
- # the job is finished, nothing to do here but exit
- yield new_tests
+ # the job is finished, nothing to do here but exit
+ yield new_tests
diff --git a/tko/parsers/version_1_unittest.py b/tko/parsers/version_1_unittest.py
index eb3ad37..33077c6 100644
--- a/tko/parsers/version_1_unittest.py
+++ b/tko/parsers/version_1_unittest.py
@@ -7,187 +7,187 @@
class test_status_line(unittest.TestCase):
- statuses = ["GOOD", "WARN", "FAIL", "ABORT"]
+ statuses = ["GOOD", "WARN", "FAIL", "ABORT"]
- def test_handles_start(self):
- line = version_1.status_line(0, "START", "----", "test",
- "", {})
- self.assertEquals(line.type, "START")
- self.assertEquals(line.status, None)
+ def test_handles_start(self):
+ line = version_1.status_line(0, "START", "----", "test",
+ "", {})
+ self.assertEquals(line.type, "START")
+ self.assertEquals(line.status, None)
- def test_handles_status(self):
- for stat in self.statuses:
- line = version_1.status_line(0, stat, "----", "test",
- "", {})
- self.assertEquals(line.type, "STATUS")
- self.assertEquals(line.status, stat)
+ def test_handles_status(self):
+ for stat in self.statuses:
+ line = version_1.status_line(0, stat, "----", "test",
+ "", {})
+ self.assertEquals(line.type, "STATUS")
+ self.assertEquals(line.status, stat)
- def test_handles_endstatus(self):
- for stat in self.statuses:
- line = version_1.status_line(0, "END " + stat, "----",
- "test", "", {})
- self.assertEquals(line.type, "END")
- self.assertEquals(line.status, stat)
+ def test_handles_endstatus(self):
+ for stat in self.statuses:
+ line = version_1.status_line(0, "END " + stat, "----",
+ "test", "", {})
+ self.assertEquals(line.type, "END")
+ self.assertEquals(line.status, stat)
- def test_fails_on_bad_status(self):
- for stat in self.statuses:
- self.assertRaises(AssertionError,
- version_1.status_line, 0,
- "BAD " + stat, "----", "test",
- "", {})
+ def test_fails_on_bad_status(self):
+ for stat in self.statuses:
+ self.assertRaises(AssertionError,
+ version_1.status_line, 0,
+ "BAD " + stat, "----", "test",
+ "", {})
- def test_saves_all_fields(self):
- line = version_1.status_line(5, "GOOD", "subdir_name",
- "test_name", "my reason here",
- {"key1": "value",
- "key2": "another value",
- "key3": "value3"})
- self.assertEquals(line.indent, 5)
- self.assertEquals(line.status, "GOOD")
- self.assertEquals(line.subdir, "subdir_name")
- self.assertEquals(line.testname, "test_name")
- self.assertEquals(line.reason, "my reason here")
- self.assertEquals(line.optional_fields,
- {"key1": "value", "key2": "another value",
- "key3": "value3"})
+ def test_saves_all_fields(self):
+ line = version_1.status_line(5, "GOOD", "subdir_name",
+ "test_name", "my reason here",
+ {"key1": "value",
+ "key2": "another value",
+ "key3": "value3"})
+ self.assertEquals(line.indent, 5)
+ self.assertEquals(line.status, "GOOD")
+ self.assertEquals(line.subdir, "subdir_name")
+ self.assertEquals(line.testname, "test_name")
+ self.assertEquals(line.reason, "my reason here")
+ self.assertEquals(line.optional_fields,
+ {"key1": "value", "key2": "another value",
+ "key3": "value3"})
- def test_parses_blank_subdir(self):
- line = version_1.status_line(0, "GOOD", "----", "test",
- "", {})
- self.assertEquals(line.subdir, None)
+ def test_parses_blank_subdir(self):
+ line = version_1.status_line(0, "GOOD", "----", "test",
+ "", {})
+ self.assertEquals(line.subdir, None)
- def test_parses_blank_testname(self):
- line = version_1.status_line(0, "GOOD", "subdir", "----",
- "", {})
- self.assertEquals(line.testname, None)
+ def test_parses_blank_testname(self):
+ line = version_1.status_line(0, "GOOD", "subdir", "----",
+ "", {})
+ self.assertEquals(line.testname, None)
- def test_parse_line_smoketest(self):
- input_data = ("\t\t\tGOOD\t----\t----\t"
- "field1=val1\tfield2=val2\tTest Passed")
- line = version_1.status_line.parse_line(input_data)
- self.assertEquals(line.indent, 3)
- self.assertEquals(line.type, "STATUS")
- self.assertEquals(line.status, "GOOD")
- self.assertEquals(line.subdir, None)
- self.assertEquals(line.testname, None)
- self.assertEquals(line.reason, "Test Passed")
- self.assertEquals(line.optional_fields,
- {"field1": "val1", "field2": "val2"})
+ def test_parse_line_smoketest(self):
+ input_data = ("\t\t\tGOOD\t----\t----\t"
+ "field1=val1\tfield2=val2\tTest Passed")
+ line = version_1.status_line.parse_line(input_data)
+ self.assertEquals(line.indent, 3)
+ self.assertEquals(line.type, "STATUS")
+ self.assertEquals(line.status, "GOOD")
+ self.assertEquals(line.subdir, None)
+ self.assertEquals(line.testname, None)
+ self.assertEquals(line.reason, "Test Passed")
+ self.assertEquals(line.optional_fields,
+ {"field1": "val1", "field2": "val2"})
- def test_parse_line_handles_newline(self):
- input_data = ("\t\tGOOD\t----\t----\t"
- "field1=val1\tfield2=val2\tNo newline here!")
- for suffix in ("", "\n"):
- line = version_1.status_line.parse_line(input_data +
- suffix)
- self.assertEquals(line.indent, 2)
- self.assertEquals(line.type, "STATUS")
- self.assertEquals(line.status, "GOOD")
- self.assertEquals(line.subdir, None)
- self.assertEquals(line.testname, None)
- self.assertEquals(line.reason, "No newline here!")
- self.assertEquals(line.optional_fields,
- {"field1": "val1",
- "field2": "val2"})
+ def test_parse_line_handles_newline(self):
+ input_data = ("\t\tGOOD\t----\t----\t"
+ "field1=val1\tfield2=val2\tNo newline here!")
+ for suffix in ("", "\n"):
+ line = version_1.status_line.parse_line(input_data +
+ suffix)
+ self.assertEquals(line.indent, 2)
+ self.assertEquals(line.type, "STATUS")
+ self.assertEquals(line.status, "GOOD")
+ self.assertEquals(line.subdir, None)
+ self.assertEquals(line.testname, None)
+ self.assertEquals(line.reason, "No newline here!")
+ self.assertEquals(line.optional_fields,
+ {"field1": "val1",
+ "field2": "val2"})
- def test_parse_line_fails_on_untabbed_lines(self):
- input_data = " GOOD\trandom\tfields\tof text"
- line = version_1.status_line.parse_line(input_data)
- self.assertEquals(line, None)
- line = version_1.status_line.parse_line(input_data.lstrip())
- self.assertEquals(line.indent, 0)
- self.assertEquals(line.type, "STATUS")
- self.assertEquals(line.status, "GOOD")
- self.assertEquals(line.subdir, "random")
- self.assertEquals(line.testname, "fields")
- self.assertEquals(line.reason, "of text")
- self.assertEquals(line.optional_fields, {})
+ def test_parse_line_fails_on_untabbed_lines(self):
+ input_data = " GOOD\trandom\tfields\tof text"
+ line = version_1.status_line.parse_line(input_data)
+ self.assertEquals(line, None)
+ line = version_1.status_line.parse_line(input_data.lstrip())
+ self.assertEquals(line.indent, 0)
+ self.assertEquals(line.type, "STATUS")
+ self.assertEquals(line.status, "GOOD")
+ self.assertEquals(line.subdir, "random")
+ self.assertEquals(line.testname, "fields")
+ self.assertEquals(line.reason, "of text")
+ self.assertEquals(line.optional_fields, {})
- def test_parse_line_fails_on_bad_optional_fields(self):
- input_data = "GOOD\tfield1\tfield2\tfield3\tfield4"
- self.assertRaises(AssertionError,
- version_1.status_line.parse_line,
- input_data)
+ def test_parse_line_fails_on_bad_optional_fields(self):
+ input_data = "GOOD\tfield1\tfield2\tfield3\tfield4"
+ self.assertRaises(AssertionError,
+ version_1.status_line.parse_line,
+ input_data)
- def test_good_reboot_passes_success_test(self):
- line = version_1.status_line(0, "NOSTATUS", None, "reboot",
- "reboot success", {})
- self.assertEquals(line.is_successful_reboot("GOOD"), True)
- self.assertEquals(line.is_successful_reboot("WARN"), True)
+ def test_good_reboot_passes_success_test(self):
+ line = version_1.status_line(0, "NOSTATUS", None, "reboot",
+ "reboot success", {})
+ self.assertEquals(line.is_successful_reboot("GOOD"), True)
+ self.assertEquals(line.is_successful_reboot("WARN"), True)
- def test_bad_reboot_passes_success_test(self):
- line = version_1.status_line(0, "NOSTATUS", None, "reboot",
- "reboot success", {})
- self.assertEquals(line.is_successful_reboot("FAIL"), False)
- self.assertEquals(line.is_successful_reboot("ABORT"), False)
+ def test_bad_reboot_passes_success_test(self):
+ line = version_1.status_line(0, "NOSTATUS", None, "reboot",
+ "reboot success", {})
+ self.assertEquals(line.is_successful_reboot("FAIL"), False)
+ self.assertEquals(line.is_successful_reboot("ABORT"), False)
- def test_get_kernel_returns_kernel_plus_patches(self):
- line = version_1.status_line(0, "GOOD", "subdir", "testname",
- "reason text",
- {"kernel": "2.6.24-rc40",
- "patch0": "first_patch 0 0",
- "patch1": "another_patch 0 0"})
- kern = line.get_kernel()
- kernel_hash = md5.new("2.6.24-rc40,0,0").hexdigest()
- self.assertEquals(kern.base, "2.6.24-rc40")
- self.assertEquals(kern.patches[0].spec, "first_patch")
- self.assertEquals(kern.patches[1].spec, "another_patch")
- self.assertEquals(len(kern.patches), 2)
- self.assertEquals(kern.kernel_hash, kernel_hash)
+ def test_get_kernel_returns_kernel_plus_patches(self):
+ line = version_1.status_line(0, "GOOD", "subdir", "testname",
+ "reason text",
+ {"kernel": "2.6.24-rc40",
+ "patch0": "first_patch 0 0",
+ "patch1": "another_patch 0 0"})
+ kern = line.get_kernel()
+ kernel_hash = md5.new("2.6.24-rc40,0,0").hexdigest()
+ self.assertEquals(kern.base, "2.6.24-rc40")
+ self.assertEquals(kern.patches[0].spec, "first_patch")
+ self.assertEquals(kern.patches[1].spec, "another_patch")
+ self.assertEquals(len(kern.patches), 2)
+ self.assertEquals(kern.kernel_hash, kernel_hash)
- def test_get_kernel_ignores_out_of_sequence_patches(self):
- line = version_1.status_line(0, "GOOD", "subdir", "testname",
- "reason text",
- {"kernel": "2.6.24-rc40",
- "patch0": "first_patch 0 0",
- "patch2": "another_patch 0 0"})
- kern = line.get_kernel()
- kernel_hash = md5.new("2.6.24-rc40,0").hexdigest()
- self.assertEquals(kern.base, "2.6.24-rc40")
- self.assertEquals(kern.patches[0].spec, "first_patch")
- self.assertEquals(len(kern.patches), 1)
- self.assertEquals(kern.kernel_hash, kernel_hash)
+ def test_get_kernel_ignores_out_of_sequence_patches(self):
+ line = version_1.status_line(0, "GOOD", "subdir", "testname",
+ "reason text",
+ {"kernel": "2.6.24-rc40",
+ "patch0": "first_patch 0 0",
+ "patch2": "another_patch 0 0"})
+ kern = line.get_kernel()
+ kernel_hash = md5.new("2.6.24-rc40,0").hexdigest()
+ self.assertEquals(kern.base, "2.6.24-rc40")
+ self.assertEquals(kern.patches[0].spec, "first_patch")
+ self.assertEquals(len(kern.patches), 1)
+ self.assertEquals(kern.kernel_hash, kernel_hash)
- def test_get_kernel_returns_unknown_with_no_kernel(self):
- line = version_1.status_line(0, "GOOD", "subdir", "testname",
- "reason text",
- {"patch0": "first_patch 0 0",
- "patch2": "another_patch 0 0"})
- kern = line.get_kernel()
- self.assertEquals(kern.base, "UNKNOWN")
- self.assertEquals(kern.patches, [])
- self.assertEquals(kern.kernel_hash, "UNKNOWN")
+ def test_get_kernel_returns_unknown_with_no_kernel(self):
+ line = version_1.status_line(0, "GOOD", "subdir", "testname",
+ "reason text",
+ {"patch0": "first_patch 0 0",
+ "patch2": "another_patch 0 0"})
+ kern = line.get_kernel()
+ self.assertEquals(kern.base, "UNKNOWN")
+ self.assertEquals(kern.patches, [])
+ self.assertEquals(kern.kernel_hash, "UNKNOWN")
- def test_get_timestamp_returns_timestamp_field(self):
- timestamp = datetime.datetime(1970, 1, 1, 4, 30)
- timestamp -= datetime.timedelta(seconds=time.timezone)
- line = version_1.status_line(0, "GOOD", "subdir", "testname",
- "reason text",
- {"timestamp": "16200"})
- self.assertEquals(timestamp, line.get_timestamp())
+ def test_get_timestamp_returns_timestamp_field(self):
+ timestamp = datetime.datetime(1970, 1, 1, 4, 30)
+ timestamp -= datetime.timedelta(seconds=time.timezone)
+ line = version_1.status_line(0, "GOOD", "subdir", "testname",
+ "reason text",
+ {"timestamp": "16200"})
+ self.assertEquals(timestamp, line.get_timestamp())
- def test_get_timestamp_returns_none_on_missing_field(self):
- line = version_1.status_line(0, "GOOD", "subdir", "testname",
- "reason text", {})
- self.assertEquals(None, line.get_timestamp())
+ def test_get_timestamp_returns_none_on_missing_field(self):
+ line = version_1.status_line(0, "GOOD", "subdir", "testname",
+ "reason text", {})
+ self.assertEquals(None, line.get_timestamp())
if __name__ == "__main__":
- unittest.main()
+ unittest.main()
diff --git a/tko/plotgraph.py b/tko/plotgraph.py
index 8b12cbe..c7af2a3 100755
--- a/tko/plotgraph.py
+++ b/tko/plotgraph.py
@@ -11,97 +11,96 @@
Popen = subprocess.Popen
def avg_dev(values):
- if len(values) == 0:
- return (0,0)
- average = float(sum(values)) / len(values)
- sum_sq_dev = sum( [(x - average) ** 2 for x in values] )
- std_dev = sqrt(sum_sq_dev / float(len(values)));
- return (average, std_dev);
+ if len(values) == 0:
+ return (0,0)
+ average = float(sum(values)) / len(values)
+ sum_sq_dev = sum( [(x - average) ** 2 for x in values] )
+ std_dev = sqrt(sum_sq_dev / float(len(values)));
+ return (average, std_dev);
class gnuplot:
- def __init__(self, title, xlabel, ylabel, xsort = sorted, size = "1180,900", keytitle = None):
- self.title = title
- self.xlabel = xlabel
- self.ylabel = ylabel
- self.data_titles = []
- self.datasets = []
- self.xsort = xsort
- self.xvalues = set([])
- self.size = size
- self.keytitle = keytitle
+ def __init__(self, title, xlabel, ylabel, xsort = sorted, size = "1180,900", keytitle = None):
+ self.title = title
+ self.xlabel = xlabel
+ self.ylabel = ylabel
+ self.data_titles = []
+ self.datasets = []
+ self.xsort = xsort
+ self.xvalues = set([])
+ self.size = size
+ self.keytitle = keytitle
- def xtics(self):
- count = 1
- tics = []
- for label in self.xsort(self.xlabels):
- tics.append('"%s" %d' % (label, count))
- count += 1
- return tics
+ def xtics(self):
+ count = 1
+ tics = []
+ for label in self.xsort(self.xlabels):
+ tics.append('"%s" %d' % (label, count))
+ count += 1
+ return tics
- def add_dataset(self, title, labeled_values):
- """
- Add a data line
+ def add_dataset(self, title, labeled_values):
+ """
+ Add a data line
- title: title of the dataset
- labeled_values: dictionary of lists
- { label : [value1, value2, ... ] , ... }
- """
- if not labeled_values:
- raise "plotgraph:add_dataset - dataset was empty! %s" %\
- title
- self.data_titles.append(title)
- data_points = {}
- for label in labeled_values:
- point = "%s %s" % avg_dev(labeled_values[label])
- data_points[label] = point
- self.xvalues.add(label)
- self.datasets.append(data_points)
+ title: title of the dataset
+ labeled_values: dictionary of lists
+ { label : [value1, value2, ... ] , ... }
+ """
+ if not labeled_values:
+ raise "plotgraph:add_dataset - dataset was empty! %s" %\
+ title
+ self.data_titles.append(title)
+ data_points = {}
+ for label in labeled_values:
+ point = "%s %s" % avg_dev(labeled_values[label])
+ data_points[label] = point
+ self.xvalues.add(label)
+ self.datasets.append(data_points)
- def plot(self, cgi_header = False, output = None, test = None):
- if cgi_header:
- print "Content-type: image/png\n"
- sys.stdout.flush()
- if test:
- g = open(test, 'w')
- else:
- p = Popen("/usr/bin/gnuplot", stdin = subprocess.PIPE)
- g = p.stdin
- g.write('set terminal png size %s\n' % self.size)
- if self.keytitle:
- g.write('set key title "%s"\n' % self.keytitle)
- g.write('set key outside\n') # outside right
- else:
- g.write('set key below\n')
- g.write('set title "%s"\n' % self.title)
- g.write('set xlabel "%s"\n' % self.xlabel)
- g.write('set ylabel "%s"\n' % self.ylabel)
- if output:
- g.write('set output "%s"\n' % output)
- g.write('set style data yerrorlines\n')
- g.write('set grid\n')
+ def plot(self, cgi_header = False, output = None, test = None):
+ if cgi_header:
+ print "Content-type: image/png\n"
+ sys.stdout.flush()
+ if test:
+ g = open(test, 'w')
+ else:
+ p = Popen("/usr/bin/gnuplot", stdin = subprocess.PIPE)
+ g = p.stdin
+ g.write('set terminal png size %s\n' % self.size)
+ if self.keytitle:
+ g.write('set key title "%s"\n' % self.keytitle)
+ g.write('set key outside\n') # outside right
+ else:
+ g.write('set key below\n')
+ g.write('set title "%s"\n' % self.title)
+ g.write('set xlabel "%s"\n' % self.xlabel)
+ g.write('set ylabel "%s"\n' % self.ylabel)
+ if output:
+ g.write('set output "%s"\n' % output)
+ g.write('set style data yerrorlines\n')
+ g.write('set grid\n')
- self.xlabels = self.xsort(list(self.xvalues))
-
- g.write('set xrange [0.5:%f]\n' % (len(self.xvalues)+0.5))
- g.write('set xtics rotate (%s)\n' % ','.join(self.xtics()))
+ self.xlabels = self.xsort(list(self.xvalues))
- plot_lines = ['"-" title "%s"' % t for t in self.data_titles]
- g.write('plot ' + ', '.join(plot_lines) + '\n')
+ g.write('set xrange [0.5:%f]\n' % (len(self.xvalues)+0.5))
+ g.write('set xtics rotate (%s)\n' % ','.join(self.xtics()))
- for dataset in self.datasets:
- count = 1
- for label in self.xlabels:
- if label in dataset:
- data = dataset[label]
- g.write("%d %s\n" % (count, str(data)))
- count += 1
- sys.stdout.flush()
- g.write('e\n')
+ plot_lines = ['"-" title "%s"' % t for t in self.data_titles]
+ g.write('plot ' + ', '.join(plot_lines) + '\n')
- g.close()
- if not test:
- sts = os.waitpid(p.pid, 0)
+ for dataset in self.datasets:
+ count = 1
+ for label in self.xlabels:
+ if label in dataset:
+ data = dataset[label]
+ g.write("%d %s\n" % (count, str(data)))
+ count += 1
+ sys.stdout.flush()
+ g.write('e\n')
+ g.close()
+ if not test:
+ sts = os.waitpid(p.pid, 0)
diff --git a/tko/query_lib.py b/tko/query_lib.py
index 09b7635..35e7280 100644
--- a/tko/query_lib.py
+++ b/tko/query_lib.py
@@ -15,76 +15,75 @@
db = db.db()
def dprint(str):
- pass
- #print "! %s<br>" % str
+ pass
+ #print "! %s<br>" % str
def parse_scrub_and_gen_condition(condition, valid_field_dict):
- me = parse_scrub_and_gen_condition # shorten the name
- compare_ops = {'=':'=', '<>':'<>', '==':'=', '!=':'<>', '>':'>',
- '<':'<', '>=':'>=', '<=':'<=', '~':'LIKE', '#':'REGEXP'}
+ me = parse_scrub_and_gen_condition # shorten the name
+ compare_ops = {'=':'=', '<>':'<>', '==':'=', '!=':'<>', '>':'>',
+ '<':'<', '>=':'>=', '<=':'<=', '~':'LIKE', '#':'REGEXP'}
- # strip white space
- condition = condition.strip()
+ # strip white space
+ condition = condition.strip()
- # ()'s
- #match = re.match(r'^[(](.+)[)]$', condition)
- #if match:
- # dprint("Matched () on %s" % condition)
- # depth = 0
- # for c in match.group(1):
- # if c == '(': depth += 1
- # if c == ')': depth -= 1
- # if depth < 0: break
- # dprint("Depth is %d" % depth)
- # if depth == 0:
- # dprint("Match...stripping ()'s")
- # return me(match.group(1), valid_field_dict)
+ # ()'s
+ #match = re.match(r'^[(](.+)[)]$', condition)
+ #if match:
+ # dprint("Matched () on %s" % condition)
+ # depth = 0
+ # for c in match.group(1):
+ # if c == '(': depth += 1
+ # if c == ')': depth -= 1
+ # if depth < 0: break
+ # dprint("Depth is %d" % depth)
+ # if depth == 0:
+ # dprint("Match...stripping ()'s")
+ # return me(match.group(1), valid_field_dict)
- # OR
- match = re.match(r'^(.+)[|](.+)$', condition)
- if match:
- dprint("Matched | on %s" % condition)
- (a_sql, a_values) = me(match.group(1), valid_field_dict)
- (b_sql, b_values) = me(match.group(2), valid_field_dict)
- return (" (%s) OR (%s) " % (a_sql, b_sql),
- a_values + b_values)
+ # OR
+ match = re.match(r'^(.+)[|](.+)$', condition)
+ if match:
+ dprint("Matched | on %s" % condition)
+ (a_sql, a_values) = me(match.group(1), valid_field_dict)
+ (b_sql, b_values) = me(match.group(2), valid_field_dict)
+ return (" (%s) OR (%s) " % (a_sql, b_sql),
+ a_values + b_values)
- # AND
- match = re.match(r'^(.+)[&](.+)$', condition)
- if match:
- dprint("Matched & on %s" % condition)
- (a_sql, a_values) = me(match.group(1), valid_field_dict)
- (b_sql, b_values) = me(match.group(2), valid_field_dict)
- return (" (%s) AND (%s) " % (a_sql, b_sql),
- a_values + b_values)
+ # AND
+ match = re.match(r'^(.+)[&](.+)$', condition)
+ if match:
+ dprint("Matched & on %s" % condition)
+ (a_sql, a_values) = me(match.group(1), valid_field_dict)
+ (b_sql, b_values) = me(match.group(2), valid_field_dict)
+ return (" (%s) AND (%s) " % (a_sql, b_sql),
+ a_values + b_values)
- # NOT
- #match = re.match(r'^[!](.+)$', condition)
- #if match:
- # dprint("Matched ! on %s" % condition)
- # (sql, values) = me(match.group(1), valid_field_dict)
- # return (" NOT (%s) " % (sql,), values)
+ # NOT
+ #match = re.match(r'^[!](.+)$', condition)
+ #if match:
+ # dprint("Matched ! on %s" % condition)
+ # (sql, values) = me(match.group(1), valid_field_dict)
+ # return (" NOT (%s) " % (sql,), values)
- # '<field> <op> <value>' where value can be quoted
- # double quotes are escaped....i.e. '''' is the same as "'"
- regex = r'^(%s)[ \t]*(%s)[ \t]*' + \
- r'(\'((\'\'|[^\'])*)\'|"((""|[^"])*)"|([^\'"].*))$'
- regex = regex % ('|'.join(valid_field_dict.keys()),
- '|'.join(compare_ops.keys()))
- match = re.match(regex, condition)
- if match:
- field = valid_field_dict[match.group(1)]
- op = compare_ops[match.group(2)]
- if match.group(5):
- val = match.group(4).replace("''", "'")
- elif match.group(7):
- val = match.group(6).replace('""', '"')
- elif match.group(8):
- val = match.group(8)
- else:
- raise "Internal error"
- return ("%s %s %%s" % (field, op), [val])
-
+ # '<field> <op> <value>' where value can be quoted
+ # double quotes are escaped....i.e. '''' is the same as "'"
+ regex = r'^(%s)[ \t]*(%s)[ \t]*' + \
+ r'(\'((\'\'|[^\'])*)\'|"((""|[^"])*)"|([^\'"].*))$'
+ regex = regex % ('|'.join(valid_field_dict.keys()),
+ '|'.join(compare_ops.keys()))
+ match = re.match(regex, condition)
+ if match:
+ field = valid_field_dict[match.group(1)]
+ op = compare_ops[match.group(2)]
+ if match.group(5):
+ val = match.group(4).replace("''", "'")
+ elif match.group(7):
+ val = match.group(6).replace('""', '"')
+ elif match.group(8):
+ val = match.group(8)
+ else:
+ raise "Internal error"
+ return ("%s %s %%s" % (field, op), [val])
- raise "Could not parse '%s' (%s)" % (condition, regex)
+ raise "Could not parse '%s' (%s)" % (condition, regex)
diff --git a/tko/reason_qualifier.py b/tko/reason_qualifier.py
index 29e50d7..6347a4f 100755
--- a/tko/reason_qualifier.py
+++ b/tko/reason_qualifier.py
@@ -2,63 +2,60 @@
class reason_counter:
- def __init__(self, wording):
- self.wording = wording
- self.num = 1
-
- def update(self, new_wording):
- self.num += 1
- self.wording = new_wording
+ def __init__(self, wording):
+ self.wording = wording
+ self.num = 1
- def html(self):
- if self.num == 1:
- return self.wording
- else:
- return "%s (%d+)" % (self.wording, self.num)
+ def update(self, new_wording):
+ self.num += 1
+ self.wording = new_wording
+
+ def html(self):
+ if self.num == 1:
+ return self.wording
+ else:
+ return "%s (%d+)" % (self.wording, self.num)
def numbers_are_irrelevant(txt):
- ## ? when do we replace numbers with NN ?
- ## By default is always, but
- ## if/when some categories of reasons choose to keep their numbers,
- ## then the function shall return False for such categories
- return True
+ ## ? when do we replace numbers with NN ?
+ ## By default is always, but
+ ## if/when some categories of reasons choose to keep their numbers,
+ ## then the function shall return False for such categories
+ return True
def aggregate_reason_fields(reasons_list):
- # each reason in the list may be a combination
- # of | - separated reasons.
- # expand into list
- reasons_txt = '|'.join(reasons_list)
- reasons = reasons_txt.split('|')
- reason_htable = {}
- for reason in reasons:
- reason_reduced = reason.strip()
- ## reduce whitespaces
- reason_reduced = re.sub(r"\s+"," ", reason_reduced)
+ # each reason in the list may be a combination
+ # of | - separated reasons.
+ # expand into list
+ reasons_txt = '|'.join(reasons_list)
+ reasons = reasons_txt.split('|')
+ reason_htable = {}
+ for reason in reasons:
+ reason_reduced = reason.strip()
+ ## reduce whitespaces
+ reason_reduced = re.sub(r"\s+"," ", reason_reduced)
- if reason_reduced == '':
- continue # ignore empty reasons
+ if reason_reduced == '':
+ continue # ignore empty reasons
- if numbers_are_irrelevant(reason_reduced):
- # reduce numbers included into reason descriptor
- # by replacing them with generic NN
- reason_reduced = re.sub(r"\d+","NN", reason_reduced)
+ if numbers_are_irrelevant(reason_reduced):
+ # reduce numbers included into reason descriptor
+ # by replacing them with generic NN
+ reason_reduced = re.sub(r"\d+","NN", reason_reduced)
- if not reason_reduced in reason_htable:
- reason_htable[reason_reduced] = reason_counter(reason)
- else:
- ## reason_counter keeps original ( non reduced )
- ## reason if it occured once
- ## if reason occured more then once, reason_counter
- ## will keep it in reduced/generalized form
- reason_htable[reason_reduced].update(reason_reduced)
+ if not reason_reduced in reason_htable:
+ reason_htable[reason_reduced] = reason_counter(reason)
+ else:
+ ## reason_counter keeps original ( non reduced )
+ ## reason if it occured once
+ ## if reason occured more then once, reason_counter
+ ## will keep it in reduced/generalized form
+ reason_htable[reason_reduced].update(reason_reduced)
- generic_reasons = reason_htable.keys()
- generic_reasons.sort(key = (lambda k: reason_htable[k].num),
- reverse = True)
- return map(lambda generic_reason: reason_htable[generic_reason].html(),
- generic_reasons)
-
-
-
+ generic_reasons = reason_htable.keys()
+ generic_reasons.sort(key = (lambda k: reason_htable[k].num),
+ reverse = True)
+ return map(lambda generic_reason: reason_htable[generic_reason].html(),
+ generic_reasons)
diff --git a/tko/retrieve_jobs b/tko/retrieve_jobs
index 196a9d0..58985c6 100755
--- a/tko/retrieve_jobs
+++ b/tko/retrieve_jobs
@@ -1,11 +1,11 @@
#!/usr/bin/python
import sys, db
-try:
- arg = sys.argv[1]
+try:
+ arg = sys.argv[1]
except:
- arg = ''
+ arg = ''
db = db.db()
for record in db.select('* from jobs ' + arg):
- print record
+ print record
diff --git a/tko/status_lib.py b/tko/status_lib.py
index 4ddfa5a..ab5c644 100644
--- a/tko/status_lib.py
+++ b/tko/status_lib.py
@@ -4,67 +4,67 @@
class status_stack(object):
- statuses = logging.job_statuses
+ statuses = logging.job_statuses
- def __init__(self):
- self.status_stack = [self.statuses[-1]]
+ def __init__(self):
+ self.status_stack = [self.statuses[-1]]
- def current_status(self):
- return self.status_stack[-1]
+ def current_status(self):
+ return self.status_stack[-1]
- def update(self, new_status):
- if new_status not in self.statuses:
- return
- old = self.statuses.index(self.current_status())
- new = self.statuses.index(new_status)
- if new < old:
- self.status_stack[-1] = new_status
+ def update(self, new_status):
+ if new_status not in self.statuses:
+ return
+ old = self.statuses.index(self.current_status())
+ new = self.statuses.index(new_status)
+ if new < old:
+ self.status_stack[-1] = new_status
- def start(self):
- self.status_stack.append(self.statuses[-1])
+ def start(self):
+ self.status_stack.append(self.statuses[-1])
- def end(self):
- result = self.status_stack.pop()
- if len(self.status_stack) == 0:
- self.status_stack.append(self.statuses[-1])
- return result
+ def end(self):
+ result = self.status_stack.pop()
+ if len(self.status_stack) == 0:
+ self.status_stack.append(self.statuses[-1])
+ return result
- def size(self):
- return len(self.status_stack) - 1
+ def size(self):
+ return len(self.status_stack) - 1
class line_buffer(object):
- def __init__(self):
- self.buffer = collections.deque()
+ def __init__(self):
+ self.buffer = collections.deque()
- def get(self):
- return self.buffer.pop()
+ def get(self):
+ return self.buffer.pop()
- def put(self, line):
- self.buffer.appendleft(line)
+ def put(self, line):
+ self.buffer.appendleft(line)
- def put_multiple(self, lines):
- self.buffer.extendleft(lines)
+ def put_multiple(self, lines):
+ self.buffer.extendleft(lines)
- def put_back(self, line):
- self.buffer.append(line)
+ def put_back(self, line):
+ self.buffer.append(line)
- def size(self):
- return len(self.buffer)
+ def size(self):
+ return len(self.buffer)
def parser(version):
- library = "autotest_lib.tko.parsers.version_%d" % version
- module = __import__(library, globals(), locals(), ["parser"])
- return module.parser()
+ library = "autotest_lib.tko.parsers.version_%d" % version
+ module = __import__(library, globals(), locals(), ["parser"])
+ return module.parser()
diff --git a/tko/status_lib_unittest.py b/tko/status_lib_unittest.py
index e8c15f2..2378f97 100644
--- a/tko/status_lib_unittest.py
+++ b/tko/status_lib_unittest.py
@@ -7,168 +7,168 @@
class line_buffer_test(unittest.TestCase):
- def test_get_empty(self):
- buf = status_lib.line_buffer()
- self.assertRaises(IndexError, buf.get)
+ def test_get_empty(self):
+ buf = status_lib.line_buffer()
+ self.assertRaises(IndexError, buf.get)
- def test_get_single(self):
- buf = status_lib.line_buffer()
- buf.put("single line")
- self.assertEquals(buf.get(), "single line")
- self.assertRaises(IndexError, buf.get)
+ def test_get_single(self):
+ buf = status_lib.line_buffer()
+ buf.put("single line")
+ self.assertEquals(buf.get(), "single line")
+ self.assertRaises(IndexError, buf.get)
- def test_is_fifo(self):
- buf = status_lib.line_buffer()
- lines = ["line #%d" for x in xrange(10)]
- for line in lines:
- buf.put(line)
- results = []
- while buf.size():
- results.append(buf.get())
- self.assertEquals(lines, results)
+ def test_is_fifo(self):
+ buf = status_lib.line_buffer()
+ lines = ["line #%d" for x in xrange(10)]
+ for line in lines:
+ buf.put(line)
+ results = []
+ while buf.size():
+ results.append(buf.get())
+ self.assertEquals(lines, results)
- def test_put_multiple_same_as_multiple_puts(self):
- buf_put, buf_multi = [status_lib.line_buffer()
- for x in xrange(2)]
- lines = ["line #%d" % x for x in xrange(10)]
- for line in lines:
- buf_put.put(line)
- buf_multi.put_multiple(lines)
- counter = 0
- while buf_put.size():
- self.assertEquals(buf_put.size(), buf_multi.size())
- line = "line #%d" % counter
- self.assertEquals(buf_put.get(), line)
- self.assertEquals(buf_multi.get(), line)
- counter += 1
+ def test_put_multiple_same_as_multiple_puts(self):
+ buf_put, buf_multi = [status_lib.line_buffer()
+ for x in xrange(2)]
+ lines = ["line #%d" % x for x in xrange(10)]
+ for line in lines:
+ buf_put.put(line)
+ buf_multi.put_multiple(lines)
+ counter = 0
+ while buf_put.size():
+ self.assertEquals(buf_put.size(), buf_multi.size())
+ line = "line #%d" % counter
+ self.assertEquals(buf_put.get(), line)
+ self.assertEquals(buf_multi.get(), line)
+ counter += 1
- def test_put_back_is_lifo(self):
- buf = status_lib.line_buffer()
- lines = ["1", "2", "3"]
- for line in lines:
- buf.put(line)
- results = []
- results.append(buf.get())
- buf.put_back("1")
- buf.put_back("0")
- while buf.size():
- results.append(buf.get())
- self.assertEquals(results, ["1", "0", "1", "2", "3"])
+ def test_put_back_is_lifo(self):
+ buf = status_lib.line_buffer()
+ lines = ["1", "2", "3"]
+ for line in lines:
+ buf.put(line)
+ results = []
+ results.append(buf.get())
+ buf.put_back("1")
+ buf.put_back("0")
+ while buf.size():
+ results.append(buf.get())
+ self.assertEquals(results, ["1", "0", "1", "2", "3"])
- def test_size_increased_by_put(self):
- buf = status_lib.line_buffer()
- self.assertEquals(buf.size(), 0)
- buf.put("1")
- buf.put("2")
- self.assertEquals(buf.size(), 2)
- buf.put("3")
- self.assertEquals(buf.size(), 3)
+ def test_size_increased_by_put(self):
+ buf = status_lib.line_buffer()
+ self.assertEquals(buf.size(), 0)
+ buf.put("1")
+ buf.put("2")
+ self.assertEquals(buf.size(), 2)
+ buf.put("3")
+ self.assertEquals(buf.size(), 3)
- def test_size_increased_by_put(self):
- buf = status_lib.line_buffer()
- self.assertEquals(buf.size(), 0)
- buf.put("1")
- buf.put("2")
- self.assertEquals(buf.size(), 2)
- buf.put("3")
- self.assertEquals(buf.size(), 3)
+ def test_size_increased_by_put(self):
+ buf = status_lib.line_buffer()
+ self.assertEquals(buf.size(), 0)
+ buf.put("1")
+ buf.put("2")
+ self.assertEquals(buf.size(), 2)
+ buf.put("3")
+ self.assertEquals(buf.size(), 3)
- def test_size_decreased_by_get(self):
- buf = status_lib.line_buffer()
- buf.put("1")
- buf.put("2")
- buf.put("3")
- self.assertEquals(buf.size(), 3)
- buf.get()
- self.assertEquals(buf.size(), 2)
- buf.get()
- buf.get()
- self.assertEquals(buf.size(), 0)
+ def test_size_decreased_by_get(self):
+ buf = status_lib.line_buffer()
+ buf.put("1")
+ buf.put("2")
+ buf.put("3")
+ self.assertEquals(buf.size(), 3)
+ buf.get()
+ self.assertEquals(buf.size(), 2)
+ buf.get()
+ buf.get()
+ self.assertEquals(buf.size(), 0)
class status_stack_test(unittest.TestCase):
- statuses = logging.job_statuses
+ statuses = logging.job_statuses
- def test_default_to_nostatus(self):
- stack = status_lib.status_stack()
- self.assertEquals(stack.current_status(), "NOSTATUS")
+ def test_default_to_nostatus(self):
+ stack = status_lib.status_stack()
+ self.assertEquals(stack.current_status(), "NOSTATUS")
- def test_default_on_start_to_nostatus(self):
- stack = status_lib.status_stack()
- stack.update("FAIL")
- stack.start()
- self.assertEquals(stack.current_status(), "NOSTATUS")
+ def test_default_on_start_to_nostatus(self):
+ stack = status_lib.status_stack()
+ stack.update("FAIL")
+ stack.start()
+ self.assertEquals(stack.current_status(), "NOSTATUS")
- def test_size_always_at_least_zero(self):
- stack = status_lib.status_stack()
- self.assertEquals(stack.size(), 0)
- stack.start()
- stack.end()
- self.assertEquals(stack.size(), 0)
- stack.end()
- self.assertEquals(stack.size(), 0)
+ def test_size_always_at_least_zero(self):
+ stack = status_lib.status_stack()
+ self.assertEquals(stack.size(), 0)
+ stack.start()
+ stack.end()
+ self.assertEquals(stack.size(), 0)
+ stack.end()
+ self.assertEquals(stack.size(), 0)
- def test_anything_overrides_nostatus(self):
- for status in self.statuses:
- stack = status_lib.status_stack()
- stack.update(status)
- self.assertEquals(stack.current_status(), status)
+ def test_anything_overrides_nostatus(self):
+ for status in self.statuses:
+ stack = status_lib.status_stack()
+ stack.update(status)
+ self.assertEquals(stack.current_status(), status)
- def test_worse_overrides_better(self):
- for i in xrange(len(self.statuses)):
- worse_status = self.statuses[i]
- for j in xrange(i + 1, len(self.statuses)):
- stack = status_lib.status_stack()
- better_status = self.statuses[j]
- stack.update(better_status)
- stack.update(worse_status)
- self.assertEquals(stack.current_status(),
- worse_status)
+ def test_worse_overrides_better(self):
+ for i in xrange(len(self.statuses)):
+ worse_status = self.statuses[i]
+ for j in xrange(i + 1, len(self.statuses)):
+ stack = status_lib.status_stack()
+ better_status = self.statuses[j]
+ stack.update(better_status)
+ stack.update(worse_status)
+ self.assertEquals(stack.current_status(),
+ worse_status)
- def test_better_never_overrides_better(self):
- for i in xrange(len(self.statuses)):
- better_status = self.statuses[i]
- for j in xrange(i):
- stack = status_lib.status_stack()
- worse_status = self.statuses[j]
- stack.update(worse_status)
- stack.update(better_status)
- self.assertEquals(stack.current_status(),
- worse_status)
+ def test_better_never_overrides_better(self):
+ for i in xrange(len(self.statuses)):
+ better_status = self.statuses[i]
+ for j in xrange(i):
+ stack = status_lib.status_stack()
+ worse_status = self.statuses[j]
+ stack.update(worse_status)
+ stack.update(better_status)
+ self.assertEquals(stack.current_status(),
+ worse_status)
- def test_stack_is_lifo(self):
- stack = status_lib.status_stack()
- stack.update("GOOD")
- stack.start()
- stack.update("FAIL")
- stack.start()
- stack.update("WARN")
- self.assertEquals(stack.end(), "WARN")
- self.assertEquals(stack.end(), "FAIL")
- self.assertEquals(stack.end(), "GOOD")
- self.assertEquals(stack.end(), "NOSTATUS")
+ def test_stack_is_lifo(self):
+ stack = status_lib.status_stack()
+ stack.update("GOOD")
+ stack.start()
+ stack.update("FAIL")
+ stack.start()
+ stack.update("WARN")
+ self.assertEquals(stack.end(), "WARN")
+ self.assertEquals(stack.end(), "FAIL")
+ self.assertEquals(stack.end(), "GOOD")
+ self.assertEquals(stack.end(), "NOSTATUS")
class parser_test(unittest.TestCase):
- available_versions = [0, 1]
- def test_can_import_available_versions(self):
- for version in self.available_versions:
- p = status_lib.parser(0)
- self.assertNotEqual(p, None)
+ available_versions = [0, 1]
+ def test_can_import_available_versions(self):
+ for version in self.available_versions:
+ p = status_lib.parser(0)
+ self.assertNotEqual(p, None)
if __name__ == "__main__":
- unittest.main()
+ unittest.main()
diff --git a/tko/unique_cookie.py b/tko/unique_cookie.py
index 6aed57d..292d12a 100644
--- a/tko/unique_cookie.py
+++ b/tko/unique_cookie.py
@@ -4,32 +4,29 @@
def unique_id(cookie_key):
- """
- Find out if remote caller has cookie set on the key.
- If not, set cookie on client side: evaluate this key by a random string.
- ( unique user identifier )
- In both scenarios return value of the cookie, be it old or newly set one
- """
- uid = ''
- ## try to retrieve uid from Cookie
- if 'HTTP_COOKIE' in os.environ:
- ## parse os.environ['HTTP_COOKIE']
- cookies = os.environ['HTTP_COOKIE'].split(';')
- key = '%s=' % cookie_key
- uid_cookies = [c for c in cookies if c.strip().startswith(key)]
+ """
+ Find out if remote caller has cookie set on the key.
+ If not, set cookie on client side: evaluate this key by a random string.
+ ( unique user identifier )
+ In both scenarios return value of the cookie, be it old or newly set one
+ """
+ uid = ''
+ ## try to retrieve uid from Cookie
+ if 'HTTP_COOKIE' in os.environ:
+ ## parse os.environ['HTTP_COOKIE']
+ cookies = os.environ['HTTP_COOKIE'].split(';')
+ key = '%s=' % cookie_key
+ uid_cookies = [c for c in cookies if c.strip().startswith(key)]
- if uid_cookies:
- assert(len(uid_cookies) == 1)
- uid_cookie = uid_cookies[0]
- uid = uid_cookie.replace(key, '')
+ if uid_cookies:
+ assert(len(uid_cookies) == 1)
+ uid_cookie = uid_cookies[0]
+ uid = uid_cookie.replace(key, '')
- if not uid:
- uid = str(random.random())[2:16] # random string of 14 digits
- set_cookie_statement = 'Set-Cookie:%s=%s;' % (cookie_key, uid)
- set_cookie_statement += 'expires=Thu, 26-Dec-2013 22:03:25 GMT;'
- print set_cookie_statement
+ if not uid:
+ uid = str(random.random())[2:16] # random string of 14 digits
+ set_cookie_statement = 'Set-Cookie:%s=%s;' % (cookie_key, uid)
+ set_cookie_statement += 'expires=Thu, 26-Dec-2013 22:03:25 GMT;'
+ print set_cookie_statement
- return uid
-
-
-
+ return uid
diff --git a/tko/utils.py b/tko/utils.py
index 42815b9..82d9216 100644
--- a/tko/utils.py
+++ b/tko/utils.py
@@ -3,16 +3,16 @@
_debug_logger = sys.stderr
def dprint(msg):
- print >> _debug_logger, msg
+ print >> _debug_logger, msg
def redirect_parser_debugging(ostream):
- global _debug_logger
- _debug_logger = ostream
+ global _debug_logger
+ _debug_logger = ostream
def get_timestamp(mapping, field):
- val = mapping.get(field, None)
- if val is not None:
- val = datetime.datetime.fromtimestamp(int(val))
- return val
+ val = mapping.get(field, None)
+ if val is not None:
+ val = datetime.datetime.fromtimestamp(int(val))
+ return val
diff --git a/tko/utils_unittest.py b/tko/utils_unittest.py
index 9368f95..8e9d8e6 100644
--- a/tko/utils_unittest.py
+++ b/tko/utils_unittest.py
@@ -7,38 +7,38 @@
class get_timestamp_test(unittest.TestCase):
- def testZeroTime(self):
- date = utils.get_timestamp({"key": "0"}, "key")
- timezone = datetime.timedelta(seconds=time.timezone)
- utc_date = date + timezone
- # should be equal to epoch, i.e. Jan 1, 1970
- self.assertEquals(utc_date.year, 1970)
- self.assertEquals(utc_date.month, 1)
- self.assertEquals(utc_date.day, 1)
- self.assertEquals(utc_date.hour, 0)
- self.assertEquals(utc_date.minute, 0)
- self.assertEquals(utc_date.second, 0)
- self.assertEquals(utc_date.microsecond, 0)
+ def testZeroTime(self):
+ date = utils.get_timestamp({"key": "0"}, "key")
+ timezone = datetime.timedelta(seconds=time.timezone)
+ utc_date = date + timezone
+ # should be equal to epoch, i.e. Jan 1, 1970
+ self.assertEquals(utc_date.year, 1970)
+ self.assertEquals(utc_date.month, 1)
+ self.assertEquals(utc_date.day, 1)
+ self.assertEquals(utc_date.hour, 0)
+ self.assertEquals(utc_date.minute, 0)
+ self.assertEquals(utc_date.second, 0)
+ self.assertEquals(utc_date.microsecond, 0)
- def test_returns_none_on_missing_value(self):
- date = utils.get_timestamp({}, "missing_key")
- self.assertEquals(date, None)
+ def test_returns_none_on_missing_value(self):
+ date = utils.get_timestamp({}, "missing_key")
+ self.assertEquals(date, None)
- def test_fails_on_non_integer_values(self):
- self.assertRaises(ValueError, utils.get_timestamp,
- {"key": "zero"}, "key")
+ def test_fails_on_non_integer_values(self):
+ self.assertRaises(ValueError, utils.get_timestamp,
+ {"key": "zero"}, "key")
- def test_date_can_be_string_or_integer(self):
- int_times = [1, 12, 123, 1234, 12345, 123456]
- str_times = [str(t) for t in int_times]
- for int_t, str_t in itertools.izip(int_times, str_times):
- date_int = utils.get_timestamp({"key": int_t}, "key")
- date_str = utils.get_timestamp({"key": str_t}, "key")
- self.assertEquals(date_int, date_str)
+ def test_date_can_be_string_or_integer(self):
+ int_times = [1, 12, 123, 1234, 12345, 123456]
+ str_times = [str(t) for t in int_times]
+ for int_t, str_t in itertools.izip(int_times, str_times):
+ date_int = utils.get_timestamp({"key": int_t}, "key")
+ date_str = utils.get_timestamp({"key": str_t}, "key")
+ self.assertEquals(date_int, date_str)
if __name__ == "__main__":
- unittest.main()
+ unittest.main()
diff --git a/tko/vertical_text.py b/tko/vertical_text.py
index 4173411..371de1c 100755
--- a/tko/vertical_text.py
+++ b/tko/vertical_text.py
@@ -39,4 +39,3 @@
f.close()
simple()
-
diff --git a/ui/dialog.py b/ui/dialog.py
index f5aa4ad..27131c7 100644
--- a/ui/dialog.py
+++ b/ui/dialog.py
@@ -134,7 +134,7 @@
"""
ExceptionShortDescription = "System error"
-
+
class PythonDialogIOError(PythonDialogSystemError):
"""Exception raised when pythondialog catches an IOError exception that \
should be passed to the calling program."""
@@ -244,7 +244,7 @@
"title": lambda title: ("--title", title),
"trim": lambda enable: _simple_option("--trim", enable),
"version": lambda enable: _simple_option("--version", enable)}
-
+
def _simple_option(option, enable):
"""Turn on or off the simplest dialog Common Options."""
@@ -298,7 +298,7 @@
ExecutableNotFound
PythonDialogOSError
-
+
"""
try:
if '/' in f:
@@ -570,7 +570,7 @@
ExecutableNotFound
PythonDialogOSError
- """
+ """
# DIALOGRC differs from the other DIALOG* variables in that:
# 1. It should be a string if not None
# 2. We may very well want it to be unset
@@ -612,18 +612,18 @@
This method is obsolete. Please remove calls to it from your
programs.
- """
- self.add_persistent_args(("--backtitle", text))
+ """
+ self.add_persistent_args(("--backtitle", text))
def _call_program(self, redirect_child_stdin, cmdargs, **kwargs):
- """Do the actual work of invoking the dialog-like program.
+ """Do the actual work of invoking the dialog-like program.
Communication with the dialog-like program is performed
through one or two pipes, depending on
`redirect_child_stdin'. There is always one pipe that is
created to allow the parent process to read what dialog
writes on its standard error stream.
-
+
If `redirect_child_stdin' is True, an additional pipe is
created whose reading end is connected to dialog's standard
input. This is used by the gauge widget to feed data to
@@ -771,7 +771,7 @@
"be the exit status of the dialog-like program, for some "
"unknown reason (-> probably a bug in the dialog-like "
"program); otherwise, we have probably found a python bug")
-
+
# We might want to check here whether exit_code is really one of
# DIALOG_OK, DIALOG_CANCEL, etc. However, I prefer not doing it
# because it would break pythondialog for no strong reason when new
@@ -806,7 +806,7 @@
return (exit_code, child_output)
def _perform(self, cmdargs, **kwargs):
- """Perform a complete dialog-like program invocation.
+ """Perform a complete dialog-like program invocation.
This function invokes the dialog-like program, waits for its
termination and returns its exit status and whatever it wrote
@@ -823,7 +823,7 @@
(exit_code, output) = \
self._wait_for_program_termination(child_pid,
child_rfd)
- return (exit_code, output)
+ return (exit_code, output)
def _strip_xdialog_newline(self, output):
"""Remove trailing newline (if any), if using Xdialog"""
@@ -833,18 +833,18 @@
# This is for compatibility with the old dialog.py
def _perform_no_options(self, cmd):
- """Call dialog without passing any more options."""
- return os.system(self._dialog_prg + ' ' + cmd)
+ """Call dialog without passing any more options."""
+ return os.system(self._dialog_prg + ' ' + cmd)
# For compatibility with the old dialog.py
def clear(self):
- """Clear the screen. Equivalent to the dialog --clear option.
+ """Clear the screen. Equivalent to the dialog --clear option.
This method is obsolete. Please remove calls to it from your
programs.
- """
- self._perform_no_options('--clear')
+ """
+ self._perform_no_options('--clear')
def calendar(self, text, height=6, width=0, day=0, month=0, year=0,
**kwargs):
@@ -857,7 +857,7 @@
month -- inititial month displayed
year -- inititial year selected (0 causes the current date
to be used as the initial date)
-
+
A calendar box displays month, day and year in separately
adjustable windows. If the values for day, month or year are
missing or negative, the current date's corresponding values
@@ -878,8 +878,8 @@
- UnexpectedDialogOutput
- PythonDialogReModuleError
- """
- (code, output) = self._perform(
+ """
+ (code, output) = self._perform(
*(["--calendar", text, str(height), str(width), str(day),
str(month), str(year)],),
**kwargs)
@@ -888,7 +888,7 @@
mo = _calendar_date_rec.match(output)
except re.error, v:
raise PythonDialogReModuleError(v)
-
+
if mo is None:
raise UnexpectedDialogOutput(
"the dialog-like program returned the following "
@@ -900,7 +900,7 @@
def checklist(self, text, height=15, width=54, list_height=7,
choices=[], **kwargs):
- """Display a checklist box.
+ """Display a checklist box.
text -- text to display in the box
height -- height of the box
@@ -937,7 +937,7 @@
# double-quote).
kwargs["separate_output"] = True
- (code, output) = self._perform(*(cmd,), **kwargs)
+ (code, output) = self._perform(*(cmd,), **kwargs)
# Since we used --separate-output, the tags are separated by a newline
# in the output. There is also a final newline after the last tag.
@@ -952,7 +952,7 @@
filepath -- initial file path
height -- height of the box
width -- width of the box
-
+
The file-selection dialog displays a text-entry window in
which you can type a filename (or directory), and above that
two windows with directory names and filenames.
@@ -979,22 +979,22 @@
exit status (an integer) of the dialog-like program and
`path' is the path chosen by the user (whose last element may
be a directory or a file).
-
+
Notable exceptions:
any exception raised by self._perform()
- """
+ """
(code, output) = self._perform(
*(["--fselect", filepath, str(height), str(width)],),
**kwargs)
output = self._strip_xdialog_newline(output)
-
- return (code, output)
-
+
+ return (code, output)
+
def gauge_start(self, text="", height=8, width=54, percent=0, **kwargs):
- """Display gauge box.
+ """Display gauge box.
text -- text to display in the box
height -- height of the box
@@ -1015,20 +1015,20 @@
-------------------
Gauge typical usage (assuming that `d' is an instance of the
- Dialog class) looks like this:
- d.gauge_start()
- # do something
- d.gauge_update(10) # 10% of the whole task is done
- # ...
- d.gauge_update(100, "any text here") # work is done
- exit_code = d.gauge_stop() # cleanup actions
+ Dialog class) looks like this:
+ d.gauge_start()
+ # do something
+ d.gauge_update(10) # 10% of the whole task is done
+ # ...
+ d.gauge_update(100, "any text here") # work is done
+ exit_code = d.gauge_stop() # cleanup actions
Notable exceptions:
- any exception raised by self._call_program()
- PythonDialogOSError
- """
+ """
(child_pid, child_rfd, child_stdin_wfd) = self._call_program(
True,
*(["--gauge", text, str(height), str(width), str(percent)],),
@@ -1041,10 +1041,10 @@
}
except os.error, v:
raise PythonDialogOSError(v.strerror)
-
+
def gauge_update(self, percent, text="", update_text=0):
- """Update a running gauge box.
-
+ """Update a running gauge box.
+
percent -- new percentage to show in the gauge meter
text -- new text to optionally display in the box
update-text -- boolean indicating whether to update the
@@ -1055,8 +1055,8 @@
called previously). If update_text is true (for instance, 1),
the text displayed in the box is also updated.
- See the `gauge_start' function's documentation for
- information about how to use a gauge.
+ See the `gauge_start' function's documentation for
+ information about how to use a gauge.
Return value: undefined.
@@ -1064,28 +1064,28 @@
is an I/O error while writing to the pipe
used to talk to the dialog-like program.
- """
- if update_text:
- gauge_data = "%d\nXXX\n%s\nXXX\n" % (percent, text)
- else:
- gauge_data = "%d\n" % percent
- try:
+ """
+ if update_text:
+ gauge_data = "%d\nXXX\n%s\nXXX\n" % (percent, text)
+ else:
+ gauge_data = "%d\n" % percent
+ try:
self._gauge_process["stdin"].write(gauge_data)
self._gauge_process["stdin"].flush()
except IOError, v:
raise PythonDialogIOError(v)
-
+
# For "compatibility" with the old dialog.py...
gauge_iterate = gauge_update
def gauge_stop(self):
- """Terminate a running gauge.
+ """Terminate a running gauge.
This function performs the appropriate cleanup actions to
terminate a running gauge (started with `gauge_start').
-
- See the `gauge_start' function's documentation for
- information about how to use a gauge.
+
+ See the `gauge_start' function's documentation for
+ information about how to use a gauge.
Return value: undefined.
@@ -1095,7 +1095,7 @@
- PythonDialogIOError can be raised if closing the pipe
used to talk to the dialog-like program fails.
- """
+ """
p = self._gauge_process
# Close the pipe that we are using to feed dialog's stdin
try:
@@ -1129,8 +1129,8 @@
any exception raised by self._perform()
- """
- return self._perform(
+ """
+ return self._perform(
*(["--infobox", text, str(height), str(width)],),
**kwargs)[0]
@@ -1157,14 +1157,14 @@
any exception raised by self._perform()
- """
+ """
(code, tag) = self._perform(
*(["--inputbox", text, str(height), str(width), init],),
**kwargs)
tag = self._strip_xdialog_newline(tag)
-
- return (code, tag)
+
+ return (code, tag)
def menu(self, text, height=15, width=54, menu_height=7, choices=[],
**kwargs):
@@ -1246,14 +1246,14 @@
any exception raised by self._perform()
- """
+ """
cmd = ["--menu", text, str(height), str(width), str(menu_height)]
for t in choices:
cmd.extend(t)
- (code, output) = self._perform(*(cmd,), **kwargs)
+ (code, output) = self._perform(*(cmd,), **kwargs)
output = self._strip_xdialog_newline(output)
-
+
if "help_button" in kwargs.keys() and output.startswith("HELP "):
return ("help", output[5:])
else:
@@ -1281,8 +1281,8 @@
any exception raised by self._perform()
- """
- return self._perform(
+ """
+ return self._perform(
*(["--msgbox", text, str(height), str(width)],),
**kwargs)[0]
@@ -1311,8 +1311,8 @@
any exception raised by self._perform()
- """
- (code, password) = self._perform(
+ """
+ (code, password) = self._perform(
*(["--passwordbox", text, str(height), str(width), init],),
**kwargs)
@@ -1322,7 +1322,7 @@
def radiolist(self, text, height=15, width=54, list_height=7,
choices=[], **kwargs):
- """Display a radiolist box.
+ """Display a radiolist box.
text -- text to display in the box
height -- height of the box
@@ -1352,7 +1352,7 @@
any exception raised by self._perform() or _to_onoff()
- """
+ """
cmd = ["--radiolist", text, str(height), str(width), str(list_height)]
for t in choices:
cmd.extend(((t[0], t[1], _to_onoff(t[2]))))
@@ -1360,11 +1360,11 @@
(code, tag) = self._perform(*(cmd,), **kwargs)
tag = self._strip_xdialog_newline(tag)
-
- return (code, tag)
+
+ return (code, tag)
def scrollbox(self, text, height=20, width=78, **kwargs):
- """Display a string in a scrollable box.
+ """Display a string in a scrollable box.
text -- text to display in the box
height -- height of the box
@@ -1386,7 +1386,7 @@
unfortunately not mentioned in its documentation, at
least in Python 2.3.3...)
- """
+ """
# In Python < 2.3, the standard library does not have
# tempfile.mkstemp(), and unfortunately, tempfile.mktemp() is
# insecure. So, I create a non-world-writable temporary directory and
@@ -1450,8 +1450,8 @@
any exception raised by self._perform()
- """
- return self._perform(
+ """
+ return self._perform(
*(["--tailbox", filename, str(height), str(width)],),
**kwargs)[0]
# No tailboxbg widget, at least for now.
@@ -1480,12 +1480,12 @@
any exception raised by self._perform()
- """
+ """
# This is for backward compatibility... not that it is
# stupid, but I prefer explicit programming.
if not "title" in kwargs.keys():
- kwargs["title"] = filename
- return self._perform(
+ kwargs["title"] = filename
+ return self._perform(
*(["--textbox", filename, str(height), str(width)],),
**kwargs)[0]
@@ -1499,7 +1499,7 @@
hour -- inititial hour selected
minute -- inititial minute selected
second -- inititial second selected
-
+
A dialog is displayed which allows you to select hour, minute
and second. If the values for hour, minute or second are
negative (or not explicitely provided, as they default to
@@ -1520,8 +1520,8 @@
- PythonDialogReModuleError
- UnexpectedDialogOutput
- """
- (code, output) = self._perform(
+ """
+ (code, output) = self._perform(
*(["--timebox", text, str(height), str(width),
str(hour), str(minute), str(second)],),
**kwargs)
@@ -1565,7 +1565,7 @@
any exception raised by self._perform()
- """
- return self._perform(
+ """
+ return self._perform(
*(["--yesno", text, str(height), str(width)],),
**kwargs)[0]
diff --git a/ui/menu b/ui/menu
index 01c4ced..d9a995b 100755
--- a/ui/menu
+++ b/ui/menu
@@ -11,383 +11,383 @@
from menu_lib import *
def handle_exit_code(d, code):
- if code in (d.DIALOG_CANCEL, d.DIALOG_ESC):
- if d.yesno("Do you want to exit Autotest Control Center?") == d.DIALOG_OK:
- sys.exit(0)
- return 0
- else:
- return 1
+ if code in (d.DIALOG_CANCEL, d.DIALOG_ESC):
+ if d.yesno("Do you want to exit Autotest Control Center?") == d.DIALOG_OK:
+ sys.exit(0)
+ return 0
+ else:
+ return 1
def atcc_t_menu(test_type, t):
- u = atcc_list_control_files(test_type, at_dir)
+ u = atcc_list_control_files(test_type, at_dir)
- (code, tag) = d.checklist(text = test_type + ":", choices = u, title = t)
+ (code, tag) = d.checklist(text = test_type + ":", choices = u, title = t)
- return tag
+ return tag
def atcc_t_run(res, test_type):
- if len(res) == 0:
- return -1
+ if len(res) == 0:
+ return -1
- atcc_setup_tmp_dirs_files(menu_dir)
+ atcc_setup_tmp_dirs_files(menu_dir)
- for i in res:
- os.system(at_dir + '/bin/autotest ' + at_dir + '/' + test_type + '/' + i)
- atcc_save_results1(i, at_dir, menu_dir)
+ for i in res:
+ os.system(at_dir + '/bin/autotest ' + at_dir + '/' + test_type + '/' + i)
+ atcc_save_results1(i, at_dir, menu_dir)
def atcc_t_p_run(res, test_type):
- if len(res) == 0:
- return -1
+ if len(res) == 0:
+ return -1
- atcc_setup_tmp_dirs_files(menu_dir)
+ atcc_setup_tmp_dirs_files(menu_dir)
- file = (menu_dir + '/tmp/parallel')
- f = open(file, 'w')
-
- for i in res:
- z = i.split('/')
- line = ("def " + z[0] + "():\n")
- z = str(line)
- f.write(z)
-
- file = (at_dir + '/' + test_type + '/' + i)
- f2 = open(file, 'r')
- k = f2.readlines()
+ file = (menu_dir + '/tmp/parallel')
+ f = open(file, 'w')
- for i in k:
- x = ("\t" + i + "\n")
- z = str(x)
- f.write(z)
+ for i in res:
+ z = i.split('/')
+ line = ("def " + z[0] + "():\n")
+ z = str(line)
+ f.write(z)
- f2.close()
+ file = (at_dir + '/' + test_type + '/' + i)
+ f2 = open(file, 'r')
+ k = f2.readlines()
- f.write('job.parallel(')
+ for i in k:
+ x = ("\t" + i + "\n")
+ z = str(x)
+ f.write(z)
- for i in range(len(res)):
- z = res[i].split('/')
- z = ('[' + z[0] + '],')
- z = str(z)
- f.write(z)
+ f2.close()
- f.write(')')
+ f.write('job.parallel(')
- f.close()
-
- os.system(at_dir + '/bin/autotest ' + menu_dir + '/tmp/parallel')
- atcc_save_results1("", at_dir, menu_dir)
+ for i in range(len(res)):
+ z = res[i].split('/')
+ z = ('[' + z[0] + '],')
+ z = str(z)
+ f.write(z)
+
+ f.write(')')
+
+ f.close()
+
+ os.system(at_dir + '/bin/autotest ' + menu_dir + '/tmp/parallel')
+ atcc_save_results1("", at_dir, menu_dir)
def atcc_profilers_run(res1, res2):
- if len(res1) == 0 or len(res2) == 0:
- return -1
+ if len(res1) == 0 or len(res2) == 0:
+ return -1
- atcc_setup_tmp_dirs_files(menu_dir)
+ atcc_setup_tmp_dirs_files(menu_dir)
- file = (menu_dir + '/tmp/profilers')
- f = open(file, 'w')
+ file = (menu_dir + '/tmp/profilers')
+ f = open(file, 'w')
- f.write('for profiler in (')
+ f.write('for profiler in (')
- for i in range(len(res1)):
- z = res1[i].split('/')
- z = ('\'' + z[0] + '\', ')
- z = str(z)
- f.write(z)
+ for i in range(len(res1)):
+ z = res1[i].split('/')
+ z = ('\'' + z[0] + '\', ')
+ z = str(z)
+ f.write(z)
- f.write('):\n')
+ f.write('):\n')
- f.write('\ttry:\n')
- f.write('\t\tprint "Testing profiler %s ..." % profiler\n')
- f.write('\t\tjob.profilers.add(profiler)\n')
+ f.write('\ttry:\n')
+ f.write('\t\tprint "Testing profiler %s ..." % profiler\n')
+ f.write('\t\tjob.profilers.add(profiler)\n')
- for i in range(len(res2)):
- z = atcc_control_file_read(res2[i], 'tests', at_dir)
- n = res2[i].split('/')
- k = ('\t\tjob.run_test(\'' + n[0] + '\',' + z + ')\n')
- k = str(k)
- f.write(k)
+ for i in range(len(res2)):
+ z = atcc_control_file_read(res2[i], 'tests', at_dir)
+ n = res2[i].split('/')
+ k = ('\t\tjob.run_test(\'' + n[0] + '\',' + z + ')\n')
+ k = str(k)
+ f.write(k)
- f.write('\t\tjob.profilers.delete(profiler)\n')
- f.write('\texcept:\n')
- f.write('\t\tprint "Test of profiler %s failed" % profiler\n')
- f.write('\t\traise\n')
+ f.write('\t\tjob.profilers.delete(profiler)\n')
+ f.write('\texcept:\n')
+ f.write('\t\tprint "Test of profiler %s failed" % profiler\n')
+ f.write('\t\traise\n')
- f.close()
+ f.close()
- os.system(at_dir + '/bin/autotest ' + menu_dir + '/tmp/profilers')
- atcc_save_results2(res1, res2, at_dir, menu_dir)
+ os.system(at_dir + '/bin/autotest ' + menu_dir + '/tmp/profilers')
+ atcc_save_results2(res1, res2, at_dir, menu_dir)
def atcc_remote_get_ip():
- while 1:
- (code, answer) = d.inputbox("IP address of remote host", init = "192.168.0.1")
+ while 1:
+ (code, answer) = d.inputbox("IP address of remote host", init = "192.168.0.1")
- if code == d.DIALOG_CANCEL or code == d.DIALOG_ESC:
- break
- return answer
+ if code == d.DIALOG_CANCEL or code == d.DIALOG_ESC:
+ break
+ return answer
def atcc_t_remote_run(ip, res, test_type):
- if len(res) == 0:
- return -1
+ if len(res) == 0:
+ return -1
- atcc_setup_tmp_dirs_files(menu_dir)
+ atcc_setup_tmp_dirs_files(menu_dir)
- file = (menu_dir + '/tmp/remote_test')
- f = open(file, 'w')
+ file = (menu_dir + '/tmp/remote_test')
+ f = open(file, 'w')
- st = str("remote_host = hosts.SSHHost(\"" + ip +"\")")
- f.write(st + "\n")
- st = str("at = autotest.Autotest()")
- f.write(st + "\n")
+ st = str("remote_host = hosts.SSHHost(\"" + ip +"\")")
+ f.write(st + "\n")
+ st = str("at = autotest.Autotest()")
+ f.write(st + "\n")
- file2 = (menu_dir + '/tmp/remote_control')
- f2 = open(file2, 'w')
+ file2 = (menu_dir + '/tmp/remote_control')
+ f2 = open(file2, 'w')
- for i in res:
- print i
- file3 = (at_dir + '/' + test_type + '/' + i)
- f3 = open(file3, 'r')
- k = f3.readlines()
- for i in k:
- x = (i + "\n")
- z = str(x)
- f2.write(z)
- f3.close()
+ for i in res:
+ print i
+ file3 = (at_dir + '/' + test_type + '/' + i)
+ f3 = open(file3, 'r')
+ k = f3.readlines()
+ for i in k:
+ x = (i + "\n")
+ z = str(x)
+ f2.write(z)
+ f3.close()
- f2.close()
- st = str("at.run(\"" + menu_dir + "/tmp/remote_control\",\"results\", remote_host)")
- f.write(st + "\n")
- f.close()
+ f2.close()
+ st = str("at.run(\"" + menu_dir + "/tmp/remote_control\",\"results\", remote_host)")
+ f.write(st + "\n")
+ f.close()
- os.system(srv_dir + '/autoserv ' + menu_dir + '/tmp/remote_test')
+ os.system(srv_dir + '/autoserv ' + menu_dir + '/tmp/remote_test')
def atcc_t_p_remote_run(ip, res, test_type):
- if len(res) == 0:
- return -1
+ if len(res) == 0:
+ return -1
- atcc_setup_tmp_dirs_files(menu_dir)
+ atcc_setup_tmp_dirs_files(menu_dir)
- file = (menu_dir + '/tmp/remote_parallel_control')
- f = open(file, 'w')
-
- for i in res:
- z = i.split('/')
- line = ("def " + z[0] + "():\n")
- z = str(line)
- f.write(z)
-
- file = (at_dir + '/' + test_type + '/' + i)
- f2 = open(file, 'r')
- k = f2.readlines()
+ file = (menu_dir + '/tmp/remote_parallel_control')
+ f = open(file, 'w')
- for i in k:
- x = ("\t" + i + "\n")
- z = str(x)
- f.write(z)
+ for i in res:
+ z = i.split('/')
+ line = ("def " + z[0] + "():\n")
+ z = str(line)
+ f.write(z)
- f2.close()
+ file = (at_dir + '/' + test_type + '/' + i)
+ f2 = open(file, 'r')
+ k = f2.readlines()
- f.write('job.parallel(')
+ for i in k:
+ x = ("\t" + i + "\n")
+ z = str(x)
+ f.write(z)
- for i in range(len(res)):
- z = res[i].split('/')
- z = ('[' + z[0] + '],')
- z = str(z)
- f.write(z)
+ f2.close()
- f.write(')')
+ f.write('job.parallel(')
- f.close()
+ for i in range(len(res)):
+ z = res[i].split('/')
+ z = ('[' + z[0] + '],')
+ z = str(z)
+ f.write(z)
- file = (menu_dir + '/tmp/remote_parallel_test')
- f = open(file, 'w')
+ f.write(')')
- st = str("remote_host = hosts.SSHHost(\"" + ip +"\")")
- f.write(st + "\n")
- st = str("at = autotest.Autotest()")
- f.write(st + "\n")
+ f.close()
- st = str("at.run(\"" + menu_dir + "/tmp/remote_parallel_control\",\"results\", remote_host)")
- f.write(st + "\n")
- f.close()
+ file = (menu_dir + '/tmp/remote_parallel_test')
+ f = open(file, 'w')
- os.system(srv_dir + '/autoserv ' + menu_dir + '/tmp/remote_parallel_test')
+ st = str("remote_host = hosts.SSHHost(\"" + ip +"\")")
+ f.write(st + "\n")
+ st = str("at = autotest.Autotest()")
+ f.write(st + "\n")
+
+ st = str("at.run(\"" + menu_dir + "/tmp/remote_parallel_control\",\"results\", remote_host)")
+ f.write(st + "\n")
+ f.close()
+
+ os.system(srv_dir + '/autoserv ' + menu_dir + '/tmp/remote_parallel_test')
def atcc_profilers_remote_run(ip, res1, res2):
- if len(res1) == 0 or len(res2) == 0:
- return -1
+ if len(res1) == 0 or len(res2) == 0:
+ return -1
- atcc_setup_tmp_dirs_files(menu_dir)
+ atcc_setup_tmp_dirs_files(menu_dir)
- file = (menu_dir + '/tmp/remote_profilers_control')
- f = open(file, 'w')
+ file = (menu_dir + '/tmp/remote_profilers_control')
+ f = open(file, 'w')
- f.write('for profiler in (')
+ f.write('for profiler in (')
- for i in range(len(res1)):
- z = res1[i].split('/')
- z = ('\'' + z[0] + '\', ')
- z = str(z)
- f.write(z)
+ for i in range(len(res1)):
+ z = res1[i].split('/')
+ z = ('\'' + z[0] + '\', ')
+ z = str(z)
+ f.write(z)
- f.write('):\n')
+ f.write('):\n')
- f.write('\ttry:\n')
- f.write('\t\tprint "Testing profiler %s ..." % profiler\n')
- f.write('\t\tjob.profilers.add(profiler)\n')
+ f.write('\ttry:\n')
+ f.write('\t\tprint "Testing profiler %s ..." % profiler\n')
+ f.write('\t\tjob.profilers.add(profiler)\n')
- for i in range(len(res2)):
- z = atcc_control_file_read(res2[i], 'tests', at_dir)
- n = res2[i].split('/')
- k = ('\t\tjob.run_test(\'' + n[0] + '\',' + z + ')\n')
- k = str(k)
- f.write(k)
+ for i in range(len(res2)):
+ z = atcc_control_file_read(res2[i], 'tests', at_dir)
+ n = res2[i].split('/')
+ k = ('\t\tjob.run_test(\'' + n[0] + '\',' + z + ')\n')
+ k = str(k)
+ f.write(k)
- f.write('\t\tjob.profilers.delete(profiler)\n')
- f.write('\texcept:\n')
- f.write('\t\tprint "Test of profiler %s failed" % profiler\n')
- f.write('\t\traise\n')
+ f.write('\t\tjob.profilers.delete(profiler)\n')
+ f.write('\texcept:\n')
+ f.write('\t\tprint "Test of profiler %s failed" % profiler\n')
+ f.write('\t\traise\n')
- f.close()
+ f.close()
- file = (menu_dir + '/tmp/remote_profilers')
- f = open(file, 'w')
+ file = (menu_dir + '/tmp/remote_profilers')
+ f = open(file, 'w')
- st = str("remote_host = hosts.SSHHost(\"" + ip +"\")")
- f.write(st + "\n")
- st = str("at = autotest.Autotest()")
- f.write(st + "\n")
+ st = str("remote_host = hosts.SSHHost(\"" + ip +"\")")
+ f.write(st + "\n")
+ st = str("at = autotest.Autotest()")
+ f.write(st + "\n")
- st = str("at.run(\"" + menu_dir + "/tmp/remote_profilers_control\",\"results\", remote_host)")
- f.write(st + "\n")
- f.close()
+ st = str("at.run(\"" + menu_dir + "/tmp/remote_profilers_control\",\"results\", remote_host)")
+ f.write(st + "\n")
+ f.close()
- os.system(srv_dir + '/autoserv ' + menu_dir + '/tmp/remote_profilers')
+ os.system(srv_dir + '/autoserv ' + menu_dir + '/tmp/remote_profilers')
def atcc_tests_results(t):
- if os.path.exists(menu_dir + "/tmp/"):
- dir_ls = dircache.listdir(menu_dir + "/tmp/")
- else:
- d.infobox(menu_dir + "/tmp/ doesn't exist")
- time.sleep(5)
- return -1
+ if os.path.exists(menu_dir + "/tmp/"):
+ dir_ls = dircache.listdir(menu_dir + "/tmp/")
+ else:
+ d.infobox(menu_dir + "/tmp/ doesn't exist")
+ time.sleep(5)
+ return -1
- if len(dir_ls) == 0:
- return -1
+ if len(dir_ls) == 0:
+ return -1
- u = []
+ u = []
- for i in dir_ls:
- k = i, ""
- u.append(k)
+ for i in dir_ls:
+ k = i, ""
+ u.append(k)
- while 1:
- (code, tag) = d.menu("Results:", choices = u, title = t)
+ while 1:
+ (code, tag) = d.menu("Results:", choices = u, title = t)
- if code == d.DIALOG_CANCEL or code == d.DIALOG_ESC:
- break
- else:
- d.textbox(menu_dir + '/tmp/' + tag, width = -1)
+ if code == d.DIALOG_CANCEL or code == d.DIALOG_ESC:
+ break
+ else:
+ d.textbox(menu_dir + '/tmp/' + tag, width = -1)
def atcc_config_show_help(tag, test_type):
- tag = tag.split('/')
- tag = tag[0]
- if os.path.exists(at_dir + '/' + test_type + '/' + tag + '/help'):
- d.textbox(at_dir + '/' + test_type + '/' + tag + '/help', width = -1)
- else:
- d.infobox(at_dir + '/' + test_type + '/' + tag + '/help' " doesn't exist")
- time.sleep(5)
+ tag = tag.split('/')
+ tag = tag[0]
+ if os.path.exists(at_dir + '/' + test_type + '/' + tag + '/help'):
+ d.textbox(at_dir + '/' + test_type + '/' + tag + '/help', width = -1)
+ else:
+ d.infobox(at_dir + '/' + test_type + '/' + tag + '/help' " doesn't exist")
+ time.sleep(5)
def atcc_config_edit(tag, test_type, ed):
- os.system(ed + " " + at_dir + '/' + test_type + '/' + tag)
+ os.system(ed + " " + at_dir + '/' + test_type + '/' + tag)
def atcc_config_help_or_edit(tag, test_type):
- ed = "vim"
- while 1:
- (code, answer) = d.inputbox("Type 'help' to see documentation, or name of favourite text editor", init = ed)
+ ed = "vim"
+ while 1:
+ (code, answer) = d.inputbox("Type 'help' to see documentation, or name of favourite text editor", init = ed)
- if code == d.DIALOG_CANCEL or code == d.DIALOG_ESC:
- break
- elif answer == "help":
- atcc_config_show_help(tag, test_type)
- continue
- else:
- ed = answer
- atcc_config_edit(tag, test_type, ed)
- break
+ if code == d.DIALOG_CANCEL or code == d.DIALOG_ESC:
+ break
+ elif answer == "help":
+ atcc_config_show_help(tag, test_type)
+ continue
+ else:
+ ed = answer
+ atcc_config_edit(tag, test_type, ed)
+ break
def atcc_config_choose_control_file(test_type, t):
- u_tmp = atcc_list_control_files(test_type, at_dir)
- u = []
- for i in u_tmp:
- k = i[0], ""
- u.append(k)
+ u_tmp = atcc_list_control_files(test_type, at_dir)
+ u = []
+ for i in u_tmp:
+ k = i[0], ""
+ u.append(k)
- while 1:
- (code, tag) = d.menu(test_type + ":", choices = u, title = t)
+ while 1:
+ (code, tag) = d.menu(test_type + ":", choices = u, title = t)
- if code == d.DIALOG_CANCEL or code == d.DIALOG_ESC:
- break
- else:
- atcc_config_help_or_edit(tag, test_type)
+ if code == d.DIALOG_CANCEL or code == d.DIALOG_ESC:
+ break
+ else:
+ atcc_config_help_or_edit(tag, test_type)
def atcc_upgrade():
- os.system("svn checkout svn://test.kernel.org/autotest/trunk " + at_dir + "/../")
+ os.system("svn checkout svn://test.kernel.org/autotest/trunk " + at_dir + "/../")
def atcc_rsync_mirror():
- os.system(at_dir + "/../mirror/mirror")
+ os.system(at_dir + "/../mirror/mirror")
def atcc_main_menu():
- while 1:
- (code, tag) = d.menu("Main menu",
- choices = [("1", "Tests (local)"),
- ("2", "Parallel tests (local)"),
- ("3", "Profilers (local)"),
- ("4", "Tests (remote)"),
- ("5", "Parallel tests (remote)"),
- ("6", "Profilers (remote)"),
- ("7", "Tests' results"),
- ("8", "Configure tests"),
- ("9", "Upgrade Autotest"),
- ("10", "Sync local kernel.org mirror")])
- if handle_exit_code(d, code):
- break
- return tag
+ while 1:
+ (code, tag) = d.menu("Main menu",
+ choices = [("1", "Tests (local)"),
+ ("2", "Parallel tests (local)"),
+ ("3", "Profilers (local)"),
+ ("4", "Tests (remote)"),
+ ("5", "Parallel tests (remote)"),
+ ("6", "Profilers (remote)"),
+ ("7", "Tests' results"),
+ ("8", "Configure tests"),
+ ("9", "Upgrade Autotest"),
+ ("10", "Sync local kernel.org mirror")])
+ if handle_exit_code(d, code):
+ break
+ return tag
def main():
- while 1:
- res = int(atcc_main_menu())
- if res == 1:
- res = atcc_t_menu(test_type = 'tests', t = 'Tests selection menu')
- atcc_t_run(res, test_type = 'tests')
- elif res == 2:
- res = atcc_t_menu(test_type = 'tests', t = 'Parallel tests selection menu')
- atcc_t_p_run(res, test_type = 'tests')
- elif res == 3:
- res1 = atcc_t_menu(test_type = 'profilers', t = 'Profilers selection menu')
- res2 = atcc_t_menu(test_type = 'tests', t = 'Tests selection menu')
- atcc_profilers_run(res1, res2)
- elif res == 4:
- ip = atcc_remote_get_ip()
- res = atcc_t_menu(test_type = 'tests', t = 'Tests selection menu')
- atcc_t_remote_run(ip, res, test_type = 'tests')
- elif res == 5:
- ip = atcc_remote_get_ip()
- res = atcc_t_menu(test_type = 'tests', t = 'Parallel tests selection menu')
- atcc_t_p_remote_run(ip, res, test_type = 'tests')
- elif res == 6:
- ip = atcc_remote_get_ip()
- res1 = atcc_t_menu(test_type = 'profilers', t = 'Profilers selection menu')
- res2 = atcc_t_menu(test_type = 'tests', t = 'Tests selection menu')
- atcc_profilers_remote_run(ip, res1, res2)
- elif res == 7:
- atcc_tests_results(t = 'Tests\' results menu')
- elif res == 8:
- atcc_config_choose_control_file(test_type = 'tests', t = 'Tests configuration menu')
- elif res == 9:
- atcc_upgrade()
- elif res == 10:
- atcc_rsync_mirror()
- elif res == 0:
- sys.exit(1)
+ while 1:
+ res = int(atcc_main_menu())
+ if res == 1:
+ res = atcc_t_menu(test_type = 'tests', t = 'Tests selection menu')
+ atcc_t_run(res, test_type = 'tests')
+ elif res == 2:
+ res = atcc_t_menu(test_type = 'tests', t = 'Parallel tests selection menu')
+ atcc_t_p_run(res, test_type = 'tests')
+ elif res == 3:
+ res1 = atcc_t_menu(test_type = 'profilers', t = 'Profilers selection menu')
+ res2 = atcc_t_menu(test_type = 'tests', t = 'Tests selection menu')
+ atcc_profilers_run(res1, res2)
+ elif res == 4:
+ ip = atcc_remote_get_ip()
+ res = atcc_t_menu(test_type = 'tests', t = 'Tests selection menu')
+ atcc_t_remote_run(ip, res, test_type = 'tests')
+ elif res == 5:
+ ip = atcc_remote_get_ip()
+ res = atcc_t_menu(test_type = 'tests', t = 'Parallel tests selection menu')
+ atcc_t_p_remote_run(ip, res, test_type = 'tests')
+ elif res == 6:
+ ip = atcc_remote_get_ip()
+ res1 = atcc_t_menu(test_type = 'profilers', t = 'Profilers selection menu')
+ res2 = atcc_t_menu(test_type = 'tests', t = 'Tests selection menu')
+ atcc_profilers_remote_run(ip, res1, res2)
+ elif res == 7:
+ atcc_tests_results(t = 'Tests\' results menu')
+ elif res == 8:
+ atcc_config_choose_control_file(test_type = 'tests', t = 'Tests configuration menu')
+ elif res == 9:
+ atcc_upgrade()
+ elif res == 10:
+ atcc_rsync_mirror()
+ elif res == 0:
+ sys.exit(1)
check_python_version()
diff --git a/ui/menu_lib.py b/ui/menu_lib.py
index 6435d55..e2f8c3c 100644
--- a/ui/menu_lib.py
+++ b/ui/menu_lib.py
@@ -1,87 +1,87 @@
import os, sys, dircache, string, re
def check_python_version():
- version = sys.version_info[0:2]
- if version < (2, 4):
- print "Python 2.4 or newer is needed"
- sys.exit(1)
+ version = sys.version_info[0:2]
+ if version < (2, 4):
+ print "Python 2.4 or newer is needed"
+ sys.exit(1)
def atcc_list_control_files(test_type, at_dir):
- dir_ls = dircache.listdir(at_dir + '/' + test_type)
- u = []
- for i in dir_ls:
- if i != ".svn":
- dir_ls2 = dircache.listdir(at_dir + '/' + test_type + '/' + i)
- for j in dir_ls2:
- result = re.match("^control", j)
- if result != None:
- z = str(i + "/" + j)
- k = z, "", 0
- u.append(k)
+ dir_ls = dircache.listdir(at_dir + '/' + test_type)
+ u = []
+ for i in dir_ls:
+ if i != ".svn":
+ dir_ls2 = dircache.listdir(at_dir + '/' + test_type + '/' + i)
+ for j in dir_ls2:
+ result = re.match("^control", j)
+ if result != None:
+ z = str(i + "/" + j)
+ k = z, "", 0
+ u.append(k)
- return u
+ return u
def atcc_control_file_read(tag, test_type, at_dir):
- file = (at_dir + '/' + test_type + '/' + tag)
- f = open(file, 'r')
+ file = (at_dir + '/' + test_type + '/' + tag)
+ f = open(file, 'r')
- z = f.readline()
- z = z.lstrip("job.run_test(")
- z = z.rstrip('\n')
- z = z.rstrip(')')
- z = z.split(',')
+ z = f.readline()
+ z = z.lstrip("job.run_test(")
+ z = z.rstrip('\n')
+ z = z.rstrip(')')
+ z = z.split(',')
- x = len(z)
+ x = len(z)
- if x == 1:
- z = ""
- elif x > 1:
- z = z[1:]
- m = ""
- for i in z:
- m += (',' + i)
+ if x == 1:
+ z = ""
+ elif x > 1:
+ z = z[1:]
+ m = ""
+ for i in z:
+ m += (',' + i)
- m = m.lstrip(',')
- m = m.strip()
- z = str(m)
+ m = m.lstrip(',')
+ m = m.strip()
+ z = str(m)
- f.close()
+ f.close()
- return z
+ return z
def atcc_setup_tmp_dirs_files(menu_dir):
- if not os.path.isdir(menu_dir + '/tmp/'):
- os.mkdir(menu_dir + '/tmp/')
- if os.path.isfile(menu_dir + '/tmp/Tests results'):
- os.remove(menu_dir + '/tmp/Tests results')
- if os.path.isfile(menu_dir + '/tmp/Possible kernel memory leaks'):
- os.remove(menu_dir + '/tmp/Possible kernel memory leaks')
+ if not os.path.isdir(menu_dir + '/tmp/'):
+ os.mkdir(menu_dir + '/tmp/')
+ if os.path.isfile(menu_dir + '/tmp/Tests results'):
+ os.remove(menu_dir + '/tmp/Tests results')
+ if os.path.isfile(menu_dir + '/tmp/Possible kernel memory leaks'):
+ os.remove(menu_dir + '/tmp/Possible kernel memory leaks')
def atcc_save_results1(i, at_dir, menu_dir):
- if i != "":
- if os.path.isfile(at_dir + '/results/default/' + i + '/debug/stderr'):
- os.system('cp ' + at_dir + '/results/default/' + i + '/debug/stderr ' + menu_dir + '/tmp/' + i + '.stderr')
- if os.path.isfile(at_dir + '/results/default/' + i + '/debug/stdout'):
- os.system('cp ' + at_dir + '/results/default/' + i + '/debug/stdout ' + menu_dir + '/tmp/' + i + '.stdout')
- if os.path.isfile(at_dir + '/results/default/status'):
- os.system('cat ' + at_dir + '/results/default/status >> ' + menu_dir + '/tmp/Tests\ results')
- if os.path.isfile('/sys/kernel/debug/memleak'):
- print "Saving possible kernel memory leaks"
- os.system('echo "' + i + '" >> ' + menu_dir + '/tmp/Possible kernel memory leaks')
- os.system('cat /sys/kernel/debug/memleak >> ' + menu_dir + '/tmp/Possible kernel memory leaks')
+ if i != "":
+ if os.path.isfile(at_dir + '/results/default/' + i + '/debug/stderr'):
+ os.system('cp ' + at_dir + '/results/default/' + i + '/debug/stderr ' + menu_dir + '/tmp/' + i + '.stderr')
+ if os.path.isfile(at_dir + '/results/default/' + i + '/debug/stdout'):
+ os.system('cp ' + at_dir + '/results/default/' + i + '/debug/stdout ' + menu_dir + '/tmp/' + i + '.stdout')
+ if os.path.isfile(at_dir + '/results/default/status'):
+ os.system('cat ' + at_dir + '/results/default/status >> ' + menu_dir + '/tmp/Tests\ results')
+ if os.path.isfile('/sys/kernel/debug/memleak'):
+ print "Saving possible kernel memory leaks"
+ os.system('echo "' + i + '" >> ' + menu_dir + '/tmp/Possible kernel memory leaks')
+ os.system('cat /sys/kernel/debug/memleak >> ' + menu_dir + '/tmp/Possible kernel memory leaks')
def atcc_save_profilers_results(i, j, at_dir, menu_dir):
- if os.path.isfile(at_dir + '/results/default/' + j + '.' + i + '/profiling/monitor'):
- os.system('cp ' + at_dir + '/results/default/' + j + '.' + i + '/profiling/monitor ' + menu_dir + '/tmp/' + j + '.monitor')
- if os.path.isfile(at_dir + '/results/default/' + j + '.' + i + '/profiling/oprofile.kernel'):
- os.system('cp ' + at_dir + '/results/default/' + j + '.' + i + '/profiling/oprofile.kernel ' + menu_dir + '/tmp/' + j + '.oprofile.kernel')
- if os.path.isfile(at_dir + '/results/default/' + j + '.' + i + '/profiling/oprofile.user'):
- os.system('cp ' + at_dir + '/results/default/' + j + '.' + i + '/profiling/oprofile.user ' + menu_dir + '/tmp/' + j + '.oprofile.user')
+ if os.path.isfile(at_dir + '/results/default/' + j + '.' + i + '/profiling/monitor'):
+ os.system('cp ' + at_dir + '/results/default/' + j + '.' + i + '/profiling/monitor ' + menu_dir + '/tmp/' + j + '.monitor')
+ if os.path.isfile(at_dir + '/results/default/' + j + '.' + i + '/profiling/oprofile.kernel'):
+ os.system('cp ' + at_dir + '/results/default/' + j + '.' + i + '/profiling/oprofile.kernel ' + menu_dir + '/tmp/' + j + '.oprofile.kernel')
+ if os.path.isfile(at_dir + '/results/default/' + j + '.' + i + '/profiling/oprofile.user'):
+ os.system('cp ' + at_dir + '/results/default/' + j + '.' + i + '/profiling/oprofile.user ' + menu_dir + '/tmp/' + j + '.oprofile.user')
def atcc_save_results2(res1, res2, at_dir, menu_dir):
- if os.path.isfile(at_dir + '/results/default/status'):
- os.system('cat ' + at_dir + '/results/default/status >> ' + menu_dir + '/tmp/Tests\ results')
+ if os.path.isfile(at_dir + '/results/default/status'):
+ os.system('cat ' + at_dir + '/results/default/status >> ' + menu_dir + '/tmp/Tests\ results')
- for i in res1:
- for j in res2:
- atcc_save_profilers_results(i, j, at_dir, menu_dir)
+ for i in res1:
+ for j in res2:
+ atcc_save_profilers_results(i, j, at_dir, menu_dir)
diff --git a/unittest_suite.py b/unittest_suite.py
index 4aab623..b8949b4 100644
--- a/unittest_suite.py
+++ b/unittest_suite.py
@@ -11,18 +11,18 @@
suites = []
def lister(dummy, dirname, files):
- loader = unittest.TestLoader()
- for f in files:
- if f.endswith('_unittest.py'):
- temp = os.path.join(dirname, f).strip('.py')
- mod = ('autotest_lib'
- + temp[len(root):].replace('/', '.'))
- suite = loader.loadTestsFromName(mod)
- suites.append(suite)
+ loader = unittest.TestLoader()
+ for f in files:
+ if f.endswith('_unittest.py'):
+ temp = os.path.join(dirname, f).strip('.py')
+ mod = ('autotest_lib'
+ + temp[len(root):].replace('/', '.'))
+ suite = loader.loadTestsFromName(mod)
+ suites.append(suite)
if __name__ == "__main__":
- os.path.walk(root, lister, None)
- alltests = unittest.TestSuite(suites)
- runner = unittest.TextTestRunner(verbosity=2)
- runner.run(alltests)
+ os.path.walk(root, lister, None)
+ alltests = unittest.TestSuite(suites)
+ runner = unittest.TextTestRunner(verbosity=2)
+ runner.run(alltests)