Merge remote branch 'cros/upstream' into tempbranch2
Merged to trunk@4816.
BUG=
TEST=we will build a new autotest server instance, and keep cautotest running and then later do a cname switch.
Review URL: http://codereview.chromium.org/3511003
Change-Id: Iee5f52f45f28f84927d6c6f9a74edc370d40288a
diff --git a/server/autotest.py b/server/autotest.py
index f918b27..eb25095 100644
--- a/server/autotest.py
+++ b/server/autotest.py
@@ -213,11 +213,10 @@
try:
self._install_using_packaging(host, autodir)
return
- except global_config.ConfigError, e:
+ except (error.PackageInstallError, error.AutoservRunError,
+ global_config.ConfigError), e:
logging.info("Could not install autotest using the packaging "
- "system: %s", e)
- except (error.PackageInstallError, error.AutoservRunError), e:
- logging.error("Could not install autotest from repos")
+ "system: %s. Trying other methods", e)
# try to install from file or directory
if self.source_material:
diff --git a/server/git.py b/server/git.py
index 93833ec..0428b80 100644
--- a/server/git.py
+++ b/server/git.py
@@ -1,74 +1,60 @@
-#
-# Copyright 2007 IBM Corp. Released under the GPL v2
-# Authors: Ryan Harper <ryanh@us.ibm.com>
-#
-
"""
This module defines a class for handling building from git repos
+
+@author: Ryan Harper (ryanh@us.ibm.com)
+@copyright: IBM 2007
"""
-__author__ = """
-ryanh@us.ibm.com (Ryan Harper)
-"""
-
-
-import os
-from autotest_lib.client.common_lib import error
+import os, warnings, logging
+from autotest_lib.client.common_lib import error, revision_control
+from autotest_lib.client.bin import os_dep
from autotest_lib.server import utils, installable_object
-class GitRepo(installable_object.InstallableObject):
+class InstallableGitRepo(installable_object.InstallableObject):
"""
- This class represents a git repo.
-
- It is used to pull down a local copy of a git repo, check if the local
- repo is up-to-date, if not update. It delegates the install to
- implementation classes.
-
+ This class helps to pick a git repo and install it in a host.
"""
-
- def __init__(self, repodir, giturl, weburl):
- super(installable_object.InstallableObject, self).__init__()
- if repodir is None:
- e_msg = 'You must provide a directory to hold the git repository'
- raise ValueError(e_msg)
- self.repodir = utils.sh_escape(repodir)
- if giturl is None:
- raise ValueError('You must provide a git URL to the repository')
+ def __init__(self, repodir, giturl, weburl=None):
+ self.repodir = repodir
self.giturl = giturl
- if weburl is None:
- raise ValueError('You must provide a http URL to the repository')
self.weburl = weburl
-
- # path to .git dir
- self.gitpath = utils.sh_escape(os.path.join(self.repodir,'.git'))
-
- # base git command , pointing to gitpath git dir
- self.gitcmdbase = 'git --git-dir=%s' % self.gitpath
-
+ self.git_repo = revision_control.GitRepo(self.repodir, self.giturl,
+ self.weburl)
# default to same remote path as local
- self.__build = os.path.dirname(self.repodir)
-
-
- def run(self, command, timeout=None, ignore_status=False):
- return utils.run(r'%s' % (utils.sh_escape(command)),
- timeout, ignore_status)
+ self._build = os.path.dirname(self.repodir)
# base install method
def install(self, host, builddir=None):
+ """
+ Install a git repo in a host. It works by pushing the downloaded source
+ code to the host.
+
+ @param host: Host object.
+ @param builddir: Directory on the host filesystem that will host the
+ source code.
+ """
# allow override of target remote dir
if builddir:
- self.__build = builddir
+ self._build = builddir
# push source to host for install
- print 'pushing %s to host:%s' %(self.source_material, self.__build)
- host.send_file(self.source_material, self.__build)
+ logging.info('Pushing code dir %s to host %s', self.source_material,
+ self._build)
+ host.send_file(self.source_material, self._build)
def gitcmd(self, cmd, ignore_status=False):
- return self.run('%s %s'%(self.gitcmdbase, cmd),
- ignore_status=ignore_status)
+ """
+ Wrapper for a git command.
+
+ @param cmd: Git subcommand (ex 'clone').
+ @param ignore_status: Whether we should supress error.CmdError
+ exceptions if the command did return exit code !=0 (True), or
+ not supress them (False).
+ """
+ return self.git_repo.gitcmd(cmd, ignore_status)
def get(self, **kwargs):
@@ -78,108 +64,75 @@
this method will leave an up-to-date version of git repo at
'giturl' in 'repodir' directory to be used by build/install
methods.
+
+ @param **kwargs: Dictionary of parameters to the method get.
"""
-
- if not self.is_repo_initialized():
- # this is your first time ...
- print 'cloning repo...'
- cmd = 'clone %s %s ' %(self.giturl, self.repodir)
- rv = self.gitcmd(cmd, True)
- if rv.exit_status != 0:
- print rv.stderr
- raise error.CmdError('Failed to clone git url', rv)
- else:
- print rv.stdout
-
- else:
- # exiting repo, check if we're up-to-date
- if self.is_out_of_date():
- print 'updating repo...'
- rv = self.gitcmd('pull', True)
- if rv.exit_status != 0:
- print rv.stderr
- e_msg = 'Failed to pull git repo data'
- raise error.CmdError(e_msg, rv)
- else:
- print 'repo up-to-date'
-
-
- # remember where the source is
self.source_material = self.repodir
+ return self.git_repo.get(**kwargs)
def get_local_head(self):
- cmd = 'log --max-count=1'
- gitlog = self.gitcmd(cmd).stdout
+ """
+ Get the top commit hash of the current local git branch.
- # parsing the commit checksum out of git log 's first entry.
- # Output looks like:
- #
- # commit 1dccba29b4e5bf99fb98c324f952386dda5b097f
- # Merge: 031b69b... df6af41...
- # Author: Avi Kivity <avi@qumranet.com>
- # Date: Tue Oct 23 10:36:11 2007 +0200
- #
- # Merge home:/home/avi/kvm/linux-2.6
- return str(gitlog.split('\n')[0]).split()[1]
+ @return: Top commit hash of local git branch
+ """
+ return self.git_repo.get_local_head()
def get_remote_head(self):
- def __needs_refresh(lines):
- tag = '<meta http-equiv="refresh" content="0"/>'
- if len(filter(lambda x: x.startswith(tag), lines)) > 0:
- return True
+ """
+ Get the top commit hash of the current remote git branch.
- return False
-
-
- # scan git web interface for revision HEAD's commit tag
- gitwebaction=';a=commit;h=HEAD'
- url = self.weburl+gitwebaction
- max_refresh = 4
- r = 0
-
- print 'checking %s for changes' %(url)
- u = utils.urlopen(url)
- lines = u.read().split('\n')
-
- while __needs_refresh(lines) and r < max_refresh:
- print 'refreshing url'
- r = r+1
- u = utils.urlopen(url)
- lines = u.read().split('\n')
-
- if r >= max_refresh:
- e_msg = 'Failed to get remote repo status, refreshed %s times' % r
- raise IndexError(e_msg)
-
- # looking for a line like:
- # <tr><td>commit</td><td # class="sha1">aadea67210c8b9e7a57744a1c2845501d2cdbac7</td></tr>
- commit_filter = lambda x: x.startswith('<tr><td>commit</td>')
- commit_line = filter(commit_filter, lines)
-
- # extract the sha1 sum from the commit line
- return str(commit_line).split('>')[4].split('<')[0]
+ @return: Top commit hash of remote git branch
+ """
+ return self.git_repo.get_remote_head()
def is_out_of_date(self):
- local_head = self.get_local_head()
- remote_head = self.get_remote_head()
+ """
+ Return whether this branch is out of date with regards to remote branch.
- # local is out-of-date, pull
- if local_head != remote_head:
- return True
-
- return False
+ @return: False, if the branch is outdated, True if it is current.
+ """
+ return self.git_repo.is_out_of_date()
def is_repo_initialized(self):
- # if we fail to get a rv of 0 out of the git log command
- # then the repo is bogus
+ """
+ Return whether the git repo was already initialized (has a top commit).
- cmd = 'log --max-count=1'
- rv = self.gitcmd(cmd, True)
- if rv.exit_status == 0:
- return True
+ @return: False, if the repo was initialized, True if it was not.
+ """
+ return self.git_repo.is_repo_initialized()
- return False
+
+ def get_revision(self):
+ """
+ Return current HEAD commit id
+ """
+ return self.git_repo.get_revision()
+
+
+ def checkout(self, remote, local=None):
+ """
+ Check out the git commit id, branch, or tag given by remote.
+
+ Optional give the local branch name as local.
+
+ @param remote: Remote commit hash
+ @param local: Local commit hash
+ @note: For git checkout tag git version >= 1.5.0 is required
+ """
+ return self.git_repo.checkout(remote, local)
+
+
+ def get_branch(self, all=False, remote_tracking=False):
+ """
+ Show the branches.
+
+ @param all: List both remote-tracking branches and local branches (True)
+ or only the local ones (False).
+ @param remote_tracking: Lists the remote-tracking branches.
+ """
+ return self.git_repo.get_branch(all, remote_tracking)
diff --git a/server/git_kernel.py b/server/git_kernel.py
index 1f4273c..9bd7ae6 100644
--- a/server/git_kernel.py
+++ b/server/git_kernel.py
@@ -1,69 +1,116 @@
-#
-# Copyright 2007 IBM Corp. Released under the GPL v2
-# Authors: Ryan Harper <ryanh@us.ibm.com>
-#
-
"""
This module defines the GitKernel class
+
+@author: Ryan Harper (ryanh@us.ibm.com)
+@copyright: IBM 2007
"""
-__author__ = """
-ryanh@us.ibm.com (Ryan Harper)
-"""
-
-
-import os
+import os, logging
import git, source_kernel
-class GitKernel(git.GitRepo):
+class GitKernel(git.InstallableGitRepo):
"""
- This class represents a git kernel repo.
+ This class represents an installable git kernel repo.
It is used to pull down a local copy of a git repo, check if the local repo
is up-to-date, if not update and then build the kernel from the git repo.
-
"""
def __init__(self, repodir, giturl, weburl):
- git.GitRepo.__init__(self, repodir, giturl, weburl)
- self.__patches = []
- self.__config = None
- self.__build = None
+ super(GitKernel, self).__init__(repodir, giturl, weburl)
+ self._patches = []
+ self._config = None
+ self._build = None
+ self._branch = None
+ self._revision = None
def configure(self, config):
- self.__config = config
+ self._config = config
def patch(self, patch):
- self.__patches.append(patch)
+ self._patches.append(patch)
- def install(self, host, build=True, builddir=None):
- # use tmpdir if no builddir specified
- # NB: pass a builddir to install() method if you
- # need to ensure the build remains after the completion
- # of a job
+ def checkout(self, revision, local=None):
+ """
+ Checkout the commit id, branch, or tag.
+
+ @param revision: Name of the git remote branch, revision or tag
+ @param local: Name of the local branch, implies -b
+ """
+ logging.info('Checking out %s', revision)
+ super(GitKernel, self).checkout(revision)
+ self._revision = super(GitKernel, self).get_revision()
+ self._branch = super(GitKernel, self).get_branch()
+ logging.info('Checked out %s on branch %s', self._revision,
+ self._branch)
+
+
+ def show_branch(self):
+ """
+ Print the current local branch name.
+ """
+ self._branch = super(GitKernel, self).get_branch()
+ logging.info(self._branch)
+
+
+ def show_branches(self, all=True):
+ """
+ Print the local and remote branches.
+
+ @param all: Whether to show all branches (True) or only local branches
+ (False).
+ """
+ self._branch = super(GitKernel, self).get_branch()
+ logging.info(super(GitKernel, self).get_branch(all=all))
+
+
+ def show_revision(self):
+ """
+ Show the current git revision.
+ """
+ self._revision = super(GitKernel, self).get_revision()
+ logging.info(self._revision)
+
+
+ def install(self, host, build=True, builddir=None, revision=None):
+ """
+ Install the git tree in a host.
+
+ @param host: Host object
+ @param build: Whether to build the source tree
+ @param builddir: Specify a build dir. If no build dir is specified,
+ the job temporary directory will be used, so the build won't
+ be persistent.
+ @param revision: Desired commit hash. If ommited, will build from HEAD
+ of the branch.
+ """
+ if revision:
+ self.checkout(revision)
+ self._revision = super(GitKernel, self).get_revision()
+ logging.info('Checked out revision: %s', self._revision)
+
if not builddir:
- self.__build = os.path.join(host.get_tmp_dir(),"build")
- print 'warning: builddir %s is not persistent' %(self.__build)
+ self._build = os.path.join(host.get_tmp_dir(), "build")
+ logging.warning('Builddir %s is not persistent (it will be erased '
+ 'in future jobs)', self._build)
# push source to host for install
- print 'pushing %s to host' %(self.source_material)
- host.send_file(self.source_material, self.__build)
- remote_source_material= os.path.join(self.__build,
- os.path.basename(self.source_material))
+ logging.info('Pushing %s to host', self.source_material)
+ host.send_file(self.source_material, self._build)
# use a source_kernel to configure, patch, build and install.
- sk = source_kernel.SourceKernel(remote_source_material)
+ sk = source_kernel.SourceKernel(self._build)
if build:
# apply patches
- for p in self.__patches:
+ for p in self._patches:
sk.patch(p)
# configure
- sk.configure(self.__config)
+ sk.configure(self._config)
# build
sk.build(host)
diff --git a/server/hosts/remote.py b/server/hosts/remote.py
index b692480..e46bc1b 100644
--- a/server/hosts/remote.py
+++ b/server/hosts/remote.py
@@ -1,7 +1,7 @@
"""This class defines the Remote host class, mixing in the SiteHost class
if it is available."""
-import os, logging
+import os, logging, urllib
from autotest_lib.client.common_lib import error
from autotest_lib.server import utils
from autotest_lib.server.hosts import base_classes, bootloader
@@ -25,6 +25,7 @@
DEFAULT_REBOOT_TIMEOUT = base_classes.Host.DEFAULT_REBOOT_TIMEOUT
LAST_BOOT_TAG = object()
+ DEFAULT_HALT_TIMEOUT = 2 * 60
VAR_LOG_MESSAGES_COPY_PATH = "/var/log/messages.autotest_start"
@@ -88,6 +89,12 @@
self.run('echo b > /proc/sysrq-trigger &')
+ def halt(self, timeout=DEFAULT_HALT_TIMEOUT, wait=True):
+ self.run('/sbin/halt')
+ if wait:
+ self.wait_down(timeout=timeout)
+
+
def reboot(self, timeout=DEFAULT_REBOOT_TIMEOUT, label=LAST_BOOT_TAG,
kernel_args=None, wait=True, fastsync=False,
reboot_cmd=None, **dargs):
@@ -213,6 +220,21 @@
return None
+ def get_all_labels(self):
+ """
+ Return all labels, or empty list if label is not set.
+ """
+ if self.job:
+ keyval_path = os.path.join(self.job.resultdir, 'host_keyvals',
+ self.hostname)
+ keyvals = utils.read_keyval(keyval_path)
+ all_labels = keyvals.get('labels', '')
+ if all_labels:
+ all_labels = all_labels.split(',')
+ return [urllib.unquote(label) for label in all_labels]
+ return []
+
+
def delete_tmp_dir(self, tmpdir):
"""
Delete the given temporary directory on the remote machine.
diff --git a/server/hosts/serial.py b/server/hosts/serial.py
index 9b4bdb2..d514dba 100644
--- a/server/hosts/serial.py
+++ b/server/hosts/serial.py
@@ -114,7 +114,7 @@
def hardreset(self, timeout=DEFAULT_REBOOT_TIMEOUT, wait=True,
- conmux_command='hardreset', num_attempts=1,
+ conmux_command='hardreset', num_attempts=1, halt=False,
**wait_for_restart_kwargs):
"""
Reach out and slap the box in the power switch.
@@ -124,6 +124,7 @@
@params wait: Whether or not to wait for the machine to reboot
@params num_attempts: Number of times to attempt hard reset erroring
on the last attempt.
+ @params halt: Halts the machine before hardresetting.
@params **wait_for_restart_kwargs: keyword arguments passed to
wait_for_restart()
"""
@@ -138,6 +139,8 @@
old_boot_id = 'unknown boot_id prior to SerialHost.hardreset'
def reboot():
+ if halt:
+ self.halt()
if not self.run_conmux(conmux_command):
self.record("ABORT", None, "reboot.start",
"hard reset unavailable")
diff --git a/server/server_job.py b/server/server_job.py
index 757257f..6ef59de 100644
--- a/server/server_job.py
+++ b/server/server_job.py
@@ -542,10 +542,12 @@
# no error occured, so we don't need to collect crashinfo
collect_crashinfo = False
- except:
+ except Exception, e:
try:
logging.exception(
'Exception escaped control file, job aborting:')
+ self.record('INFO', None, None, str(e),
+ {'job_abort_reason': str(e)})
except:
pass # don't let logging exceptions here interfere
raise