Allow use of local packages for developers to use when running tests.
The culprit behind the checksum logic not being run is client_logger in
autotest. I've changed client_logger such that it can be Site'ized
and added the necessary implementation in site_autotest to get this
working correctly for us.
The key issues were
1) client_logger ignored all requests for checksum files.
2) client_logger assumed you always wanted to re-tar code through
the autotest workflow.
In addition, we should exclude packages from being wiped on every
installed hence the additional excluded dirs.
BUG=chromium-os:26561
TEST=Ran it, saved 30 seconds of 70 second tests. Did not re-install
unnecessarily.
Change-Id: I7ac0a878f19296472f3b79e8cbe4cee641177215
Reviewed-on: https://gerrit.chromium.org/gerrit/17572
Reviewed-by: Chris Sosa <sosa@chromium.org>
Tested-by: Chris Sosa <sosa@chromium.org>
Commit-Ready: Chris Sosa <sosa@chromium.org>
diff --git a/PRESUBMIT.cfg b/PRESUBMIT.cfg
new file mode 100644
index 0000000..64da5e6
--- /dev/null
+++ b/PRESUBMIT.cfg
@@ -0,0 +1,5 @@
+# This config file disables some of the ChromiumOS source style checks.
+# Comment out the disable-flags for any checks you want to leave enabled.
+
+[Hook Overrides]
+cros_license_check: false
diff --git a/server/autotest.py b/server/autotest.py
index 672bc50..186c0e7 100644
--- a/server/autotest.py
+++ b/server/autotest.py
@@ -158,7 +158,8 @@
def _install_using_send_file(self, host, autodir):
- dirs_to_exclude = set(["tests", "site_tests", "deps", "profilers"])
+ dirs_to_exclude = set(["tests", "site_tests", "deps", "profilers",
+ "packages"])
light_files = [os.path.join(self.source_material, f)
for f in os.listdir(self.source_material)
if f not in dirs_to_exclude]
@@ -837,7 +838,7 @@
# a file-like object for catching stderr from an autotest client and
# extracting status logs from it
-class client_logger(object):
+class BaseClientLogger(object):
"""Partial file object to write to both stdout and
the status log file. We only implement those methods
utils.run() actually calls.
@@ -1049,10 +1050,19 @@
__file__, "autotest_lib.server.site_autotest", "_SiteRun", _BaseRun)
+SiteClientLogger = client_utils.import_site_class(
+ __file__, "autotest_lib.server.site_autotest", "SiteClientLogger",
+ BaseClientLogger)
+
+
class Autotest(SiteAutotest):
pass
+class client_logger(SiteClientLogger):
+ pass
+
+
class _Run(_SiteRun):
pass
diff --git a/server/site_autotest.py b/server/site_autotest.py
index fed9b95..0d1d28e 100755
--- a/server/site_autotest.py
+++ b/server/site_autotest.py
@@ -4,7 +4,7 @@
import logging
import os
-from autotest_lib.client.common_lib import global_config
+from autotest_lib.client.common_lib import error, global_config
from autotest_lib.server import installable_object, autoserv_parser
@@ -14,7 +14,7 @@
class SiteAutotest(installable_object.InstallableObject):
- def get(self, location = None):
+ def get(self, location=None):
if not location:
location = os.path.join(self.serverdir, '../client')
location = os.path.abspath(location)
@@ -88,5 +88,63 @@
super(SiteAutotest, self).install(host=host, autodir=autodir)
+class SiteClientLogger(object):
+ """Overrides default client logger to allow for using a local package cache.
+ """
+
+ def _process_line(self, line):
+ """Returns the package checksum file if it exists."""
+ logging.debug(line)
+ fetch_package_match = self.fetch_package_parser.search(line)
+ if fetch_package_match:
+ pkg_name, dest_path, fifo_path = fetch_package_match.groups()
+ serve_packages = global_config.global_config.get_config_value(
+ "PACKAGES", "serve_packages_from_autoserv", type=bool)
+ if serve_packages and pkg_name == 'packages.checksum':
+ package_served = False
+ try:
+ checksum_file = os.path.join(
+ self.job.pkgmgr.pkgmgr_dir, 'packages', pkg_name)
+ if os.path.exists(checksum_file):
+ self.host.send_file(checksum_file, dest_path)
+ package_served = True
+ except error.AutoservRunError:
+ msg = "Package checksum file not found, continuing anyway"
+ logging.exception(msg)
+
+ if package_served:
+ try:
+ # When fetching a package, the client expects to be
+ # notified when the fetching is complete. Autotest
+ # does this pushing a B to a fifo queue to the client.
+ self.host.run("echo B > %s" % fifo_path)
+ except error.AutoservRunError:
+ msg = "Checksum installation failed, continuing anyway"
+ logging.exception(msg)
+ finally:
+ return
+
+ # Fall through to process the line using the default method.
+ super(SiteClientLogger, self)._process_line(line)
+
+
+ def _send_tarball(self, pkg_name, remote_dest):
+ """Uses tarballs in package manager by default."""
+ try:
+ server_package = os.path.join(self.job.pkgmgr.pkgmgr_dir,
+ 'packages', pkg_name)
+ if os.path.exists(server_package):
+ self.host.send_file(server_package, remote_dest)
+ return
+
+ except error.AutoservRunError:
+ msg = ("Package %s could not be sent from the package cache." %
+ pkg_name)
+ logging.exception(msg)
+
+ # Fall through to send tarball the default method.
+ super(SiteClientLogger, self)._send_tarball(pkg_name, remote_dest)
+
+
class _SiteRun(object):
pass