mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 1 | # Copyright 2007 Google Inc. Released under the GPL v2 |
Dan Shi | b669cbd | 2013-09-13 11:17:17 -0700 | [diff] [blame] | 2 | #pylint: disable-msg=C0111 |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 3 | |
Dan Shi | 9f879fb | 2017-05-26 15:44:04 -0700 | [diff] [blame] | 4 | import glob |
beeps | 07f53b9 | 2013-01-08 12:55:10 -0800 | [diff] [blame] | 5 | import logging |
Dan Shi | 9f879fb | 2017-05-26 15:44:04 -0700 | [diff] [blame] | 6 | import os |
| 7 | import re |
| 8 | import sys |
| 9 | import tempfile |
| 10 | import time |
| 11 | import traceback |
Prathmesh Prabhu | 7cc1153 | 2016-11-23 17:51:08 -0800 | [diff] [blame] | 12 | |
Prathmesh Prabhu | 7cc1153 | 2016-11-23 17:51:08 -0800 | [diff] [blame] | 13 | import common |
Dan Shi | 010c0bc | 2017-06-21 17:02:51 -0700 | [diff] [blame] | 14 | from autotest_lib.client.bin.result_tools import runner as result_tools_runner |
| 15 | from autotest_lib.client.common_lib import autotemp |
| 16 | from autotest_lib.client.common_lib import base_job |
| 17 | from autotest_lib.client.common_lib import error |
Dan Shi | 1889ca1 | 2015-04-15 09:36:06 -0700 | [diff] [blame] | 18 | from autotest_lib.client.common_lib import global_config |
Allen Li | 944ac46 | 2017-02-07 15:57:20 -0800 | [diff] [blame] | 19 | from autotest_lib.client.common_lib import packages |
Allen Li | 7962369 | 2017-08-17 16:27:11 -0700 | [diff] [blame] | 20 | from autotest_lib.client.common_lib import utils as client_utils |
Dan Shi | 9f879fb | 2017-05-26 15:44:04 -0700 | [diff] [blame] | 21 | from autotest_lib.server import installable_object |
Dan Shi | 9f879fb | 2017-05-26 15:44:04 -0700 | [diff] [blame] | 22 | from autotest_lib.server import utils |
Allen Li | d5abdab | 2017-02-07 16:03:43 -0800 | [diff] [blame] | 23 | from autotest_lib.server import utils as server_utils |
Allen Li | d5abdab | 2017-02-07 16:03:43 -0800 | [diff] [blame] | 24 | from autotest_lib.server.cros.dynamic_suite.constants import JOB_REPO_URL |
| 25 | |
mbligh | 3c7a150 | 2008-07-24 18:08:47 +0000 | [diff] [blame] | 26 | |
Dan Shi | 5e2efb7 | 2017-02-07 11:40:23 -0800 | [diff] [blame] | 27 | try: |
| 28 | from chromite.lib import metrics |
| 29 | except ImportError: |
| 30 | metrics = client_utils.metrics_mock |
| 31 | |
| 32 | |
Dale Curtis | cb7bfaf | 2011-06-07 16:21:57 -0700 | [diff] [blame] | 33 | AUTOTEST_SVN = 'svn://test.kernel.org/autotest/trunk/client' |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 34 | AUTOTEST_HTTP = 'http://test.kernel.org/svn/autotest/trunk/client' |
| 35 | |
Allen Li | 944ac46 | 2017-02-07 15:57:20 -0800 | [diff] [blame] | 36 | _CONFIG = global_config.global_config |
| 37 | AUTOSERV_PREBUILD = _CONFIG.get_config_value( |
Dan Shi | c958a3d | 2017-07-06 14:43:18 -0700 | [diff] [blame] | 38 | 'AUTOSERV', 'enable_server_prebuild', type=bool, default=False) |
jadmanski | 2a89dac | 2010-06-11 14:32:58 +0000 | [diff] [blame] | 39 | |
Kevin Cheng | c51cc1f | 2017-11-17 15:49:55 -0800 | [diff] [blame] | 40 | # Match on a line like this: |
| 41 | # FAIL test_name test_name timestamp=1 localtime=Nov 15 12:43:10 <fail_msg> |
| 42 | _FAIL_STATUS_RE = re.compile( |
Kevin Cheng | 5684c51 | 2017-12-01 11:12:32 -0800 | [diff] [blame] | 43 | r'\s*FAIL.*localtime=.*\s*.*\s*[0-9]+:[0-9]+:[0-9]+\s*(?P<fail_msg>.*)') |
Kevin Cheng | c51cc1f | 2017-11-17 15:49:55 -0800 | [diff] [blame] | 44 | |
| 45 | |
showard | ad812bf | 2009-10-20 23:49:56 +0000 | [diff] [blame] | 46 | class AutodirNotFoundError(Exception): |
| 47 | """No Autotest installation could be found.""" |
| 48 | |
| 49 | |
beeps | 607428b | 2013-03-25 16:43:20 -0700 | [diff] [blame] | 50 | class AutotestFailure(Exception): |
| 51 | """Gereric exception class for failures during a test run.""" |
| 52 | |
| 53 | |
| 54 | class AutotestAbort(AutotestFailure): |
| 55 | """ |
| 56 | AutotestAborts are thrown when the DUT seems fine, |
| 57 | and the test doesn't give us an explicit reason for |
| 58 | failure; In this case we have no choice but to abort. |
| 59 | """ |
| 60 | |
| 61 | |
| 62 | class AutotestDeviceError(AutotestFailure): |
| 63 | """ |
| 64 | Exceptions that inherit from AutotestDeviceError |
| 65 | are thrown when we can determine the current |
| 66 | state of the DUT and conclude that it probably |
| 67 | lead to the test failing; these exceptions lead |
| 68 | to failures instead of aborts. |
| 69 | """ |
| 70 | |
| 71 | |
| 72 | class AutotestDeviceNotPingable(AutotestDeviceError): |
| 73 | """Error for when a DUT becomes unpingable.""" |
| 74 | |
| 75 | |
| 76 | class AutotestDeviceNotSSHable(AutotestDeviceError): |
| 77 | """Error for when a DUT is pingable but not SSHable.""" |
| 78 | |
| 79 | |
| 80 | class AutotestDeviceRebooted(AutotestDeviceError): |
| 81 | """Error for when a DUT rebooted unexpectedly.""" |
| 82 | |
| 83 | |
Allen Li | d5abdab | 2017-02-07 16:03:43 -0800 | [diff] [blame] | 84 | class Autotest(installable_object.InstallableObject): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 85 | """ |
| 86 | This class represents the Autotest program. |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 87 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 88 | Autotest is used to run tests automatically and collect the results. |
| 89 | It also supports profilers. |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 90 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 91 | Implementation details: |
| 92 | This is a leaf class in an abstract class hierarchy, it must |
| 93 | implement the unimplemented methods in parent classes. |
| 94 | """ |
mbligh | 119c12a | 2007-11-12 22:13:44 +0000 | [diff] [blame] | 95 | |
Dale Curtis | cb7bfaf | 2011-06-07 16:21:57 -0700 | [diff] [blame] | 96 | def __init__(self, host=None): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 97 | self.host = host |
| 98 | self.got = False |
| 99 | self.installed = False |
| 100 | self.serverdir = utils.get_server_dir() |
Allen Li | d5abdab | 2017-02-07 16:03:43 -0800 | [diff] [blame] | 101 | super(Autotest, self).__init__() |
mbligh | c8949b8 | 2007-07-23 16:33:58 +0000 | [diff] [blame] | 102 | |
mbligh | dc735a2 | 2007-08-02 16:54:37 +0000 | [diff] [blame] | 103 | |
jadmanski | f22fea8 | 2008-11-26 20:57:07 +0000 | [diff] [blame] | 104 | install_in_tmpdir = False |
| 105 | @classmethod |
| 106 | def set_install_in_tmpdir(cls, flag): |
| 107 | """ Sets a flag that controls whether or not Autotest should by |
| 108 | default be installed in a "standard" directory (e.g. |
| 109 | /home/autotest, /usr/local/autotest) or a temporary directory. """ |
| 110 | cls.install_in_tmpdir = flag |
| 111 | |
| 112 | |
showard | ad812bf | 2009-10-20 23:49:56 +0000 | [diff] [blame] | 113 | @classmethod |
| 114 | def get_client_autodir_paths(cls, host): |
| 115 | return global_config.global_config.get_config_value( |
| 116 | 'AUTOSERV', 'client_autodir_paths', type=list) |
| 117 | |
| 118 | |
| 119 | @classmethod |
| 120 | def get_installed_autodir(cls, host): |
| 121 | """ |
| 122 | Find where the Autotest client is installed on the host. |
| 123 | @returns an absolute path to an installed Autotest client root. |
| 124 | @raises AutodirNotFoundError if no Autotest installation can be found. |
| 125 | """ |
| 126 | autodir = host.get_autodir() |
| 127 | if autodir: |
| 128 | logging.debug('Using existing host autodir: %s', autodir) |
| 129 | return autodir |
| 130 | |
| 131 | for path in Autotest.get_client_autodir_paths(host): |
| 132 | try: |
| 133 | autotest_binary = os.path.join(path, 'bin', 'autotest') |
Allen Li | ad719c1 | 2017-06-27 23:48:04 +0000 | [diff] [blame] | 134 | host.run('test -x %s' % utils.sh_escape(autotest_binary)) |
| 135 | host.run('test -w %s' % utils.sh_escape(path)) |
showard | ad812bf | 2009-10-20 23:49:56 +0000 | [diff] [blame] | 136 | logging.debug('Found existing autodir at %s', path) |
| 137 | return path |
Justin Giorgi | a4a843d | 2017-03-07 13:09:48 -0800 | [diff] [blame] | 138 | except error.GenericHostRunError: |
showard | ad812bf | 2009-10-20 23:49:56 +0000 | [diff] [blame] | 139 | logging.debug('%s does not exist on %s', autotest_binary, |
| 140 | host.hostname) |
| 141 | raise AutodirNotFoundError |
| 142 | |
| 143 | |
| 144 | @classmethod |
| 145 | def get_install_dir(cls, host): |
| 146 | """ |
| 147 | Determines the location where autotest should be installed on |
jadmanski | f22fea8 | 2008-11-26 20:57:07 +0000 | [diff] [blame] | 148 | host. If self.install_in_tmpdir is set, it will return a unique |
showard | ad812bf | 2009-10-20 23:49:56 +0000 | [diff] [blame] | 149 | temporary directory that autotest can be installed in. Otherwise, looks |
| 150 | for an existing installation to use; if none is found, looks for a |
| 151 | usable directory in the global config client_autodir_paths. |
| 152 | """ |
jadmanski | f22fea8 | 2008-11-26 20:57:07 +0000 | [diff] [blame] | 153 | try: |
lmr | 9dcf083 | 2009-12-08 21:28:55 +0000 | [diff] [blame] | 154 | install_dir = cls.get_installed_autodir(host) |
showard | ad812bf | 2009-10-20 23:49:56 +0000 | [diff] [blame] | 155 | except AutodirNotFoundError: |
lmr | 9dcf083 | 2009-12-08 21:28:55 +0000 | [diff] [blame] | 156 | install_dir = cls._find_installable_dir(host) |
| 157 | |
| 158 | if cls.install_in_tmpdir: |
| 159 | return host.get_tmp_dir(parent=install_dir) |
| 160 | return install_dir |
showard | ad812bf | 2009-10-20 23:49:56 +0000 | [diff] [blame] | 161 | |
| 162 | |
| 163 | @classmethod |
| 164 | def _find_installable_dir(cls, host): |
| 165 | client_autodir_paths = cls.get_client_autodir_paths(host) |
| 166 | for path in client_autodir_paths: |
| 167 | try: |
Allen Li | ad719c1 | 2017-06-27 23:48:04 +0000 | [diff] [blame] | 168 | host.run('mkdir -p %s' % utils.sh_escape(path)) |
| 169 | host.run('test -w %s' % utils.sh_escape(path)) |
showard | ad812bf | 2009-10-20 23:49:56 +0000 | [diff] [blame] | 170 | return path |
| 171 | except error.AutoservRunError: |
| 172 | logging.debug('Failed to create %s', path) |
Prathmesh Prabhu | 7cc1153 | 2016-11-23 17:51:08 -0800 | [diff] [blame] | 173 | metrics.Counter( |
| 174 | 'chromeos/autotest/errors/no_autotest_install_path').increment( |
| 175 | fields={'dut_host_name': host.hostname}) |
showard | ad812bf | 2009-10-20 23:49:56 +0000 | [diff] [blame] | 176 | raise error.AutoservInstallError( |
Dale Curtis | 74a314b | 2011-06-23 14:55:46 -0700 | [diff] [blame] | 177 | 'Unable to find a place to install Autotest; tried %s' % |
showard | ad812bf | 2009-10-20 23:49:56 +0000 | [diff] [blame] | 178 | ', '.join(client_autodir_paths)) |
jadmanski | f22fea8 | 2008-11-26 20:57:07 +0000 | [diff] [blame] | 179 | |
| 180 | |
Eric Li | d656d56 | 2011-04-20 11:48:29 -0700 | [diff] [blame] | 181 | def get_fetch_location(self): |
Allen Li | d5abdab | 2017-02-07 16:03:43 -0800 | [diff] [blame] | 182 | """Generate list of locations where autotest can look for packages. |
| 183 | |
Richard Barnette | 71854c7 | 2018-03-30 14:22:09 -0700 | [diff] [blame] | 184 | Hosts are tagged with an attribute containing the URL from which |
| 185 | to source packages when running a test on that host. |
Allen Li | d5abdab | 2017-02-07 16:03:43 -0800 | [diff] [blame] | 186 | |
| 187 | @returns the list of candidate locations to check for packages. |
| 188 | """ |
Eric Li | d656d56 | 2011-04-20 11:48:29 -0700 | [diff] [blame] | 189 | c = global_config.global_config |
| 190 | repos = c.get_config_value("PACKAGES", 'fetch_location', type=list, |
| 191 | default=[]) |
| 192 | repos.reverse() |
Allen Li | d5abdab | 2017-02-07 16:03:43 -0800 | [diff] [blame] | 193 | |
Richard Barnette | 71854c7 | 2018-03-30 14:22:09 -0700 | [diff] [blame] | 194 | if not server_utils.is_inside_chroot(): |
| 195 | # Only try to get fetch location from host attribute if the |
| 196 | # test is not running inside chroot. |
| 197 | # |
| 198 | # Look for the repo url via the host attribute. If we are |
| 199 | # not running with a full AFE autoserv will fall back to |
| 200 | # serving packages itself from whatever source version it is |
| 201 | # sync'd to rather than using the proper artifacts for the |
| 202 | # build on the host. |
Allen Li | d5abdab | 2017-02-07 16:03:43 -0800 | [diff] [blame] | 203 | found_repo = self._get_fetch_location_from_host_attribute() |
| 204 | if found_repo is not None: |
| 205 | # Add our new repo to the end, the package manager will |
| 206 | # later reverse the list of repositories resulting in ours |
| 207 | # being first |
| 208 | repos.append(found_repo) |
| 209 | |
Eric Li | d656d56 | 2011-04-20 11:48:29 -0700 | [diff] [blame] | 210 | return repos |
| 211 | |
| 212 | |
Allen Li | d5abdab | 2017-02-07 16:03:43 -0800 | [diff] [blame] | 213 | def _get_fetch_location_from_host_attribute(self): |
| 214 | """Get repo to use for packages from host attribute, if possible. |
| 215 | |
| 216 | Hosts are tagged with an attribute containing the URL |
| 217 | from which to source packages when running a test on that host. |
| 218 | If self.host is set, attempt to look this attribute in the host info. |
| 219 | |
| 220 | @returns value of the 'job_repo_url' host attribute, if present. |
| 221 | """ |
| 222 | if not self.host: |
| 223 | return None |
| 224 | |
| 225 | try: |
| 226 | info = self.host.host_info_store.get() |
| 227 | except Exception as e: |
| 228 | # TODO(pprabhu): We really want to catch host_info.StoreError here, |
| 229 | # but we can't import host_info from this module. |
| 230 | # - autotest_lib.hosts.host_info pulls in (naturally) |
| 231 | # autotest_lib.hosts.__init__ |
| 232 | # - This pulls in all the host classes ever defined |
| 233 | # - That includes abstract_ssh, which depends on autotest |
| 234 | logging.warning('Failed to obtain host info: %r', e) |
| 235 | logging.warning('Skipping autotest fetch location based on %s', |
| 236 | JOB_REPO_URL) |
| 237 | return None |
| 238 | |
| 239 | job_repo_url = info.attributes.get(JOB_REPO_URL, '') |
| 240 | if not job_repo_url: |
| 241 | logging.warning("No %s for %s", JOB_REPO_URL, self.host) |
| 242 | return None |
| 243 | |
| 244 | logging.info('Got job repo url from host attributes: %s', |
| 245 | job_repo_url) |
| 246 | return job_repo_url |
| 247 | |
| 248 | |
Dan Shi | b669cbd | 2013-09-13 11:17:17 -0700 | [diff] [blame] | 249 | def install(self, host=None, autodir=None, use_packaging=True): |
Allen Li | d5abdab | 2017-02-07 16:03:43 -0800 | [diff] [blame] | 250 | """Install autotest. If |host| is not None, stores it in |self.host|. |
| 251 | |
| 252 | @param host A Host instance on which autotest will be installed |
| 253 | @param autodir Location on the remote host to install to |
| 254 | @param use_packaging Enable install modes that use the packaging system. |
| 255 | |
| 256 | """ |
| 257 | if host: |
| 258 | self.host = host |
Dan Shi | b669cbd | 2013-09-13 11:17:17 -0700 | [diff] [blame] | 259 | self._install(host=host, autodir=autodir, use_packaging=use_packaging) |
jadmanski | 54f90af | 2008-10-10 16:20:55 +0000 | [diff] [blame] | 260 | |
| 261 | |
mbligh | b8aa75b | 2009-09-18 16:50:37 +0000 | [diff] [blame] | 262 | def install_full_client(self, host=None, autodir=None): |
| 263 | self._install(host=host, autodir=autodir, use_autoserv=False, |
| 264 | use_packaging=False) |
| 265 | |
| 266 | |
mbligh | bccad48 | 2009-08-24 22:08:31 +0000 | [diff] [blame] | 267 | def install_no_autoserv(self, host=None, autodir=None): |
mbligh | b8aa75b | 2009-09-18 16:50:37 +0000 | [diff] [blame] | 268 | self._install(host=host, autodir=autodir, use_autoserv=False) |
mbligh | bccad48 | 2009-08-24 22:08:31 +0000 | [diff] [blame] | 269 | |
| 270 | |
mbligh | b8aa75b | 2009-09-18 16:50:37 +0000 | [diff] [blame] | 271 | def _install_using_packaging(self, host, autodir): |
Eric Li | d656d56 | 2011-04-20 11:48:29 -0700 | [diff] [blame] | 272 | repos = self.get_fetch_location() |
mbligh | b8aa75b | 2009-09-18 16:50:37 +0000 | [diff] [blame] | 273 | if not repos: |
| 274 | raise error.PackageInstallError("No repos to install an " |
| 275 | "autotest client from") |
Matt Mallett | 6da5d6b | 2018-09-27 12:57:47 -0700 | [diff] [blame] | 276 | # Make sure devserver has the autotest package staged |
| 277 | host.verify_job_repo_url() |
mbligh | b8aa75b | 2009-09-18 16:50:37 +0000 | [diff] [blame] | 278 | pkgmgr = packages.PackageManager(autodir, hostname=host.hostname, |
| 279 | repo_urls=repos, |
| 280 | do_locking=False, |
| 281 | run_function=host.run, |
| 282 | run_function_dargs=dict(timeout=600)) |
| 283 | # The packages dir is used to store all the packages that |
| 284 | # are fetched on that client. (for the tests,deps etc. |
| 285 | # too apart from the client) |
| 286 | pkg_dir = os.path.join(autodir, 'packages') |
Dan Shi | 86de094 | 2017-08-24 21:37:24 -0700 | [diff] [blame] | 287 | # clean up the autodir except for the packages and result_tools |
| 288 | # directory. |
| 289 | host.run('cd %s && ls | grep -v "^packages$" | grep -v "^result_tools$"' |
Allen Li | ad719c1 | 2017-06-27 23:48:04 +0000 | [diff] [blame] | 290 | ' | xargs rm -rf && rm -rf .[!.]*' % autodir) |
mbligh | b8aa75b | 2009-09-18 16:50:37 +0000 | [diff] [blame] | 291 | pkgmgr.install_pkg('autotest', 'client', pkg_dir, autodir, |
| 292 | preserve_install_dir=True) |
| 293 | self.installed = True |
| 294 | |
| 295 | |
| 296 | def _install_using_send_file(self, host, autodir): |
Chris Sosa | 3ee5d5c | 2012-02-23 11:18:41 -0800 | [diff] [blame] | 297 | dirs_to_exclude = set(["tests", "site_tests", "deps", "profilers", |
| 298 | "packages"]) |
mbligh | b8aa75b | 2009-09-18 16:50:37 +0000 | [diff] [blame] | 299 | light_files = [os.path.join(self.source_material, f) |
| 300 | for f in os.listdir(self.source_material) |
| 301 | if f not in dirs_to_exclude] |
| 302 | host.send_file(light_files, autodir, delete_dest=True) |
| 303 | |
| 304 | # create empty dirs for all the stuff we excluded |
| 305 | commands = [] |
| 306 | for path in dirs_to_exclude: |
| 307 | abs_path = os.path.join(autodir, path) |
| 308 | abs_path = utils.sh_escape(abs_path) |
| 309 | commands.append("mkdir -p '%s'" % abs_path) |
| 310 | commands.append("touch '%s'/__init__.py" % abs_path) |
Allen Li | ad719c1 | 2017-06-27 23:48:04 +0000 | [diff] [blame] | 311 | host.run(';'.join(commands)) |
mbligh | b8aa75b | 2009-09-18 16:50:37 +0000 | [diff] [blame] | 312 | |
| 313 | |
| 314 | def _install(self, host=None, autodir=None, use_autoserv=True, |
| 315 | use_packaging=True): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 316 | """ |
| 317 | Install autotest. If get() was not called previously, an |
| 318 | attempt will be made to install from the autotest svn |
| 319 | repository. |
mbligh | 9a3f5e5 | 2008-05-28 21:21:43 +0000 | [diff] [blame] | 320 | |
mbligh | bccad48 | 2009-08-24 22:08:31 +0000 | [diff] [blame] | 321 | @param host A Host instance on which autotest will be installed |
| 322 | @param autodir Location on the remote host to install to |
mbligh | b8aa75b | 2009-09-18 16:50:37 +0000 | [diff] [blame] | 323 | @param use_autoserv Enable install modes that depend on the client |
mbligh | bccad48 | 2009-08-24 22:08:31 +0000 | [diff] [blame] | 324 | running with the autoserv harness |
mbligh | b8aa75b | 2009-09-18 16:50:37 +0000 | [diff] [blame] | 325 | @param use_packaging Enable install modes that use the packaging system |
mbligh | 9a3f5e5 | 2008-05-28 21:21:43 +0000 | [diff] [blame] | 326 | |
mbligh | bccad48 | 2009-08-24 22:08:31 +0000 | [diff] [blame] | 327 | @exception AutoservError if a tarball was not specified and |
| 328 | the target host does not have svn installed in its path |
| 329 | """ |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 330 | if not host: |
| 331 | host = self.host |
| 332 | if not self.got: |
| 333 | self.get() |
| 334 | host.wait_up(timeout=30) |
| 335 | host.setup() |
showard | b18134f | 2009-03-20 20:52:18 +0000 | [diff] [blame] | 336 | logging.info("Installing autotest on %s", host.hostname) |
mbligh | 40f122a | 2007-11-03 23:08:46 +0000 | [diff] [blame] | 337 | |
jadmanski | 54f90af | 2008-10-10 16:20:55 +0000 | [diff] [blame] | 338 | # set up the autotest directory on the remote machine |
| 339 | if not autodir: |
showard | ad812bf | 2009-10-20 23:49:56 +0000 | [diff] [blame] | 340 | autodir = self.get_install_dir(host) |
| 341 | logging.info('Using installation dir %s', autodir) |
mbligh | 0562e65 | 2008-08-20 20:11:45 +0000 | [diff] [blame] | 342 | host.set_autodir(autodir) |
Allen Li | ad719c1 | 2017-06-27 23:48:04 +0000 | [diff] [blame] | 343 | host.run('mkdir -p %s' % utils.sh_escape(autodir)) |
mbligh | 40f122a | 2007-11-03 23:08:46 +0000 | [diff] [blame] | 344 | |
jadmanski | 1c3c07b | 2009-03-03 23:29:36 +0000 | [diff] [blame] | 345 | # make sure there are no files in $AUTODIR/results |
| 346 | results_path = os.path.join(autodir, 'results') |
Allen Li | ad719c1 | 2017-06-27 23:48:04 +0000 | [diff] [blame] | 347 | host.run('rm -rf %s/*' % utils.sh_escape(results_path), |
| 348 | ignore_status=True) |
jadmanski | 1c3c07b | 2009-03-03 23:29:36 +0000 | [diff] [blame] | 349 | |
mbligh | c5ddfd1 | 2008-08-04 17:15:00 +0000 | [diff] [blame] | 350 | # Fetch the autotest client from the nearest repository |
mbligh | b8aa75b | 2009-09-18 16:50:37 +0000 | [diff] [blame] | 351 | if use_packaging: |
| 352 | try: |
| 353 | self._install_using_packaging(host, autodir) |
Dan Shi | b669cbd | 2013-09-13 11:17:17 -0700 | [diff] [blame] | 354 | logging.info("Installation of autotest completed using the " |
| 355 | "packaging system.") |
mbligh | b8aa75b | 2009-09-18 16:50:37 +0000 | [diff] [blame] | 356 | return |
Eric Li | 6f27d4f | 2010-09-29 10:55:17 -0700 | [diff] [blame] | 357 | except (error.PackageInstallError, error.AutoservRunError, |
| 358 | global_config.ConfigError), e: |
mbligh | b8aa75b | 2009-09-18 16:50:37 +0000 | [diff] [blame] | 359 | logging.info("Could not install autotest using the packaging " |
Dale Curtis | cb7bfaf | 2011-06-07 16:21:57 -0700 | [diff] [blame] | 360 | "system: %s. Trying other methods", e) |
Dan Shi | 1889ca1 | 2015-04-15 09:36:06 -0700 | [diff] [blame] | 361 | else: |
| 362 | # Delete the package checksum file to force dut updating local |
| 363 | # packages. |
| 364 | command = ('rm -f "%s"' % |
Allen Li | bf0c441 | 2017-02-03 16:49:53 -0800 | [diff] [blame] | 365 | (os.path.join(autodir, packages.CHECKSUM_FILE))) |
Allen Li | ad719c1 | 2017-06-27 23:48:04 +0000 | [diff] [blame] | 366 | host.run(command) |
mbligh | c5ddfd1 | 2008-08-04 17:15:00 +0000 | [diff] [blame] | 367 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 368 | # try to install from file or directory |
| 369 | if self.source_material: |
jadmanski | 69bdaac | 2010-07-28 16:27:20 +0000 | [diff] [blame] | 370 | c = global_config.global_config |
| 371 | supports_autoserv_packaging = c.get_config_value( |
| 372 | "PACKAGES", "serve_packages_from_autoserv", type=bool) |
| 373 | # Copy autotest recursively |
| 374 | if supports_autoserv_packaging and use_autoserv: |
| 375 | self._install_using_send_file(host, autodir) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 376 | else: |
jadmanski | 69bdaac | 2010-07-28 16:27:20 +0000 | [diff] [blame] | 377 | host.send_file(self.source_material, autodir, delete_dest=True) |
Dan Shi | b669cbd | 2013-09-13 11:17:17 -0700 | [diff] [blame] | 378 | logging.info("Installation of autotest completed from %s", |
| 379 | self.source_material) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 380 | self.installed = True |
Allen Li | 7962369 | 2017-08-17 16:27:11 -0700 | [diff] [blame] | 381 | else: |
| 382 | # if that fails try to install using svn |
| 383 | if utils.run('which svn').exit_status: |
| 384 | raise error.AutoservError( |
| 385 | 'svn not found on target machine: %s' % |
| 386 | host.hostname) |
| 387 | try: |
| 388 | host.run('svn checkout %s %s' % (AUTOTEST_SVN, autodir)) |
| 389 | except error.AutoservRunError, e: |
| 390 | host.run('svn checkout %s %s' % (AUTOTEST_HTTP, autodir)) |
| 391 | logging.info("Installation of autotest completed using SVN.") |
| 392 | self.installed = True |
mbligh | 9133490 | 2007-09-28 01:47:59 +0000 | [diff] [blame] | 393 | |
Allen Li | d5abdab | 2017-02-07 16:03:43 -0800 | [diff] [blame] | 394 | # TODO(milleral): http://crbug.com/258161 |
| 395 | # Send over the most recent global_config.ini after installation if one |
| 396 | # is available. |
| 397 | # This code is a bit duplicated from |
| 398 | # _Run._create_client_config_file, but oh well. |
| 399 | if self.installed and self.source_material: |
Allen Li | 7962369 | 2017-08-17 16:27:11 -0700 | [diff] [blame] | 400 | self._send_shadow_config() |
| 401 | |
| 402 | def _send_shadow_config(self): |
| 403 | logging.info('Installing updated global_config.ini.') |
| 404 | destination = os.path.join(self.host.get_autodir(), |
| 405 | 'global_config.ini') |
| 406 | with tempfile.NamedTemporaryFile() as client_config: |
| 407 | config = global_config.global_config |
| 408 | client_section = config.get_section_values('CLIENT') |
| 409 | client_section.write(client_config) |
| 410 | client_config.flush() |
| 411 | self.host.send_file(client_config.name, destination) |
Allen Li | d5abdab | 2017-02-07 16:03:43 -0800 | [diff] [blame] | 412 | |
mbligh | 9133490 | 2007-09-28 01:47:59 +0000 | [diff] [blame] | 413 | |
jadmanski | 7c7aff3 | 2009-03-25 22:43:07 +0000 | [diff] [blame] | 414 | def uninstall(self, host=None): |
| 415 | """ |
| 416 | Uninstall (i.e. delete) autotest. Removes the autotest client install |
| 417 | from the specified host. |
| 418 | |
| 419 | @params host a Host instance from which the client will be removed |
| 420 | """ |
| 421 | if not self.installed: |
| 422 | return |
| 423 | if not host: |
| 424 | host = self.host |
| 425 | autodir = host.get_autodir() |
| 426 | if not autodir: |
| 427 | return |
| 428 | |
| 429 | # perform the actual uninstall |
Allen Li | ad719c1 | 2017-06-27 23:48:04 +0000 | [diff] [blame] | 430 | host.run("rm -rf %s" % utils.sh_escape(autodir), ignore_status=True) |
jadmanski | 7c7aff3 | 2009-03-25 22:43:07 +0000 | [diff] [blame] | 431 | host.set_autodir(None) |
| 432 | self.installed = False |
| 433 | |
| 434 | |
Dale Curtis | cb7bfaf | 2011-06-07 16:21:57 -0700 | [diff] [blame] | 435 | def get(self, location=None): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 436 | if not location: |
| 437 | location = os.path.join(self.serverdir, '../client') |
| 438 | location = os.path.abspath(location) |
Allen Li | d5abdab | 2017-02-07 16:03:43 -0800 | [diff] [blame] | 439 | installable_object.InstallableObject.get(self, location) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 440 | self.got = True |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 441 | |
| 442 | |
mbligh | e7d9c60 | 2009-07-02 19:02:33 +0000 | [diff] [blame] | 443 | def run(self, control_file, results_dir='.', host=None, timeout=None, |
Prathmesh Prabhu | beaee90 | 2017-07-26 11:58:44 -0700 | [diff] [blame] | 444 | tag=None, parallel_flag=False, background=False, |
| 445 | client_disconnect_timeout=None, use_packaging=True): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 446 | """ |
| 447 | Run an autotest job on the remote machine. |
mbligh | 9a3f5e5 | 2008-05-28 21:21:43 +0000 | [diff] [blame] | 448 | |
mbligh | e7d9c60 | 2009-07-02 19:02:33 +0000 | [diff] [blame] | 449 | @param control_file: An open file-like-obj of the control file. |
| 450 | @param results_dir: A str path where the results should be stored |
| 451 | on the local filesystem. |
| 452 | @param host: A Host instance on which the control file should |
| 453 | be run. |
| 454 | @param timeout: Maximum number of seconds to wait for the run or None. |
| 455 | @param tag: Tag name for the client side instance of autotest. |
| 456 | @param parallel_flag: Flag set when multiple jobs are run at the |
| 457 | same time. |
Prathmesh Prabhu | beaee90 | 2017-07-26 11:58:44 -0700 | [diff] [blame] | 458 | @param background: Indicates that the client should be launched as |
| 459 | a background job; the code calling run will be responsible |
| 460 | for monitoring the client and collecting the results. |
mbligh | e7d9c60 | 2009-07-02 19:02:33 +0000 | [diff] [blame] | 461 | @param client_disconnect_timeout: Seconds to wait for the remote host |
Dale Curtis | cb7bfaf | 2011-06-07 16:21:57 -0700 | [diff] [blame] | 462 | to come back after a reboot. Defaults to the host setting for |
| 463 | DEFAULT_REBOOT_TIMEOUT. |
mbligh | e7d9c60 | 2009-07-02 19:02:33 +0000 | [diff] [blame] | 464 | |
| 465 | @raises AutotestRunError: If there is a problem executing |
| 466 | the control file. |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 467 | """ |
Dan Shi | b669cbd | 2013-09-13 11:17:17 -0700 | [diff] [blame] | 468 | host = self._get_host_and_setup(host, use_packaging=use_packaging) |
xixuan | 02b6fee | 2017-02-01 18:35:20 -0800 | [diff] [blame] | 469 | logging.debug('Autotest job starts on remote host: %s', |
| 470 | host.hostname) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 471 | results_dir = os.path.abspath(results_dir) |
mbligh | c1cbc99 | 2008-05-27 20:01:45 +0000 | [diff] [blame] | 472 | |
Dale Curtis | cb7bfaf | 2011-06-07 16:21:57 -0700 | [diff] [blame] | 473 | if client_disconnect_timeout is None: |
| 474 | client_disconnect_timeout = host.DEFAULT_REBOOT_TIMEOUT |
| 475 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 476 | if tag: |
| 477 | results_dir = os.path.join(results_dir, tag) |
mbligh | c1cbc99 | 2008-05-27 20:01:45 +0000 | [diff] [blame] | 478 | |
Prathmesh Prabhu | beaee90 | 2017-07-26 11:58:44 -0700 | [diff] [blame] | 479 | atrun = _Run(host, results_dir, tag, parallel_flag, background) |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 480 | self._do_run(control_file, results_dir, host, atrun, timeout, |
Dan Shi | b669cbd | 2013-09-13 11:17:17 -0700 | [diff] [blame] | 481 | client_disconnect_timeout, use_packaging=use_packaging) |
mbligh | d8b3925 | 2008-03-20 21:15:03 +0000 | [diff] [blame] | 482 | |
| 483 | |
Dan Shi | b669cbd | 2013-09-13 11:17:17 -0700 | [diff] [blame] | 484 | def _get_host_and_setup(self, host, use_packaging=True): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 485 | if not host: |
| 486 | host = self.host |
| 487 | if not self.installed: |
Dan Shi | b669cbd | 2013-09-13 11:17:17 -0700 | [diff] [blame] | 488 | self.install(host, use_packaging=use_packaging) |
mbligh | 9133490 | 2007-09-28 01:47:59 +0000 | [diff] [blame] | 489 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 490 | host.wait_up(timeout=30) |
| 491 | return host |
mbligh | d8b3925 | 2008-03-20 21:15:03 +0000 | [diff] [blame] | 492 | |
| 493 | |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 494 | def _do_run(self, control_file, results_dir, host, atrun, timeout, |
Dan Shi | b669cbd | 2013-09-13 11:17:17 -0700 | [diff] [blame] | 495 | client_disconnect_timeout, use_packaging=True): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 496 | try: |
| 497 | atrun.verify_machine() |
| 498 | except: |
showard | b18134f | 2009-03-20 20:52:18 +0000 | [diff] [blame] | 499 | logging.error("Verify failed on %s. Reinstalling autotest", |
| 500 | host.hostname) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 501 | self.install(host) |
Fang Deng | b9cd83c | 2015-01-27 10:16:08 -0800 | [diff] [blame] | 502 | atrun.verify_machine() |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 503 | debug = os.path.join(results_dir, 'debug') |
| 504 | try: |
| 505 | os.makedirs(debug) |
mbligh | 0910844 | 2008-10-15 16:27:38 +0000 | [diff] [blame] | 506 | except Exception: |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 507 | pass |
mbligh | 9a3f5e5 | 2008-05-28 21:21:43 +0000 | [diff] [blame] | 508 | |
mbligh | 0910844 | 2008-10-15 16:27:38 +0000 | [diff] [blame] | 509 | delete_file_list = [atrun.remote_control_file, |
| 510 | atrun.remote_control_file + '.state', |
| 511 | atrun.manual_control_file, |
| 512 | atrun.manual_control_file + '.state'] |
| 513 | cmd = ';'.join('rm -f ' + control for control in delete_file_list) |
Allen Li | ad719c1 | 2017-06-27 23:48:04 +0000 | [diff] [blame] | 514 | host.run(cmd, ignore_status=True) |
mbligh | 9a3f5e5 | 2008-05-28 21:21:43 +0000 | [diff] [blame] | 515 | |
Dale Curtis | 386eea7 | 2011-09-21 18:43:04 -0700 | [diff] [blame] | 516 | tmppath = utils.get(control_file, local_copy=True) |
mbligh | c5ddfd1 | 2008-08-04 17:15:00 +0000 | [diff] [blame] | 517 | |
jadmanski | cb0e161 | 2009-02-27 18:03:10 +0000 | [diff] [blame] | 518 | # build up the initialization prologue for the control file |
| 519 | prologue_lines = [] |
jadmanski | 23afbec | 2008-09-17 18:12:07 +0000 | [diff] [blame] | 520 | |
mbligh | 2f07683 | 2010-03-30 17:08:20 +0000 | [diff] [blame] | 521 | # Add the additional user arguments |
jadmanski | 808f4b1 | 2010-04-09 22:30:31 +0000 | [diff] [blame] | 522 | prologue_lines.append("args = %r\n" % self.job.args) |
mbligh | 2f07683 | 2010-03-30 17:08:20 +0000 | [diff] [blame] | 523 | |
mbligh | 0910844 | 2008-10-15 16:27:38 +0000 | [diff] [blame] | 524 | # If the packaging system is being used, add the repository list. |
mbligh | ddc9a40 | 2010-01-15 20:33:34 +0000 | [diff] [blame] | 525 | repos = None |
mbligh | c5ddfd1 | 2008-08-04 17:15:00 +0000 | [diff] [blame] | 526 | try: |
Dan Shi | b669cbd | 2013-09-13 11:17:17 -0700 | [diff] [blame] | 527 | if use_packaging: |
| 528 | repos = self.get_fetch_location() |
| 529 | prologue_lines.append('job.add_repository(%s)\n' % repos) |
| 530 | else: |
| 531 | logging.debug('use_packaging is set to False, do not add any ' |
| 532 | 'repository.') |
mbligh | c5ddfd1 | 2008-08-04 17:15:00 +0000 | [diff] [blame] | 533 | except global_config.ConfigError, e: |
mbligh | ddc9a40 | 2010-01-15 20:33:34 +0000 | [diff] [blame] | 534 | # If repos is defined packaging is enabled so log the error |
| 535 | if repos: |
| 536 | logging.error(e) |
mbligh | c5ddfd1 | 2008-08-04 17:15:00 +0000 | [diff] [blame] | 537 | |
jadmanski | e2eef7b | 2009-03-03 23:55:13 +0000 | [diff] [blame] | 538 | # on full-size installs, turn on any profilers the server is using |
Prathmesh Prabhu | beaee90 | 2017-07-26 11:58:44 -0700 | [diff] [blame] | 539 | if not atrun.background: |
| 540 | running_profilers = host.job.profilers.add_log.iteritems() |
| 541 | for profiler, (args, dargs) in running_profilers: |
| 542 | call_args = [repr(profiler)] |
| 543 | call_args += [repr(arg) for arg in args] |
| 544 | call_args += ["%s=%r" % item for item in dargs.iteritems()] |
| 545 | prologue_lines.append("job.profilers.add(%s)\n" |
| 546 | % ", ".join(call_args)) |
jadmanski | e2eef7b | 2009-03-03 23:55:13 +0000 | [diff] [blame] | 547 | cfile = "".join(prologue_lines) |
| 548 | |
mbligh | 0910844 | 2008-10-15 16:27:38 +0000 | [diff] [blame] | 549 | cfile += open(tmppath).read() |
| 550 | open(tmppath, "w").write(cfile) |
mbligh | c5ddfd1 | 2008-08-04 17:15:00 +0000 | [diff] [blame] | 551 | |
jadmanski | c09fc15 | 2008-10-15 17:56:59 +0000 | [diff] [blame] | 552 | # Create and copy state file to remote_control_file + '.state' |
mbligh | fc3da5b | 2010-01-06 18:37:22 +0000 | [diff] [blame] | 553 | state_file = host.job.preprocess_client_state() |
mbligh | fbf73ae | 2009-12-19 05:22:42 +0000 | [diff] [blame] | 554 | host.send_file(state_file, atrun.remote_control_file + '.init.state') |
jadmanski | c09fc15 | 2008-10-15 17:56:59 +0000 | [diff] [blame] | 555 | os.remove(state_file) |
| 556 | |
mbligh | c5ddfd1 | 2008-08-04 17:15:00 +0000 | [diff] [blame] | 557 | # Copy control_file to remote_control_file on the host |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 558 | host.send_file(tmppath, atrun.remote_control_file) |
| 559 | if os.path.abspath(tmppath) != os.path.abspath(control_file): |
| 560 | os.remove(tmppath) |
mbligh | 0e4613b | 2007-10-29 16:55:07 +0000 | [diff] [blame] | 561 | |
jadmanski | 6bb32d7 | 2009-03-19 20:25:24 +0000 | [diff] [blame] | 562 | atrun.execute_control( |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 563 | timeout=timeout, |
Dale Curtis | 9285ddf | 2011-01-05 11:47:24 -0800 | [diff] [blame] | 564 | client_disconnect_timeout=client_disconnect_timeout) |
jadmanski | 23afbec | 2008-09-17 18:12:07 +0000 | [diff] [blame] | 565 | |
| 566 | |
Kevin Cheng | c51cc1f | 2017-11-17 15:49:55 -0800 | [diff] [blame] | 567 | @staticmethod |
| 568 | def extract_test_failure_msg(failure_status_line): |
| 569 | """Extract the test failure message from the status line. |
| 570 | |
| 571 | @param failure_status_line: String of test failure status line, it will |
| 572 | look like: |
| 573 | FAIL <test name> <test name> timestamp=<ts> localtime=<lt> <reason> |
| 574 | |
| 575 | @returns String of the reason, return empty string if we can't regex out |
| 576 | reason. |
| 577 | """ |
| 578 | fail_msg = '' |
| 579 | match = _FAIL_STATUS_RE.match(failure_status_line) |
| 580 | if match: |
| 581 | fail_msg = match.group('fail_msg') |
| 582 | return fail_msg |
| 583 | |
| 584 | |
harpreet | f531d07 | 2016-04-19 18:37:26 -0700 | [diff] [blame] | 585 | @classmethod |
| 586 | def _check_client_test_result(cls, host, test_name): |
| 587 | """ |
| 588 | Check result of client test. |
| 589 | Autotest will store results in the file name status. |
| 590 | We check that second to last line in that file begins with 'END GOOD' |
| 591 | |
| 592 | @raises TestFail: If client test does not pass. |
| 593 | """ |
| 594 | client_result_dir = '%s/results/default' % host.autodir |
| 595 | command = 'tail -2 %s/status | head -1' % client_result_dir |
Allen Li | ad719c1 | 2017-06-27 23:48:04 +0000 | [diff] [blame] | 596 | status = host.run(command).stdout.strip() |
harpreet | f531d07 | 2016-04-19 18:37:26 -0700 | [diff] [blame] | 597 | logging.info(status) |
| 598 | if status[:8] != 'END GOOD': |
Kevin Cheng | 821e261 | 2017-11-27 10:21:15 -0800 | [diff] [blame] | 599 | test_fail_status_line_cmd = ( |
| 600 | 'grep "^\s*FAIL\s*%s" %s/status | tail -n 1' % |
| 601 | (test_name, client_result_dir)) |
Kevin Cheng | c51cc1f | 2017-11-17 15:49:55 -0800 | [diff] [blame] | 602 | test_fail_msg = cls.extract_test_failure_msg( |
| 603 | host.run(test_fail_status_line_cmd).stdout.strip()) |
| 604 | test_fail_msg_reason = ('' if not test_fail_msg |
| 605 | else ' (reason: %s)' % test_fail_msg) |
| 606 | test_fail_status = '%s client test did not pass%s.' % ( |
| 607 | test_name, test_fail_msg_reason) |
| 608 | raise error.TestFail(test_fail_status) |
harpreet | f531d07 | 2016-04-19 18:37:26 -0700 | [diff] [blame] | 609 | |
| 610 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 611 | def run_timed_test(self, test_name, results_dir='.', host=None, |
Prathmesh Prabhu | beaee90 | 2017-07-26 11:58:44 -0700 | [diff] [blame] | 612 | timeout=None, parallel_flag=False, background=False, |
Dennis Jeffrey | b0be88b | 2013-04-18 11:18:38 -0700 | [diff] [blame] | 613 | client_disconnect_timeout=None, *args, **dargs): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 614 | """ |
| 615 | Assemble a tiny little control file to just run one test, |
| 616 | and run it as an autotest client-side test |
| 617 | """ |
| 618 | if not host: |
| 619 | host = self.host |
| 620 | if not self.installed: |
| 621 | self.install(host) |
Max Timkovich | 4657b96 | 2018-03-26 17:01:53 -0700 | [diff] [blame] | 622 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 623 | opts = ["%s=%s" % (o[0], repr(o[1])) for o in dargs.items()] |
| 624 | cmd = ", ".join([repr(test_name)] + map(repr, args) + opts) |
| 625 | control = "job.run_test(%s)\n" % cmd |
Dennis Jeffrey | b0be88b | 2013-04-18 11:18:38 -0700 | [diff] [blame] | 626 | self.run(control, results_dir, host, timeout=timeout, |
Prathmesh Prabhu | beaee90 | 2017-07-26 11:58:44 -0700 | [diff] [blame] | 627 | parallel_flag=parallel_flag, background=background, |
Andrew Bresticker | 2da1b76 | 2013-01-15 16:11:18 -0800 | [diff] [blame] | 628 | client_disconnect_timeout=client_disconnect_timeout) |
mbligh | 0e4613b | 2007-10-29 16:55:07 +0000 | [diff] [blame] | 629 | |
harpreet | f045206 | 2016-04-21 11:22:46 -0700 | [diff] [blame] | 630 | if dargs.get('check_client_result', False): |
| 631 | self._check_client_test_result(host, test_name) |
harpreet | f531d07 | 2016-04-19 18:37:26 -0700 | [diff] [blame] | 632 | |
mbligh | 0e4613b | 2007-10-29 16:55:07 +0000 | [diff] [blame] | 633 | |
Dennis Jeffrey | b0be88b | 2013-04-18 11:18:38 -0700 | [diff] [blame] | 634 | def run_test(self, test_name, results_dir='.', host=None, |
Prathmesh Prabhu | beaee90 | 2017-07-26 11:58:44 -0700 | [diff] [blame] | 635 | parallel_flag=False, background=False, |
Andrew Bresticker | 2da1b76 | 2013-01-15 16:11:18 -0800 | [diff] [blame] | 636 | client_disconnect_timeout=None, *args, **dargs): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 637 | self.run_timed_test(test_name, results_dir, host, timeout=None, |
Prathmesh Prabhu | beaee90 | 2017-07-26 11:58:44 -0700 | [diff] [blame] | 638 | parallel_flag=parallel_flag, background=background, |
Andrew Bresticker | 2da1b76 | 2013-01-15 16:11:18 -0800 | [diff] [blame] | 639 | client_disconnect_timeout=client_disconnect_timeout, |
jadmanski | c98c470 | 2009-01-05 15:50:06 +0000 | [diff] [blame] | 640 | *args, **dargs) |
mbligh | d54832b | 2007-07-25 16:46:56 +0000 | [diff] [blame] | 641 | |
| 642 | |
Allen Li | d5abdab | 2017-02-07 16:03:43 -0800 | [diff] [blame] | 643 | def run_static_method(self, module, method, results_dir='.', host=None, |
| 644 | *args): |
| 645 | """Runs a non-instance method with |args| from |module| on the client. |
| 646 | |
| 647 | This method runs a static/class/module autotest method on the client. |
| 648 | For example: |
| 649 | run_static_method("autotest_lib.client.cros.cros_ui", "reboot") |
| 650 | |
| 651 | Will run autotest_lib.client.cros.cros_ui.reboot() on the client. |
| 652 | |
| 653 | @param module: module name as you would refer to it when importing in a |
| 654 | control file. e.g. autotest_lib.client.common_lib.module_name. |
| 655 | @param method: the method you want to call. |
| 656 | @param results_dir: A str path where the results should be stored |
| 657 | on the local filesystem. |
| 658 | @param host: A Host instance on which the control file should |
| 659 | be run. |
| 660 | @param args: args to pass to the method. |
| 661 | """ |
| 662 | control = "\n".join(["import %s" % module, |
| 663 | "%s.%s(%s)\n" % (module, method, |
| 664 | ','.join(map(repr, args)))]) |
| 665 | self.run(control, results_dir=results_dir, host=host) |
| 666 | |
| 667 | |
Allen Li | 24381f3 | 2017-02-07 15:43:37 -0800 | [diff] [blame] | 668 | class _Run(object): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 669 | """ |
| 670 | Represents a run of autotest control file. This class maintains |
| 671 | all the state necessary as an autotest control file is executed. |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 672 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 673 | It is not intended to be used directly, rather control files |
| 674 | should be run using the run method in Autotest. |
| 675 | """ |
Prathmesh Prabhu | beaee90 | 2017-07-26 11:58:44 -0700 | [diff] [blame] | 676 | def __init__(self, host, results_dir, tag, parallel_flag, background): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 677 | self.host = host |
| 678 | self.results_dir = results_dir |
| 679 | self.env = host.env |
| 680 | self.tag = tag |
| 681 | self.parallel_flag = parallel_flag |
Prathmesh Prabhu | beaee90 | 2017-07-26 11:58:44 -0700 | [diff] [blame] | 682 | self.background = background |
showard | ad812bf | 2009-10-20 23:49:56 +0000 | [diff] [blame] | 683 | self.autodir = Autotest.get_installed_autodir(self.host) |
mbligh | 78bf535 | 2008-07-11 20:27:36 +0000 | [diff] [blame] | 684 | control = os.path.join(self.autodir, 'control') |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 685 | if tag: |
mbligh | 78bf535 | 2008-07-11 20:27:36 +0000 | [diff] [blame] | 686 | control += '.' + tag |
| 687 | self.manual_control_file = control |
| 688 | self.remote_control_file = control + '.autoserv' |
lmr | 6d08b3c | 2009-11-18 19:26:38 +0000 | [diff] [blame] | 689 | self.config_file = os.path.join(self.autodir, 'global_config.ini') |
mbligh | dc735a2 | 2007-08-02 16:54:37 +0000 | [diff] [blame] | 690 | |
| 691 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 692 | def verify_machine(self): |
| 693 | binary = os.path.join(self.autodir, 'bin/autotest') |
| 694 | try: |
Allen Li | ad719c1 | 2017-06-27 23:48:04 +0000 | [diff] [blame] | 695 | self.host.run('ls %s > /dev/null 2>&1' % binary) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 696 | except: |
lmr | d6d27ed | 2009-12-08 19:58:33 +0000 | [diff] [blame] | 697 | raise error.AutoservInstallError( |
| 698 | "Autotest does not appear to be installed") |
mbligh | dc735a2 | 2007-08-02 16:54:37 +0000 | [diff] [blame] | 699 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 700 | if not self.parallel_flag: |
| 701 | tmpdir = os.path.join(self.autodir, 'tmp') |
| 702 | download = os.path.join(self.autodir, 'tests/download') |
Allen Li | ad719c1 | 2017-06-27 23:48:04 +0000 | [diff] [blame] | 703 | self.host.run('umount %s' % tmpdir, ignore_status=True) |
| 704 | self.host.run('umount %s' % download, ignore_status=True) |
mbligh | dc735a2 | 2007-08-02 16:54:37 +0000 | [diff] [blame] | 705 | |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 706 | |
Prathmesh Prabhu | cb4f549 | 2017-07-26 18:57:21 +0000 | [diff] [blame] | 707 | def get_base_cmd_args(self, section): |
showard | 234b296 | 2009-07-28 20:02:30 +0000 | [diff] [blame] | 708 | args = ['--verbose'] |
Prathmesh Prabhu | cb4f549 | 2017-07-26 18:57:21 +0000 | [diff] [blame] | 709 | if section > 0: |
| 710 | args.append('-c') |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 711 | if self.tag: |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 712 | args.append('-t %s' % self.tag) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 713 | if self.host.job.use_external_logging(): |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 714 | args.append('-l') |
mbligh | ce955fc | 2009-08-24 21:59:02 +0000 | [diff] [blame] | 715 | if self.host.hostname: |
| 716 | args.append('--hostname=%s' % self.host.hostname) |
mbligh | 0d0f67d | 2009-11-06 03:15:03 +0000 | [diff] [blame] | 717 | args.append('--user=%s' % self.host.job.user) |
mbligh | ce955fc | 2009-08-24 21:59:02 +0000 | [diff] [blame] | 718 | |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 719 | args.append(self.remote_control_file) |
| 720 | return args |
| 721 | |
| 722 | |
Prathmesh Prabhu | beaee90 | 2017-07-26 11:58:44 -0700 | [diff] [blame] | 723 | def get_background_cmd(self, section): |
| 724 | cmd = ['nohup', os.path.join(self.autodir, 'bin/autotest_client')] |
| 725 | cmd += self.get_base_cmd_args(section) |
| 726 | cmd += ['>/dev/null', '2>/dev/null', '&'] |
| 727 | return ' '.join(cmd) |
| 728 | |
| 729 | |
Prathmesh Prabhu | cb4f549 | 2017-07-26 18:57:21 +0000 | [diff] [blame] | 730 | def get_daemon_cmd(self, section, monitor_dir): |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 731 | cmd = ['nohup', os.path.join(self.autodir, 'bin/autotestd'), |
| 732 | monitor_dir, '-H autoserv'] |
Prathmesh Prabhu | cb4f549 | 2017-07-26 18:57:21 +0000 | [diff] [blame] | 733 | cmd += self.get_base_cmd_args(section) |
jadmanski | 69bdaac | 2010-07-28 16:27:20 +0000 | [diff] [blame] | 734 | cmd += ['>/dev/null', '2>/dev/null', '&'] |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 735 | return ' '.join(cmd) |
| 736 | |
| 737 | |
| 738 | def get_monitor_cmd(self, monitor_dir, stdout_read, stderr_read): |
| 739 | cmd = [os.path.join(self.autodir, 'bin', 'autotestd_monitor'), |
| 740 | monitor_dir, str(stdout_read), str(stderr_read)] |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 741 | return ' '.join(cmd) |
mbligh | adf2aab | 2007-11-29 18:16:43 +0000 | [diff] [blame] | 742 | |
mbligh | d8b3925 | 2008-03-20 21:15:03 +0000 | [diff] [blame] | 743 | |
jadmanski | 4d03cf6 | 2010-03-04 18:32:28 +0000 | [diff] [blame] | 744 | def get_client_log(self): |
| 745 | """Find what the "next" client.* prefix should be |
| 746 | |
| 747 | @returns A string of the form client.INTEGER that should be prefixed |
| 748 | to all client debug log files. |
| 749 | """ |
| 750 | max_digit = -1 |
| 751 | debug_dir = os.path.join(self.results_dir, 'debug') |
| 752 | client_logs = glob.glob(os.path.join(debug_dir, 'client.*.*')) |
| 753 | for log in client_logs: |
| 754 | _, number, _ = log.split('.', 2) |
| 755 | if number.isdigit(): |
| 756 | max_digit = max(max_digit, int(number)) |
| 757 | return 'client.%d' % (max_digit + 1) |
| 758 | |
| 759 | |
| 760 | def copy_client_config_file(self, client_log_prefix=None): |
| 761 | """ |
| 762 | Create and copy the client config file based on the server config. |
| 763 | |
| 764 | @param client_log_prefix: Optional prefix to prepend to log files. |
| 765 | """ |
| 766 | client_config_file = self._create_client_config_file(client_log_prefix) |
| 767 | self.host.send_file(client_config_file, self.config_file) |
| 768 | os.remove(client_config_file) |
| 769 | |
| 770 | |
| 771 | def _create_client_config_file(self, client_log_prefix=None): |
| 772 | """ |
| 773 | Create a temporary file with the [CLIENT] section configuration values |
| 774 | taken from the server global_config.ini. |
| 775 | |
| 776 | @param client_log_prefix: Optional prefix to prepend to log files. |
| 777 | |
| 778 | @return: Path of the temporary file generated. |
| 779 | """ |
| 780 | config = global_config.global_config.get_section_values('CLIENT') |
| 781 | if client_log_prefix: |
| 782 | config.set('CLIENT', 'default_logging_name', client_log_prefix) |
| 783 | return self._create_aux_file(config.write) |
| 784 | |
| 785 | |
| 786 | def _create_aux_file(self, func, *args): |
| 787 | """ |
| 788 | Creates a temporary file and writes content to it according to a |
| 789 | content creation function. The file object is appended to *args, which |
| 790 | is then passed to the content creation function |
| 791 | |
| 792 | @param func: Function that will be used to write content to the |
| 793 | temporary file. |
| 794 | @param *args: List of parameters that func takes. |
| 795 | @return: Path to the temporary file that was created. |
| 796 | """ |
| 797 | fd, path = tempfile.mkstemp(dir=self.host.job.tmpdir) |
| 798 | aux_file = os.fdopen(fd, "w") |
| 799 | try: |
| 800 | list_args = list(args) |
| 801 | list_args.append(aux_file) |
| 802 | func(*list_args) |
| 803 | finally: |
| 804 | aux_file.close() |
| 805 | return path |
mbligh | d8b3925 | 2008-03-20 21:15:03 +0000 | [diff] [blame] | 806 | |
| 807 | |
jadmanski | b264ed0 | 2009-01-12 23:54:27 +0000 | [diff] [blame] | 808 | @staticmethod |
| 809 | def is_client_job_finished(last_line): |
Fei Shao | 24a583c | 2019-01-30 17:14:25 +0800 | [diff] [blame] | 810 | return bool(re.match(r'^\t*END .*\t[\w.-]+\t[\w.-]+\t.*$', last_line)) |
jadmanski | b264ed0 | 2009-01-12 23:54:27 +0000 | [diff] [blame] | 811 | |
| 812 | |
| 813 | @staticmethod |
| 814 | def is_client_job_rebooting(last_line): |
Fei Shao | 24a583c | 2019-01-30 17:14:25 +0800 | [diff] [blame] | 815 | return bool(re.match(r'^\t*GOOD\t[\w.-]+\treboot\.start.*$', last_line)) |
jadmanski | b264ed0 | 2009-01-12 23:54:27 +0000 | [diff] [blame] | 816 | |
| 817 | |
Ilja H. Friedel | ede3e7b | 2019-12-13 20:28:51 -0800 | [diff] [blame] | 818 | # Roughly ordered list from concrete to less specific reboot causes. |
| 819 | _failure_reasons = [ |
| 820 | # Try to find possible reasons leading towards failure. |
| 821 | ('ethernet recovery methods have failed. Rebooting.', |
| 822 | 'dead ethernet dongle crbug/1031035'), |
| 823 | # GPU hangs are not always recovered from. |
| 824 | ('[drm:amdgpu_job_timedout] \*ERROR\* ring gfx timeout', |
| 825 | 'drm ring gfx timeout'), |
| 826 | ('[drm:do_aquire_global_lock] \*ERROR(.*)hw_done or flip_done timed', |
| 827 | 'drm hw/flip timeout'), |
| 828 | ('[drm:i915_hangcheck_hung] \*ERROR\* Hangcheck(.*)GPU hung', |
| 829 | 'drm GPU hung'), |
| 830 | # TODO(ihf): try to get a better magic signature for kernel crashes. |
| 831 | ('BUG: unable to handle kernel paging request', 'kernel paging'), |
| 832 | ('Kernel panic - not syncing: Out of memory', 'kernel out of memory'), |
| 833 | ('Kernel panic - not syncing', 'kernel panic'), |
| 834 | # Fish for user mode killing OOM messages. Shows unstable system. |
| 835 | ('out_of_memory', 'process out of memory'), |
| 836 | # Reboot was bad enough to have truncated the logs. |
| 837 | ('crash_reporter(.*)Stored kcrash', 'kcrash'), |
| 838 | ('crash_reporter(.*)Last shutdown was not clean', 'not clean'), |
| 839 | ] |
| 840 | |
| 841 | def _diagnose_reboot(self): |
| 842 | """ |
| 843 | Runs diagnostic check on a rebooted DUT. |
| 844 | |
| 845 | TODO(ihf): if this analysis is useful consider moving the code to the |
| 846 | DUT into a script and call it from here. This is more |
| 847 | powerful and might be cleaner to grow in functionality. But |
| 848 | it may also be less robust if stateful is damaged during the |
| 849 | reboot. |
| 850 | |
| 851 | @returns msg describing reboot reason. |
| 852 | """ |
| 853 | reasons = [] |
| 854 | for (message, bucket) in self._failure_reasons: |
Ilja H. Friedel | 8443b25 | 2019-12-18 13:13:04 -0800 | [diff] [blame] | 855 | # Use -a option for grep to avoid "binary file" warning to stdout. |
Ilja H. Friedel | ede3e7b | 2019-12-13 20:28:51 -0800 | [diff] [blame] | 856 | # The grep -v is added to not match itself in the log (across jobs). |
| 857 | # Using grep is slightly problematic as it finds any reason, not |
| 858 | # just the most recent reason (since 2 boots ago), so it may guess |
| 859 | # wrong. Multiple reboots are unusual in the lab setting though and |
| 860 | # it is better to have a reasonable guess than no reason at all. |
| 861 | found = self.host.run( |
Ilja H. Friedel | 8443b25 | 2019-12-18 13:13:04 -0800 | [diff] [blame] | 862 | "grep -aE '" + message + "' /var/log/messages | grep -av grep", |
Ilja H. Friedel | ede3e7b | 2019-12-13 20:28:51 -0800 | [diff] [blame] | 863 | ignore_status=True |
| 864 | ).stdout |
| 865 | if found and found.strip(): |
| 866 | reasons.append(bucket) |
| 867 | signature = 'reason unknown' |
| 868 | if reasons: |
| 869 | # Concatenate possible reasons found to obtain a magic signature. |
| 870 | signature = ', '.join(reasons) |
| 871 | return ('DUT rebooted during the test run. (%s)\n' % signature) |
| 872 | |
| 873 | |
beeps | 607428b | 2013-03-25 16:43:20 -0700 | [diff] [blame] | 874 | def _diagnose_dut(self, old_boot_id=None): |
| 875 | """ |
| 876 | Run diagnostic checks on a DUT. |
| 877 | |
| 878 | 1. ping: A dead host will not respond to pings. |
| 879 | 2. ssh (happens with 3.): DUT hangs usually fail in authentication |
| 880 | but respond to pings. |
| 881 | 3. Check if a reboot occured: A healthy but unexpected reboot leaves the |
| 882 | host running with a new boot id. |
| 883 | |
| 884 | This method will always raise an exception from the AutotestFailure |
| 885 | family and should only get called when the reason for a test failing |
| 886 | is ambiguous. |
| 887 | |
| 888 | @raises AutotestDeviceNotPingable: If the DUT doesn't respond to ping. |
| 889 | @raises AutotestDeviceNotSSHable: If we cannot SSH into the DUT. |
| 890 | @raises AutotestDeviceRebooted: If the boot id changed. |
| 891 | @raises AutotestAbort: If none of the above exceptions were raised. |
| 892 | Since we have no recourse we must abort at this stage. |
| 893 | """ |
| 894 | msg = 'Autotest client terminated unexpectedly: ' |
| 895 | if utils.ping(self.host.hostname, tries=1, deadline=1) != 0: |
| 896 | msg += 'DUT is no longer pingable, it may have rebooted or hung.\n' |
| 897 | raise AutotestDeviceNotPingable(msg) |
| 898 | |
| 899 | if old_boot_id: |
| 900 | try: |
| 901 | new_boot_id = self.host.get_boot_id(timeout=60) |
| 902 | except Exception as e: |
| 903 | msg += ('DUT is pingable but not SSHable, it most likely' |
beeps | 1476881 | 2013-09-25 12:58:45 -0700 | [diff] [blame] | 904 | ' sporadically rebooted during testing. %s\n' % str(e)) |
beeps | 607428b | 2013-03-25 16:43:20 -0700 | [diff] [blame] | 905 | raise AutotestDeviceNotSSHable(msg) |
| 906 | else: |
| 907 | if new_boot_id != old_boot_id: |
Ilja H. Friedel | ede3e7b | 2019-12-13 20:28:51 -0800 | [diff] [blame] | 908 | msg += self._diagnose_reboot() |
beeps | 607428b | 2013-03-25 16:43:20 -0700 | [diff] [blame] | 909 | raise AutotestDeviceRebooted(msg) |
| 910 | |
| 911 | msg += ('DUT is pingable, SSHable and did NOT restart ' |
| 912 | 'un-expectedly. We probably lost connectivity during the ' |
| 913 | 'test.') |
| 914 | else: |
| 915 | msg += ('DUT is pingable, could not determine if an un-expected ' |
| 916 | 'reboot occured during the test.') |
| 917 | |
| 918 | raise AutotestAbort(msg) |
| 919 | |
| 920 | |
beeps | 07f53b9 | 2013-01-08 12:55:10 -0800 | [diff] [blame] | 921 | def log_unexpected_abort(self, stderr_redirector, old_boot_id=None): |
| 922 | """ |
beeps | 607428b | 2013-03-25 16:43:20 -0700 | [diff] [blame] | 923 | Logs that something unexpected happened, then tries to diagnose the |
| 924 | failure. The purpose of this function is only to close out the status |
| 925 | log with the appropriate error message, not to critically terminate |
| 926 | the program. |
beeps | 07f53b9 | 2013-01-08 12:55:10 -0800 | [diff] [blame] | 927 | |
| 928 | @param stderr_redirector: log stream. |
| 929 | @param old_boot_id: boot id used to infer if a reboot occured. |
| 930 | """ |
jadmanski | a61edad | 2009-05-21 22:17:49 +0000 | [diff] [blame] | 931 | stderr_redirector.flush_all_buffers() |
beeps | 607428b | 2013-03-25 16:43:20 -0700 | [diff] [blame] | 932 | try: |
| 933 | self._diagnose_dut(old_boot_id) |
| 934 | except AutotestFailure as e: |
| 935 | self.host.job.record('END ABORT', None, None, str(e)) |
jadmanski | b264ed0 | 2009-01-12 23:54:27 +0000 | [diff] [blame] | 936 | |
| 937 | |
Prathmesh Prabhu | beaee90 | 2017-07-26 11:58:44 -0700 | [diff] [blame] | 938 | def _execute_in_background(self, section, timeout): |
| 939 | full_cmd = self.get_background_cmd(section) |
| 940 | devnull = open(os.devnull, "w") |
| 941 | |
| 942 | self.copy_client_config_file(self.get_client_log()) |
| 943 | |
| 944 | self.host.job.push_execution_context(self.results_dir) |
| 945 | try: |
| 946 | result = self.host.run(full_cmd, ignore_status=True, |
| 947 | timeout=timeout, |
| 948 | stdout_tee=devnull, |
| 949 | stderr_tee=devnull) |
| 950 | finally: |
| 951 | self.host.job.pop_execution_context() |
| 952 | |
| 953 | return result |
| 954 | |
| 955 | |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 956 | @staticmethod |
| 957 | def _strip_stderr_prologue(stderr): |
| 958 | """Strips the 'standard' prologue that get pre-pended to every |
| 959 | remote command and returns the text that was actually written to |
| 960 | stderr by the remote command.""" |
| 961 | stderr_lines = stderr.split("\n")[1:] |
| 962 | if not stderr_lines: |
| 963 | return "" |
| 964 | elif stderr_lines[0].startswith("NOTE: autotestd_monitor"): |
| 965 | del stderr_lines[0] |
| 966 | return "\n".join(stderr_lines) |
| 967 | |
| 968 | |
Prathmesh Prabhu | cb4f549 | 2017-07-26 18:57:21 +0000 | [diff] [blame] | 969 | def _execute_daemon(self, section, timeout, stderr_redirector, |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 970 | client_disconnect_timeout): |
| 971 | monitor_dir = self.host.get_tmp_dir() |
Prathmesh Prabhu | cb4f549 | 2017-07-26 18:57:21 +0000 | [diff] [blame] | 972 | daemon_cmd = self.get_daemon_cmd(section, monitor_dir) |
jadmanski | 4d03cf6 | 2010-03-04 18:32:28 +0000 | [diff] [blame] | 973 | |
| 974 | # grab the location for the server-side client log file |
| 975 | client_log_prefix = self.get_client_log() |
| 976 | client_log_path = os.path.join(self.results_dir, 'debug', |
| 977 | client_log_prefix + '.log') |
| 978 | client_log = open(client_log_path, 'w', 0) |
| 979 | self.copy_client_config_file(client_log_prefix) |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 980 | |
| 981 | stdout_read = stderr_read = 0 |
mbligh | 0d0f67d | 2009-11-06 03:15:03 +0000 | [diff] [blame] | 982 | self.host.job.push_execution_context(self.results_dir) |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 983 | try: |
Allen Li | ad719c1 | 2017-06-27 23:48:04 +0000 | [diff] [blame] | 984 | self.host.run(daemon_cmd, ignore_status=True, timeout=timeout) |
jadmanski | 91d56a9 | 2009-04-01 15:20:40 +0000 | [diff] [blame] | 985 | disconnect_warnings = [] |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 986 | while True: |
| 987 | monitor_cmd = self.get_monitor_cmd(monitor_dir, stdout_read, |
| 988 | stderr_read) |
| 989 | try: |
Allen Li | ad719c1 | 2017-06-27 23:48:04 +0000 | [diff] [blame] | 990 | result = self.host.run(monitor_cmd, ignore_status=True, |
| 991 | timeout=timeout, |
| 992 | stdout_tee=client_log, |
| 993 | stderr_tee=stderr_redirector) |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 994 | except error.AutoservRunError, e: |
| 995 | result = e.result_obj |
| 996 | result.exit_status = None |
jadmanski | 91d56a9 | 2009-04-01 15:20:40 +0000 | [diff] [blame] | 997 | disconnect_warnings.append(e.description) |
| 998 | |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 999 | stderr_redirector.log_warning( |
jadmanski | 91d56a9 | 2009-04-01 15:20:40 +0000 | [diff] [blame] | 1000 | "Autotest client was disconnected: %s" % e.description, |
| 1001 | "NETWORK") |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 1002 | except error.AutoservSSHTimeout: |
| 1003 | result = utils.CmdResult(monitor_cmd, "", "", None, 0) |
| 1004 | stderr_redirector.log_warning( |
jadmanski | 91d56a9 | 2009-04-01 15:20:40 +0000 | [diff] [blame] | 1005 | "Attempt to connect to Autotest client timed out", |
| 1006 | "NETWORK") |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 1007 | |
| 1008 | stdout_read += len(result.stdout) |
| 1009 | stderr_read += len(self._strip_stderr_prologue(result.stderr)) |
| 1010 | |
| 1011 | if result.exit_status is not None: |
Simran Basi | bca10a6 | 2013-01-24 15:52:35 -0800 | [diff] [blame] | 1012 | # TODO (crosbug.com/38224)- sbasi: Remove extra logging. |
| 1013 | logging.debug('Result exit status is %d.', |
| 1014 | result.exit_status) |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 1015 | return result |
| 1016 | elif not self.host.wait_up(client_disconnect_timeout): |
| 1017 | raise error.AutoservSSHTimeout( |
| 1018 | "client was disconnected, reconnect timed out") |
| 1019 | finally: |
jadmanski | 4d03cf6 | 2010-03-04 18:32:28 +0000 | [diff] [blame] | 1020 | client_log.close() |
mbligh | 0d0f67d | 2009-11-06 03:15:03 +0000 | [diff] [blame] | 1021 | self.host.job.pop_execution_context() |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 1022 | |
| 1023 | |
Prathmesh Prabhu | cb4f549 | 2017-07-26 18:57:21 +0000 | [diff] [blame] | 1024 | def execute_section(self, section, timeout, stderr_redirector, |
| 1025 | client_disconnect_timeout): |
| 1026 | # TODO(crbug.com/684311) The claim is that section is never more than 0 |
| 1027 | # in pratice. After validating for a week or so, delete all support of |
| 1028 | # multiple sections. |
| 1029 | metrics.Counter('chromeos/autotest/autotest/sections').increment( |
| 1030 | fields={'is_first_section': (section == 0)}) |
| 1031 | logging.info("Executing %s/bin/autotest %s/control phase %d", |
| 1032 | self.autodir, self.autodir, section) |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 1033 | |
Prathmesh Prabhu | beaee90 | 2017-07-26 11:58:44 -0700 | [diff] [blame] | 1034 | if self.background: |
| 1035 | result = self._execute_in_background(section, timeout) |
| 1036 | else: |
| 1037 | result = self._execute_daemon(section, timeout, stderr_redirector, |
| 1038 | client_disconnect_timeout) |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 1039 | |
| 1040 | last_line = stderr_redirector.last_line |
mbligh | 2bf2db6 | 2007-11-27 00:53:18 +0000 | [diff] [blame] | 1041 | |
jadmanski | b264ed0 | 2009-01-12 23:54:27 +0000 | [diff] [blame] | 1042 | # check if we failed hard enough to warrant an exception |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1043 | if result.exit_status == 1: |
jadmanski | b264ed0 | 2009-01-12 23:54:27 +0000 | [diff] [blame] | 1044 | err = error.AutotestRunError("client job was aborted") |
Prathmesh Prabhu | beaee90 | 2017-07-26 11:58:44 -0700 | [diff] [blame] | 1045 | elif not self.background and not result.stderr: |
jadmanski | b264ed0 | 2009-01-12 23:54:27 +0000 | [diff] [blame] | 1046 | err = error.AutotestRunError( |
Prathmesh Prabhu | cb4f549 | 2017-07-26 18:57:21 +0000 | [diff] [blame] | 1047 | "execute_section %s failed to return anything\n" |
| 1048 | "stdout:%s\n" % (section, result.stdout)) |
jadmanski | b264ed0 | 2009-01-12 23:54:27 +0000 | [diff] [blame] | 1049 | else: |
| 1050 | err = None |
mbligh | 0e4613b | 2007-10-29 16:55:07 +0000 | [diff] [blame] | 1051 | |
jadmanski | b264ed0 | 2009-01-12 23:54:27 +0000 | [diff] [blame] | 1052 | # log something if the client failed AND never finished logging |
Dale Curtis | 9285ddf | 2011-01-05 11:47:24 -0800 | [diff] [blame] | 1053 | if err and not self.is_client_job_finished(last_line): |
jadmanski | a61edad | 2009-05-21 22:17:49 +0000 | [diff] [blame] | 1054 | self.log_unexpected_abort(stderr_redirector) |
jadmanski | b264ed0 | 2009-01-12 23:54:27 +0000 | [diff] [blame] | 1055 | |
| 1056 | if err: |
| 1057 | raise err |
| 1058 | else: |
| 1059 | return stderr_redirector.last_line |
jadmanski | 4600e34 | 2008-10-29 22:54:00 +0000 | [diff] [blame] | 1060 | |
| 1061 | |
Prathmesh Prabhu | cb4f549 | 2017-07-26 18:57:21 +0000 | [diff] [blame] | 1062 | def _wait_for_reboot(self, old_boot_id): |
| 1063 | logging.info("Client is rebooting") |
| 1064 | logging.info("Waiting for client to halt") |
| 1065 | if not self.host.wait_down(self.host.WAIT_DOWN_REBOOT_TIMEOUT, |
| 1066 | old_boot_id=old_boot_id): |
| 1067 | err = "%s failed to shutdown after %d" |
| 1068 | err %= (self.host.hostname, self.host.WAIT_DOWN_REBOOT_TIMEOUT) |
| 1069 | raise error.AutotestRunError(err) |
| 1070 | logging.info("Client down, waiting for restart") |
| 1071 | if not self.host.wait_up(self.host.DEFAULT_REBOOT_TIMEOUT): |
| 1072 | # since reboot failed |
| 1073 | # hardreset the machine once if possible |
| 1074 | # before failing this control file |
| 1075 | warning = "%s did not come back up, hard resetting" |
| 1076 | warning %= self.host.hostname |
| 1077 | logging.warning(warning) |
| 1078 | try: |
| 1079 | self.host.hardreset(wait=False) |
| 1080 | except (AttributeError, error.AutoservUnsupportedError): |
| 1081 | warning = "Hard reset unsupported on %s" |
| 1082 | warning %= self.host.hostname |
| 1083 | logging.warning(warning) |
| 1084 | raise error.AutotestRunError("%s failed to boot after %ds" % |
| 1085 | (self.host.hostname, |
| 1086 | self.host.DEFAULT_REBOOT_TIMEOUT)) |
| 1087 | self.host.reboot_followup() |
| 1088 | |
| 1089 | |
Dale Curtis | 9285ddf | 2011-01-05 11:47:24 -0800 | [diff] [blame] | 1090 | def execute_control(self, timeout=None, client_disconnect_timeout=None): |
Prathmesh Prabhu | beaee90 | 2017-07-26 11:58:44 -0700 | [diff] [blame] | 1091 | if not self.background: |
| 1092 | collector = log_collector(self.host, self.tag, self.results_dir) |
| 1093 | hostname = self.host.hostname |
| 1094 | remote_results = collector.client_results_dir |
| 1095 | local_results = collector.server_results_dir |
| 1096 | self.host.job.add_client_log(hostname, remote_results, |
| 1097 | local_results) |
| 1098 | job_record_context = self.host.job.get_record_context() |
Prathmesh Prabhu | cb4f549 | 2017-07-26 18:57:21 +0000 | [diff] [blame] | 1099 | |
| 1100 | section = 0 |
| 1101 | start_time = time.time() |
| 1102 | |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 1103 | logger = client_logger(self.host, self.tag, self.results_dir) |
jadmanski | 4600e34 | 2008-10-29 22:54:00 +0000 | [diff] [blame] | 1104 | try: |
Prathmesh Prabhu | cb4f549 | 2017-07-26 18:57:21 +0000 | [diff] [blame] | 1105 | while not timeout or time.time() < start_time + timeout: |
| 1106 | if timeout: |
| 1107 | section_timeout = start_time + timeout - time.time() |
| 1108 | else: |
| 1109 | section_timeout = None |
| 1110 | boot_id = self.host.get_boot_id() |
| 1111 | last = self.execute_section(section, section_timeout, |
| 1112 | logger, client_disconnect_timeout) |
Prathmesh Prabhu | beaee90 | 2017-07-26 11:58:44 -0700 | [diff] [blame] | 1113 | if self.background: |
| 1114 | return |
Prathmesh Prabhu | cb4f549 | 2017-07-26 18:57:21 +0000 | [diff] [blame] | 1115 | section += 1 |
| 1116 | if self.is_client_job_finished(last): |
| 1117 | logging.info("Client complete") |
| 1118 | return |
| 1119 | elif self.is_client_job_rebooting(last): |
| 1120 | try: |
| 1121 | self._wait_for_reboot(boot_id) |
| 1122 | except error.AutotestRunError, e: |
| 1123 | self.host.job.record("ABORT", None, "reboot", str(e)) |
| 1124 | self.host.job.record("END ABORT", None, None, str(e)) |
| 1125 | raise |
| 1126 | continue |
jadmanski | 4600e34 | 2008-10-29 22:54:00 +0000 | [diff] [blame] | 1127 | |
Prathmesh Prabhu | cb4f549 | 2017-07-26 18:57:21 +0000 | [diff] [blame] | 1128 | # If a test fails without probable cause we try to bucket it's |
| 1129 | # failure into one of 2 categories. If we can determine the |
| 1130 | # current state of the device and it is suspicious, we close the |
| 1131 | # status lines indicating a failure. If we either cannot |
| 1132 | # determine the state of the device, or it appears totally |
| 1133 | # healthy, we give up and abort. |
| 1134 | try: |
| 1135 | self._diagnose_dut(boot_id) |
| 1136 | except AutotestDeviceError as e: |
| 1137 | # The status lines of the test are pretty much tailed to |
| 1138 | # our log, with indentation, from the client job on the DUT. |
| 1139 | # So if the DUT goes down unexpectedly we'll end up with a |
| 1140 | # malformed status log unless we manually unwind the status |
| 1141 | # stack. Ideally we would want to write a nice wrapper like |
| 1142 | # server_job methods run_reboot, run_group but they expect |
| 1143 | # reboots and we don't. |
| 1144 | self.host.job.record('FAIL', None, None, str(e)) |
| 1145 | self.host.job.record('END FAIL', None, None) |
| 1146 | self.host.job.record('END GOOD', None, None) |
| 1147 | self.host.job.failed_with_device_error = True |
| 1148 | return |
| 1149 | except AutotestAbort as e: |
| 1150 | self.host.job.record('ABORT', None, None, str(e)) |
| 1151 | self.host.job.record('END ABORT', None, None) |
jadmanski | 4600e34 | 2008-10-29 22:54:00 +0000 | [diff] [blame] | 1152 | |
Prathmesh Prabhu | cb4f549 | 2017-07-26 18:57:21 +0000 | [diff] [blame] | 1153 | # give the client machine a chance to recover from a crash |
| 1154 | self.host.wait_up( |
| 1155 | self.host.HOURS_TO_WAIT_FOR_RECOVERY * 3600) |
| 1156 | logging.debug('Unexpected final status message from ' |
| 1157 | 'client %s: %s', self.host.hostname, last) |
| 1158 | # The line 'last' may have sensitive phrases, like |
| 1159 | # 'END GOOD', which breaks the tko parser. So the error |
| 1160 | # message will exclude it, since it will be recorded to |
| 1161 | # status.log. |
| 1162 | msg = ("Aborting - unexpected final status message from " |
| 1163 | "client on %s\n") % self.host.hostname |
| 1164 | raise error.AutotestRunError(msg) |
jadmanski | 4600e34 | 2008-10-29 22:54:00 +0000 | [diff] [blame] | 1165 | finally: |
xixuan | 02b6fee | 2017-02-01 18:35:20 -0800 | [diff] [blame] | 1166 | logging.debug('Autotest job finishes running. Below is the ' |
| 1167 | 'post-processing operations.') |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 1168 | logger.close() |
Prathmesh Prabhu | beaee90 | 2017-07-26 11:58:44 -0700 | [diff] [blame] | 1169 | if not self.background: |
| 1170 | collector.collect_client_job_results() |
| 1171 | collector.remove_redundant_client_logs() |
| 1172 | state_file = os.path.basename(self.remote_control_file |
| 1173 | + '.state') |
| 1174 | state_path = os.path.join(self.results_dir, state_file) |
| 1175 | self.host.job.postprocess_client_state(state_path) |
| 1176 | self.host.job.remove_client_log(hostname, remote_results, |
| 1177 | local_results) |
| 1178 | job_record_context.restore() |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 1179 | |
xixuan | 02b6fee | 2017-02-01 18:35:20 -0800 | [diff] [blame] | 1180 | logging.debug('Autotest job finishes.') |
| 1181 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1182 | # should only get here if we timed out |
| 1183 | assert timeout |
| 1184 | raise error.AutotestTimeoutError() |
mbligh | 0e4613b | 2007-10-29 16:55:07 +0000 | [diff] [blame] | 1185 | |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 1186 | |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 1187 | class log_collector(object): |
| 1188 | def __init__(self, host, client_tag, results_dir): |
| 1189 | self.host = host |
| 1190 | if not client_tag: |
| 1191 | client_tag = "default" |
| 1192 | self.client_results_dir = os.path.join(host.get_autodir(), "results", |
| 1193 | client_tag) |
| 1194 | self.server_results_dir = results_dir |
| 1195 | |
| 1196 | |
| 1197 | def collect_client_job_results(self): |
| 1198 | """ A method that collects all the current results of a running |
| 1199 | client job into the results dir. By default does nothing as no |
| 1200 | client job is running, but when running a client job you can override |
| 1201 | this with something that will actually do something. """ |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 1202 | # make an effort to wait for the machine to come up |
| 1203 | try: |
| 1204 | self.host.wait_up(timeout=30) |
| 1205 | except error.AutoservError: |
| 1206 | # don't worry about any errors, we'll try and |
| 1207 | # get the results anyway |
| 1208 | pass |
| 1209 | |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 1210 | # Copy all dirs in default to results_dir |
| 1211 | try: |
Dan Shi | 010c0bc | 2017-06-21 17:02:51 -0700 | [diff] [blame] | 1212 | # Build test result directory summary |
| 1213 | result_tools_runner.run_on_client( |
Dan Shi | c958a3d | 2017-07-06 14:43:18 -0700 | [diff] [blame] | 1214 | self.host, self.client_results_dir) |
Dan Shi | 9f879fb | 2017-05-26 15:44:04 -0700 | [diff] [blame] | 1215 | |
| 1216 | with metrics.SecondsTimer( |
Prathmesh Prabhu | 7cc1153 | 2016-11-23 17:51:08 -0800 | [diff] [blame] | 1217 | 'chromeos/autotest/job/log_collection_duration', |
| 1218 | fields={'dut_host_name': self.host.hostname}): |
| 1219 | self.host.get_file( |
| 1220 | self.client_results_dir + '/', |
| 1221 | self.server_results_dir, |
| 1222 | preserve_symlinks=True) |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 1223 | except Exception: |
| 1224 | # well, don't stop running just because we couldn't get logs |
showard | b18134f | 2009-03-20 20:52:18 +0000 | [diff] [blame] | 1225 | e_msg = "Unexpected error copying test result logs, continuing ..." |
| 1226 | logging.error(e_msg) |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 1227 | traceback.print_exc(file=sys.stdout) |
| 1228 | |
| 1229 | |
jadmanski | 4d03cf6 | 2010-03-04 18:32:28 +0000 | [diff] [blame] | 1230 | def remove_redundant_client_logs(self): |
| 1231 | """Remove client.*.log files in favour of client.*.DEBUG files.""" |
| 1232 | debug_dir = os.path.join(self.server_results_dir, 'debug') |
| 1233 | debug_files = [f for f in os.listdir(debug_dir) |
| 1234 | if re.search(r'^client\.\d+\.DEBUG$', f)] |
| 1235 | for debug_file in debug_files: |
| 1236 | log_file = debug_file.replace('DEBUG', 'log') |
| 1237 | log_file = os.path.join(debug_dir, log_file) |
| 1238 | if os.path.exists(log_file): |
| 1239 | os.remove(log_file) |
| 1240 | |
| 1241 | |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 1242 | # a file-like object for catching stderr from an autotest client and |
| 1243 | # extracting status logs from it |
Allen Li | 944ac46 | 2017-02-07 15:57:20 -0800 | [diff] [blame] | 1244 | class client_logger(object): |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 1245 | """Partial file object to write to both stdout and |
| 1246 | the status log file. We only implement those methods |
| 1247 | utils.run() actually calls. |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 1248 | """ |
| 1249 | status_parser = re.compile(r"^AUTOTEST_STATUS:([^:]*):(.*)$") |
| 1250 | test_complete_parser = re.compile(r"^AUTOTEST_TEST_COMPLETE:(.*)$") |
jadmanski | b1a5113 | 2009-08-07 16:45:50 +0000 | [diff] [blame] | 1251 | fetch_package_parser = re.compile( |
| 1252 | r"^AUTOTEST_FETCH_PACKAGE:([^:]*):([^:]*):(.*)$") |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 1253 | extract_indent = re.compile(r"^(\t*).*$") |
jadmanski | efe4ebf | 2009-05-21 22:12:30 +0000 | [diff] [blame] | 1254 | extract_timestamp = re.compile(r".*\ttimestamp=(\d+)\t.*$") |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 1255 | |
| 1256 | def __init__(self, host, tag, server_results_dir): |
| 1257 | self.host = host |
| 1258 | self.job = host.job |
| 1259 | self.log_collector = log_collector(host, tag, server_results_dir) |
| 1260 | self.leftover = "" |
| 1261 | self.last_line = "" |
| 1262 | self.logs = {} |
jadmanski | efe4ebf | 2009-05-21 22:12:30 +0000 | [diff] [blame] | 1263 | |
| 1264 | |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 1265 | def _process_log_dict(self, log_dict): |
| 1266 | log_list = log_dict.pop("logs", []) |
| 1267 | for key in sorted(log_dict.iterkeys()): |
| 1268 | log_list += self._process_log_dict(log_dict.pop(key)) |
| 1269 | return log_list |
| 1270 | |
| 1271 | |
| 1272 | def _process_logs(self): |
| 1273 | """Go through the accumulated logs in self.log and print them |
| 1274 | out to stdout and the status log. Note that this processes |
| 1275 | logs in an ordering where: |
| 1276 | |
| 1277 | 1) logs to different tags are never interleaved |
| 1278 | 2) logs to x.y come before logs to x.y.z for all z |
| 1279 | 3) logs to x.y come before x.z whenever y < z |
| 1280 | |
| 1281 | Note that this will in general not be the same as the |
| 1282 | chronological ordering of the logs. However, if a chronological |
| 1283 | ordering is desired that one can be reconstructed from the |
| 1284 | status log by looking at timestamp lines.""" |
| 1285 | log_list = self._process_log_dict(self.logs) |
jadmanski | 2a89dac | 2010-06-11 14:32:58 +0000 | [diff] [blame] | 1286 | for entry in log_list: |
| 1287 | self.job.record_entry(entry, log_in_subdir=False) |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 1288 | if log_list: |
jadmanski | 2a89dac | 2010-06-11 14:32:58 +0000 | [diff] [blame] | 1289 | self.last_line = log_list[-1].render() |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 1290 | |
| 1291 | |
| 1292 | def _process_quoted_line(self, tag, line): |
| 1293 | """Process a line quoted with an AUTOTEST_STATUS flag. If the |
| 1294 | tag is blank then we want to push out all the data we've been |
| 1295 | building up in self.logs, and then the newest line. If the |
| 1296 | tag is not blank, then push the line into the logs for handling |
| 1297 | later.""" |
jadmanski | 2a89dac | 2010-06-11 14:32:58 +0000 | [diff] [blame] | 1298 | entry = base_job.status_log_entry.parse(line) |
| 1299 | if entry is None: |
| 1300 | return # the line contains no status lines |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 1301 | if tag == "": |
| 1302 | self._process_logs() |
jadmanski | 2a89dac | 2010-06-11 14:32:58 +0000 | [diff] [blame] | 1303 | self.job.record_entry(entry, log_in_subdir=False) |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 1304 | self.last_line = line |
| 1305 | else: |
| 1306 | tag_parts = [int(x) for x in tag.split(".")] |
| 1307 | log_dict = self.logs |
| 1308 | for part in tag_parts: |
| 1309 | log_dict = log_dict.setdefault(part, {}) |
| 1310 | log_list = log_dict.setdefault("logs", []) |
jadmanski | 2a89dac | 2010-06-11 14:32:58 +0000 | [diff] [blame] | 1311 | log_list.append(entry) |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 1312 | |
| 1313 | |
jadmanski | f37df84 | 2009-02-11 00:03:26 +0000 | [diff] [blame] | 1314 | def _process_info_line(self, line): |
| 1315 | """Check if line is an INFO line, and if it is, interpret any control |
| 1316 | messages (e.g. enabling/disabling warnings) that it may contain.""" |
| 1317 | match = re.search(r"^\t*INFO\t----\t----(.*)\t[^\t]*$", line) |
| 1318 | if not match: |
| 1319 | return # not an INFO line |
| 1320 | for field in match.group(1).split('\t'): |
| 1321 | if field.startswith("warnings.enable="): |
jadmanski | 16a7ff7 | 2009-04-01 18:19:53 +0000 | [diff] [blame] | 1322 | func = self.job.warning_manager.enable_warnings |
jadmanski | f37df84 | 2009-02-11 00:03:26 +0000 | [diff] [blame] | 1323 | elif field.startswith("warnings.disable="): |
jadmanski | 16a7ff7 | 2009-04-01 18:19:53 +0000 | [diff] [blame] | 1324 | func = self.job.warning_manager.disable_warnings |
jadmanski | f37df84 | 2009-02-11 00:03:26 +0000 | [diff] [blame] | 1325 | else: |
| 1326 | continue |
| 1327 | warning_type = field.split("=", 1)[1] |
jadmanski | 16a7ff7 | 2009-04-01 18:19:53 +0000 | [diff] [blame] | 1328 | func(warning_type) |
jadmanski | f37df84 | 2009-02-11 00:03:26 +0000 | [diff] [blame] | 1329 | |
| 1330 | |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 1331 | def _process_line(self, line): |
Allen Li | 944ac46 | 2017-02-07 15:57:20 -0800 | [diff] [blame] | 1332 | """Write out a line of data to the appropriate stream. |
| 1333 | |
| 1334 | Returns the package checksum file if it exists. |
| 1335 | |
| 1336 | Status lines sent by autotest will be prepended with |
| 1337 | "AUTOTEST_STATUS", and all other lines are ssh error messages. |
| 1338 | """ |
| 1339 | logging.debug(line) |
| 1340 | fetch_package_match = self.fetch_package_parser.search(line) |
| 1341 | if fetch_package_match: |
| 1342 | pkg_name, dest_path, fifo_path = fetch_package_match.groups() |
| 1343 | serve_packages = _CONFIG.get_config_value( |
| 1344 | "PACKAGES", "serve_packages_from_autoserv", type=bool) |
| 1345 | if serve_packages and pkg_name == 'packages.checksum': |
| 1346 | try: |
| 1347 | checksum_file = os.path.join( |
| 1348 | self.job.pkgmgr.pkgmgr_dir, 'packages', pkg_name) |
| 1349 | if os.path.exists(checksum_file): |
| 1350 | self.host.send_file(checksum_file, dest_path) |
| 1351 | except error.AutoservRunError: |
| 1352 | msg = "Package checksum file not found, continuing anyway" |
| 1353 | logging.exception(msg) |
| 1354 | |
| 1355 | try: |
| 1356 | # When fetching a package, the client expects to be |
| 1357 | # notified when the fetching is complete. Autotest |
| 1358 | # does this pushing a B to a fifo queue to the client. |
| 1359 | self.host.run("echo B > %s" % fifo_path) |
| 1360 | except error.AutoservRunError: |
| 1361 | msg = "Checksum installation failed, continuing anyway" |
| 1362 | logging.exception(msg) |
| 1363 | finally: |
| 1364 | return |
| 1365 | |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 1366 | status_match = self.status_parser.search(line) |
| 1367 | test_complete_match = self.test_complete_parser.search(line) |
jadmanski | b1a5113 | 2009-08-07 16:45:50 +0000 | [diff] [blame] | 1368 | fetch_package_match = self.fetch_package_parser.search(line) |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 1369 | if status_match: |
| 1370 | tag, line = status_match.groups() |
jadmanski | f37df84 | 2009-02-11 00:03:26 +0000 | [diff] [blame] | 1371 | self._process_info_line(line) |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 1372 | self._process_quoted_line(tag, line) |
| 1373 | elif test_complete_match: |
jadmanski | fcc0d5d | 2009-02-12 21:52:54 +0000 | [diff] [blame] | 1374 | self._process_logs() |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 1375 | fifo_path, = test_complete_match.groups() |
mbligh | 060c471 | 2009-12-29 02:43:35 +0000 | [diff] [blame] | 1376 | try: |
| 1377 | self.log_collector.collect_client_job_results() |
Allen Li | ad719c1 | 2017-06-27 23:48:04 +0000 | [diff] [blame] | 1378 | self.host.run("echo A > %s" % fifo_path) |
mbligh | 060c471 | 2009-12-29 02:43:35 +0000 | [diff] [blame] | 1379 | except Exception: |
| 1380 | msg = "Post-test log collection failed, continuing anyway" |
| 1381 | logging.exception(msg) |
jadmanski | b1a5113 | 2009-08-07 16:45:50 +0000 | [diff] [blame] | 1382 | elif fetch_package_match: |
| 1383 | pkg_name, dest_path, fifo_path = fetch_package_match.groups() |
jadmanski | ede7e24 | 2009-08-10 15:43:33 +0000 | [diff] [blame] | 1384 | serve_packages = global_config.global_config.get_config_value( |
| 1385 | "PACKAGES", "serve_packages_from_autoserv", type=bool) |
| 1386 | if serve_packages and pkg_name.endswith(".tar.bz2"): |
| 1387 | try: |
| 1388 | self._send_tarball(pkg_name, dest_path) |
| 1389 | except Exception: |
| 1390 | msg = "Package tarball creation failed, continuing anyway" |
| 1391 | logging.exception(msg) |
mbligh | 060c471 | 2009-12-29 02:43:35 +0000 | [diff] [blame] | 1392 | try: |
Allen Li | ad719c1 | 2017-06-27 23:48:04 +0000 | [diff] [blame] | 1393 | self.host.run("echo B > %s" % fifo_path) |
mbligh | 060c471 | 2009-12-29 02:43:35 +0000 | [diff] [blame] | 1394 | except Exception: |
| 1395 | msg = "Package tarball installation failed, continuing anyway" |
| 1396 | logging.exception(msg) |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 1397 | else: |
showard | b18134f | 2009-03-20 20:52:18 +0000 | [diff] [blame] | 1398 | logging.info(line) |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 1399 | |
| 1400 | |
jadmanski | ede7e24 | 2009-08-10 15:43:33 +0000 | [diff] [blame] | 1401 | def _send_tarball(self, pkg_name, remote_dest): |
Allen Li | 944ac46 | 2017-02-07 15:57:20 -0800 | [diff] [blame] | 1402 | """Uses tarballs in package manager by default.""" |
| 1403 | try: |
| 1404 | server_package = os.path.join(self.job.pkgmgr.pkgmgr_dir, |
| 1405 | 'packages', pkg_name) |
| 1406 | if os.path.exists(server_package): |
| 1407 | self.host.send_file(server_package, remote_dest) |
| 1408 | return |
| 1409 | |
| 1410 | except error.AutoservRunError: |
| 1411 | msg = ("Package %s could not be sent from the package cache." % |
| 1412 | pkg_name) |
| 1413 | logging.exception(msg) |
| 1414 | |
jadmanski | ede7e24 | 2009-08-10 15:43:33 +0000 | [diff] [blame] | 1415 | name, pkg_type = self.job.pkgmgr.parse_tarball_name(pkg_name) |
| 1416 | src_dirs = [] |
| 1417 | if pkg_type == 'test': |
mbligh | 1f572e5 | 2010-04-01 17:15:53 +0000 | [diff] [blame] | 1418 | for test_dir in ['site_tests', 'tests']: |
| 1419 | src_dir = os.path.join(self.job.clientdir, test_dir, name) |
| 1420 | if os.path.exists(src_dir): |
| 1421 | src_dirs += [src_dir] |
mbligh | 1f572e5 | 2010-04-01 17:15:53 +0000 | [diff] [blame] | 1422 | break |
jadmanski | ede7e24 | 2009-08-10 15:43:33 +0000 | [diff] [blame] | 1423 | elif pkg_type == 'profiler': |
| 1424 | src_dirs += [os.path.join(self.job.clientdir, 'profilers', name)] |
| 1425 | elif pkg_type == 'dep': |
| 1426 | src_dirs += [os.path.join(self.job.clientdir, 'deps', name)] |
| 1427 | elif pkg_type == 'client': |
| 1428 | return # you must already have a client to hit this anyway |
| 1429 | else: |
| 1430 | return # no other types are supported |
| 1431 | |
| 1432 | # iterate over src_dirs until we find one that exists, then tar it |
| 1433 | for src_dir in src_dirs: |
| 1434 | if os.path.exists(src_dir): |
| 1435 | try: |
| 1436 | logging.info('Bundling %s into %s', src_dir, pkg_name) |
| 1437 | temp_dir = autotemp.tempdir(unique_id='autoserv-packager', |
| 1438 | dir=self.job.tmpdir) |
| 1439 | tarball_path = self.job.pkgmgr.tar_package( |
mbligh | bccad48 | 2009-08-24 22:08:31 +0000 | [diff] [blame] | 1440 | pkg_name, src_dir, temp_dir.name, " .") |
jadmanski | ede7e24 | 2009-08-10 15:43:33 +0000 | [diff] [blame] | 1441 | self.host.send_file(tarball_path, remote_dest) |
| 1442 | finally: |
| 1443 | temp_dir.clean() |
| 1444 | return |
| 1445 | |
| 1446 | |
jadmanski | 91d56a9 | 2009-04-01 15:20:40 +0000 | [diff] [blame] | 1447 | def log_warning(self, msg, warning_type): |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 1448 | """Injects a WARN message into the current status logging stream.""" |
jadmanski | 91d56a9 | 2009-04-01 15:20:40 +0000 | [diff] [blame] | 1449 | timestamp = int(time.time()) |
| 1450 | if self.job.warning_manager.is_valid(timestamp, warning_type): |
Eric Li | d656d56 | 2011-04-20 11:48:29 -0700 | [diff] [blame] | 1451 | self.job.record('WARN', None, None, msg) |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 1452 | |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 1453 | |
| 1454 | def write(self, data): |
jadmanski | 2a89dac | 2010-06-11 14:32:58 +0000 | [diff] [blame] | 1455 | # now start processing the existing buffer and the new data |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 1456 | data = self.leftover + data |
mbligh | 060c471 | 2009-12-29 02:43:35 +0000 | [diff] [blame] | 1457 | lines = data.split('\n') |
| 1458 | processed_lines = 0 |
| 1459 | try: |
| 1460 | # process all the buffered data except the last line |
| 1461 | # ignore the last line since we may not have all of it yet |
| 1462 | for line in lines[:-1]: |
mbligh | 060c471 | 2009-12-29 02:43:35 +0000 | [diff] [blame] | 1463 | self._process_line(line) |
| 1464 | processed_lines += 1 |
| 1465 | finally: |
| 1466 | # save any unprocessed lines for future processing |
| 1467 | self.leftover = '\n'.join(lines[processed_lines:]) |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 1468 | |
| 1469 | |
| 1470 | def flush(self): |
| 1471 | sys.stdout.flush() |
| 1472 | |
| 1473 | |
jadmanski | a61edad | 2009-05-21 22:17:49 +0000 | [diff] [blame] | 1474 | def flush_all_buffers(self): |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 1475 | if self.leftover: |
| 1476 | self._process_line(self.leftover) |
jadmanski | a61edad | 2009-05-21 22:17:49 +0000 | [diff] [blame] | 1477 | self.leftover = "" |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 1478 | self._process_logs() |
| 1479 | self.flush() |
| 1480 | |
| 1481 | |
jadmanski | a61edad | 2009-05-21 22:17:49 +0000 | [diff] [blame] | 1482 | def close(self): |
| 1483 | self.flush_all_buffers() |