mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 1 | # Copyright 2007 Google Inc. Released under the GPL v2 |
| 2 | |
showard | b18134f | 2009-03-20 20:52:18 +0000 | [diff] [blame^] | 3 | import re, os, sys, traceback, subprocess, tempfile, time, pickle, glob, logging |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 4 | from autotest_lib.server import installable_object, utils |
showard | b18134f | 2009-03-20 20:52:18 +0000 | [diff] [blame^] | 5 | from autotest_lib.client.common_lib import log, error |
mbligh | 0910844 | 2008-10-15 16:27:38 +0000 | [diff] [blame] | 6 | from autotest_lib.client.common_lib import global_config, packages |
mbligh | a700772 | 2009-01-13 00:37:11 +0000 | [diff] [blame] | 7 | from autotest_lib.client.common_lib import utils as client_utils |
mbligh | 3c7a150 | 2008-07-24 18:08:47 +0000 | [diff] [blame] | 8 | |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 9 | AUTOTEST_SVN = 'svn://test.kernel.org/autotest/trunk/client' |
| 10 | AUTOTEST_HTTP = 'http://test.kernel.org/svn/autotest/trunk/client' |
| 11 | |
| 12 | # Timeouts for powering down and up respectively |
| 13 | HALT_TIME = 300 |
mbligh | 07c1eac | 2007-11-05 18:39:29 +0000 | [diff] [blame] | 14 | BOOT_TIME = 1800 |
jadmanski | ec85914 | 2008-05-29 21:33:39 +0000 | [diff] [blame] | 15 | CRASH_RECOVERY_TIME = 9000 |
mbligh | 0e4613b | 2007-10-29 16:55:07 +0000 | [diff] [blame] | 16 | |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 17 | |
mbligh | d8b3925 | 2008-03-20 21:15:03 +0000 | [diff] [blame] | 18 | class BaseAutotest(installable_object.InstallableObject): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 19 | """ |
| 20 | This class represents the Autotest program. |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 21 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 22 | Autotest is used to run tests automatically and collect the results. |
| 23 | It also supports profilers. |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 24 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 25 | Implementation details: |
| 26 | This is a leaf class in an abstract class hierarchy, it must |
| 27 | implement the unimplemented methods in parent classes. |
| 28 | """ |
mbligh | 119c12a | 2007-11-12 22:13:44 +0000 | [diff] [blame] | 29 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 30 | def __init__(self, host = None): |
| 31 | self.host = host |
| 32 | self.got = False |
| 33 | self.installed = False |
jadmanski | e2eef7b | 2009-03-03 23:55:13 +0000 | [diff] [blame] | 34 | self.lightweight = False |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 35 | self.serverdir = utils.get_server_dir() |
| 36 | super(BaseAutotest, self).__init__() |
mbligh | c8949b8 | 2007-07-23 16:33:58 +0000 | [diff] [blame] | 37 | |
mbligh | dc735a2 | 2007-08-02 16:54:37 +0000 | [diff] [blame] | 38 | |
jadmanski | f22fea8 | 2008-11-26 20:57:07 +0000 | [diff] [blame] | 39 | install_in_tmpdir = False |
| 40 | @classmethod |
| 41 | def set_install_in_tmpdir(cls, flag): |
| 42 | """ Sets a flag that controls whether or not Autotest should by |
| 43 | default be installed in a "standard" directory (e.g. |
| 44 | /home/autotest, /usr/local/autotest) or a temporary directory. """ |
| 45 | cls.install_in_tmpdir = flag |
| 46 | |
| 47 | |
| 48 | def _get_install_dir(self, host): |
| 49 | """ Determines the location where autotest should be installed on |
| 50 | host. If self.install_in_tmpdir is set, it will return a unique |
| 51 | temporary directory that autotest can be installed in. """ |
| 52 | try: |
| 53 | autodir = _get_autodir(host) |
| 54 | except error.AutotestRunError: |
| 55 | autodir = '/usr/local/autotest' |
| 56 | if self.install_in_tmpdir: |
| 57 | autodir = host.get_tmp_dir(parent=autodir) |
| 58 | return autodir |
| 59 | |
| 60 | |
mbligh | 1b3b376 | 2008-09-25 02:46:34 +0000 | [diff] [blame] | 61 | @log.record |
mbligh | b3c0c91 | 2008-11-27 00:32:45 +0000 | [diff] [blame] | 62 | def install(self, host=None, autodir=None): |
| 63 | self._install(host=host, autodir=autodir) |
jadmanski | 54f90af | 2008-10-10 16:20:55 +0000 | [diff] [blame] | 64 | |
| 65 | |
| 66 | def install_base(self, host=None, autodir=None): |
| 67 | """ Performs a lightweight autotest install. Useful for when you |
| 68 | want to run some client-side code but don't want to pay the cost |
| 69 | of a full installation. """ |
| 70 | self._install(host=host, autodir=autodir, lightweight=True) |
| 71 | |
| 72 | |
| 73 | def _install(self, host=None, autodir=None, lightweight=False): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 74 | """ |
| 75 | Install autotest. If get() was not called previously, an |
| 76 | attempt will be made to install from the autotest svn |
| 77 | repository. |
mbligh | 9a3f5e5 | 2008-05-28 21:21:43 +0000 | [diff] [blame] | 78 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 79 | Args: |
jadmanski | 54f90af | 2008-10-10 16:20:55 +0000 | [diff] [blame] | 80 | host: a Host instance on which autotest will be installed |
| 81 | autodir: location on the remote host to install to |
| 82 | lightweight: exclude tests, deps and profilers, if possible |
mbligh | 9a3f5e5 | 2008-05-28 21:21:43 +0000 | [diff] [blame] | 83 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 84 | Raises: |
jadmanski | 54f90af | 2008-10-10 16:20:55 +0000 | [diff] [blame] | 85 | AutoservError: if a tarball was not specified and |
| 86 | the target host does not have svn installed in its path""" |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 87 | if not host: |
| 88 | host = self.host |
| 89 | if not self.got: |
| 90 | self.get() |
| 91 | host.wait_up(timeout=30) |
| 92 | host.setup() |
showard | b18134f | 2009-03-20 20:52:18 +0000 | [diff] [blame^] | 93 | logging.info("Installing autotest on %s", host.hostname) |
mbligh | 40f122a | 2007-11-03 23:08:46 +0000 | [diff] [blame] | 94 | |
jadmanski | 54f90af | 2008-10-10 16:20:55 +0000 | [diff] [blame] | 95 | # set up the autotest directory on the remote machine |
| 96 | if not autodir: |
jadmanski | f22fea8 | 2008-11-26 20:57:07 +0000 | [diff] [blame] | 97 | autodir = self._get_install_dir(host) |
mbligh | 0562e65 | 2008-08-20 20:11:45 +0000 | [diff] [blame] | 98 | host.set_autodir(autodir) |
jadmanski | 3c23694 | 2009-03-04 17:51:26 +0000 | [diff] [blame] | 99 | host.run('mkdir -p %s' % utils.sh_escape(autodir)) |
mbligh | 40f122a | 2007-11-03 23:08:46 +0000 | [diff] [blame] | 100 | |
jadmanski | 1c3c07b | 2009-03-03 23:29:36 +0000 | [diff] [blame] | 101 | # make sure there are no files in $AUTODIR/results |
| 102 | results_path = os.path.join(autodir, 'results') |
jadmanski | 3c23694 | 2009-03-04 17:51:26 +0000 | [diff] [blame] | 103 | host.run('rm -rf %s/*' % utils.sh_escape(results_path), |
jadmanski | 1c3c07b | 2009-03-03 23:29:36 +0000 | [diff] [blame] | 104 | ignore_status=True) |
| 105 | |
mbligh | c5ddfd1 | 2008-08-04 17:15:00 +0000 | [diff] [blame] | 106 | # Fetch the autotest client from the nearest repository |
| 107 | try: |
| 108 | c = global_config.global_config |
| 109 | repos = c.get_config_value("PACKAGES", 'fetch_location', type=list) |
mbligh | 76d19f7 | 2008-10-15 16:24:43 +0000 | [diff] [blame] | 110 | pkgmgr = packages.PackageManager(autodir, hostname=host.hostname, |
| 111 | repo_urls=repos, |
mbligh | 1e3b099 | 2008-10-14 16:29:54 +0000 | [diff] [blame] | 112 | do_locking=False, |
| 113 | run_function=host.run, |
| 114 | run_function_dargs=dict(timeout=600)) |
mbligh | c5ddfd1 | 2008-08-04 17:15:00 +0000 | [diff] [blame] | 115 | # The packages dir is used to store all the packages that |
| 116 | # are fetched on that client. (for the tests,deps etc. |
| 117 | # too apart from the client) |
| 118 | pkg_dir = os.path.join(autodir, 'packages') |
| 119 | # clean up the autodir except for the packages directory |
| 120 | host.run('cd %s && ls | grep -v "^packages$"' |
| 121 | ' | xargs rm -rf && rm -rf .[^.]*' % autodir) |
| 122 | pkgmgr.install_pkg('autotest', 'client', pkg_dir, autodir, |
| 123 | preserve_install_dir=True) |
| 124 | self.installed = True |
jadmanski | e2eef7b | 2009-03-03 23:55:13 +0000 | [diff] [blame] | 125 | self.lightweight = lightweight |
mbligh | c5ddfd1 | 2008-08-04 17:15:00 +0000 | [diff] [blame] | 126 | return |
| 127 | except global_config.ConfigError, e: |
showard | b18134f | 2009-03-20 20:52:18 +0000 | [diff] [blame^] | 128 | logging.error("Could not install autotest using the packaging" |
| 129 | "system: %s", e) |
mbligh | c5ddfd1 | 2008-08-04 17:15:00 +0000 | [diff] [blame] | 130 | except (packages.PackageInstallError, error.AutoservRunError), e: |
showard | b18134f | 2009-03-20 20:52:18 +0000 | [diff] [blame^] | 131 | logging.error("Could not install autotest from %s", repos) |
mbligh | c5ddfd1 | 2008-08-04 17:15:00 +0000 | [diff] [blame] | 132 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 133 | # try to install from file or directory |
| 134 | if self.source_material: |
| 135 | if os.path.isdir(self.source_material): |
| 136 | # Copy autotest recursively |
jadmanski | 54f90af | 2008-10-10 16:20:55 +0000 | [diff] [blame] | 137 | if lightweight: |
| 138 | dirs_to_exclude = set(["tests", "site_tests", "deps", |
| 139 | "tools", "profilers"]) |
| 140 | light_files = [os.path.join(self.source_material, f) |
| 141 | for f in os.listdir(self.source_material) |
| 142 | if f not in dirs_to_exclude] |
mbligh | 89e258d | 2008-10-24 13:58:08 +0000 | [diff] [blame] | 143 | host.send_file(light_files, autodir, delete_dest=True) |
jadmanski | 54f90af | 2008-10-10 16:20:55 +0000 | [diff] [blame] | 144 | |
| 145 | # create empty dirs for all the stuff we excluded |
| 146 | commands = [] |
| 147 | for path in dirs_to_exclude: |
| 148 | abs_path = os.path.join(autodir, path) |
| 149 | abs_path = utils.sh_escape(abs_path) |
| 150 | commands.append("mkdir -p '%s'" % abs_path) |
| 151 | host.run(';'.join(commands)) |
| 152 | else: |
mbligh | 89e258d | 2008-10-24 13:58:08 +0000 | [diff] [blame] | 153 | host.send_file(self.source_material, autodir, |
| 154 | delete_dest=True) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 155 | else: |
| 156 | # Copy autotest via tarball |
| 157 | e_msg = 'Installation method not yet implemented!' |
| 158 | raise NotImplementedError(e_msg) |
showard | b18134f | 2009-03-20 20:52:18 +0000 | [diff] [blame^] | 159 | logging.info("Installation of autotest completed") |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 160 | self.installed = True |
jadmanski | e2eef7b | 2009-03-03 23:55:13 +0000 | [diff] [blame] | 161 | self.lightweight = lightweight |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 162 | return |
mbligh | 9133490 | 2007-09-28 01:47:59 +0000 | [diff] [blame] | 163 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 164 | # if that fails try to install using svn |
| 165 | if utils.run('which svn').exit_status: |
mbligh | 78bf535 | 2008-07-11 20:27:36 +0000 | [diff] [blame] | 166 | raise error.AutoservError('svn not found on target machine: %s' |
| 167 | % host.name) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 168 | try: |
mbligh | 78bf535 | 2008-07-11 20:27:36 +0000 | [diff] [blame] | 169 | host.run('svn checkout %s %s' % (AUTOTEST_SVN, autodir)) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 170 | except error.AutoservRunError, e: |
mbligh | 78bf535 | 2008-07-11 20:27:36 +0000 | [diff] [blame] | 171 | host.run('svn checkout %s %s' % (AUTOTEST_HTTP, autodir)) |
showard | b18134f | 2009-03-20 20:52:18 +0000 | [diff] [blame^] | 172 | logging.info("Installation of autotest completed") |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 173 | self.installed = True |
jadmanski | e2eef7b | 2009-03-03 23:55:13 +0000 | [diff] [blame] | 174 | self.lightweight = lightweight |
mbligh | 9133490 | 2007-09-28 01:47:59 +0000 | [diff] [blame] | 175 | |
| 176 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 177 | def get(self, location = None): |
| 178 | if not location: |
| 179 | location = os.path.join(self.serverdir, '../client') |
| 180 | location = os.path.abspath(location) |
| 181 | # If there's stuff run on our client directory already, it |
| 182 | # can cause problems. Try giving it a quick clean first. |
| 183 | cwd = os.getcwd() |
| 184 | os.chdir(location) |
| 185 | os.system('tools/make_clean') |
| 186 | os.chdir(cwd) |
| 187 | super(BaseAutotest, self).get(location) |
| 188 | self.got = True |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 189 | |
| 190 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 191 | def run(self, control_file, results_dir = '.', host = None, |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 192 | timeout=None, tag=None, parallel_flag=False, background=False, |
| 193 | client_disconnect_timeout=1800): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 194 | """ |
| 195 | Run an autotest job on the remote machine. |
mbligh | 9a3f5e5 | 2008-05-28 21:21:43 +0000 | [diff] [blame] | 196 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 197 | Args: |
| 198 | control_file: an open file-like-obj of the control file |
| 199 | results_dir: a str path where the results should be stored |
| 200 | on the local filesystem |
| 201 | host: a Host instance on which the control file should |
| 202 | be run |
| 203 | tag: tag name for the client side instance of autotest |
| 204 | parallel_flag: flag set when multiple jobs are run at the |
| 205 | same time |
mbligh | b3c0c91 | 2008-11-27 00:32:45 +0000 | [diff] [blame] | 206 | background: indicates that the client should be launched as |
| 207 | a background job; the code calling run will be |
| 208 | responsible for monitoring the client and |
| 209 | collecting the results |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 210 | Raises: |
| 211 | AutotestRunError: if there is a problem executing |
| 212 | the control file |
| 213 | """ |
| 214 | host = self._get_host_and_setup(host) |
| 215 | results_dir = os.path.abspath(results_dir) |
mbligh | c1cbc99 | 2008-05-27 20:01:45 +0000 | [diff] [blame] | 216 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 217 | if tag: |
| 218 | results_dir = os.path.join(results_dir, tag) |
mbligh | c1cbc99 | 2008-05-27 20:01:45 +0000 | [diff] [blame] | 219 | |
mbligh | b3c0c91 | 2008-11-27 00:32:45 +0000 | [diff] [blame] | 220 | atrun = _Run(host, results_dir, tag, parallel_flag, background) |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 221 | self._do_run(control_file, results_dir, host, atrun, timeout, |
| 222 | client_disconnect_timeout) |
mbligh | d8b3925 | 2008-03-20 21:15:03 +0000 | [diff] [blame] | 223 | |
| 224 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 225 | def _get_host_and_setup(self, host): |
| 226 | if not host: |
| 227 | host = self.host |
| 228 | if not self.installed: |
| 229 | self.install(host) |
mbligh | 9133490 | 2007-09-28 01:47:59 +0000 | [diff] [blame] | 230 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 231 | host.wait_up(timeout=30) |
| 232 | return host |
mbligh | d8b3925 | 2008-03-20 21:15:03 +0000 | [diff] [blame] | 233 | |
| 234 | |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 235 | def _do_run(self, control_file, results_dir, host, atrun, timeout, |
| 236 | client_disconnect_timeout): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 237 | try: |
| 238 | atrun.verify_machine() |
| 239 | except: |
showard | b18134f | 2009-03-20 20:52:18 +0000 | [diff] [blame^] | 240 | logging.error("Verify failed on %s. Reinstalling autotest", |
| 241 | host.hostname) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 242 | self.install(host) |
| 243 | atrun.verify_machine() |
| 244 | debug = os.path.join(results_dir, 'debug') |
| 245 | try: |
| 246 | os.makedirs(debug) |
mbligh | 0910844 | 2008-10-15 16:27:38 +0000 | [diff] [blame] | 247 | except Exception: |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 248 | pass |
mbligh | 9a3f5e5 | 2008-05-28 21:21:43 +0000 | [diff] [blame] | 249 | |
mbligh | 0910844 | 2008-10-15 16:27:38 +0000 | [diff] [blame] | 250 | delete_file_list = [atrun.remote_control_file, |
| 251 | atrun.remote_control_file + '.state', |
| 252 | atrun.manual_control_file, |
| 253 | atrun.manual_control_file + '.state'] |
| 254 | cmd = ';'.join('rm -f ' + control for control in delete_file_list) |
| 255 | host.run(cmd, ignore_status=True) |
mbligh | 9a3f5e5 | 2008-05-28 21:21:43 +0000 | [diff] [blame] | 256 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 257 | tmppath = utils.get(control_file) |
mbligh | c5ddfd1 | 2008-08-04 17:15:00 +0000 | [diff] [blame] | 258 | |
jadmanski | cb0e161 | 2009-02-27 18:03:10 +0000 | [diff] [blame] | 259 | # build up the initialization prologue for the control file |
| 260 | prologue_lines = [] |
| 261 | prologue_lines.append("job.default_boot_tag(%r)\n" |
| 262 | % host.job.last_boot_tag) |
| 263 | prologue_lines.append("job.default_test_cleanup(%r)\n" |
| 264 | % host.job.run_test_cleanup) |
jadmanski | 23afbec | 2008-09-17 18:12:07 +0000 | [diff] [blame] | 265 | |
mbligh | 0910844 | 2008-10-15 16:27:38 +0000 | [diff] [blame] | 266 | # If the packaging system is being used, add the repository list. |
mbligh | c5ddfd1 | 2008-08-04 17:15:00 +0000 | [diff] [blame] | 267 | try: |
mbligh | c5ddfd1 | 2008-08-04 17:15:00 +0000 | [diff] [blame] | 268 | c = global_config.global_config |
| 269 | repos = c.get_config_value("PACKAGES", 'fetch_location', type=list) |
mbligh | 76d19f7 | 2008-10-15 16:24:43 +0000 | [diff] [blame] | 270 | pkgmgr = packages.PackageManager('autotest', hostname=host.hostname, |
| 271 | repo_urls=repos) |
jadmanski | e2eef7b | 2009-03-03 23:55:13 +0000 | [diff] [blame] | 272 | prologue_lines.append('job.add_repository(%s)\n' |
| 273 | % pkgmgr.repo_urls) |
mbligh | c5ddfd1 | 2008-08-04 17:15:00 +0000 | [diff] [blame] | 274 | except global_config.ConfigError, e: |
| 275 | pass |
| 276 | |
jadmanski | e2eef7b | 2009-03-03 23:55:13 +0000 | [diff] [blame] | 277 | # on full-size installs, turn on any profilers the server is using |
| 278 | if not self.lightweight: |
| 279 | running_profilers = host.job.profilers.add_log.iteritems() |
| 280 | for profiler, (args, dargs) in running_profilers: |
| 281 | call_args = [repr(profiler)] |
| 282 | call_args += [repr(arg) for arg in args] |
| 283 | call_args += ["%s=%r" % item for item in dargs.iteritems()] |
| 284 | prologue_lines.append("job.profilers.add(%s)\n" |
| 285 | % ", ".join(call_args)) |
| 286 | cfile = "".join(prologue_lines) |
| 287 | |
mbligh | 0910844 | 2008-10-15 16:27:38 +0000 | [diff] [blame] | 288 | cfile += open(tmppath).read() |
| 289 | open(tmppath, "w").write(cfile) |
mbligh | c5ddfd1 | 2008-08-04 17:15:00 +0000 | [diff] [blame] | 290 | |
jadmanski | c09fc15 | 2008-10-15 17:56:59 +0000 | [diff] [blame] | 291 | # Create and copy state file to remote_control_file + '.state' |
| 292 | sysinfo_state = {"__sysinfo": host.job.sysinfo.serialize()} |
| 293 | state_file = self._create_state_file(host.job, sysinfo_state) |
| 294 | host.send_file(state_file, atrun.remote_control_file + '.state') |
| 295 | os.remove(state_file) |
| 296 | |
mbligh | c5ddfd1 | 2008-08-04 17:15:00 +0000 | [diff] [blame] | 297 | # Copy control_file to remote_control_file on the host |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 298 | host.send_file(tmppath, atrun.remote_control_file) |
| 299 | if os.path.abspath(tmppath) != os.path.abspath(control_file): |
| 300 | os.remove(tmppath) |
mbligh | 0e4613b | 2007-10-29 16:55:07 +0000 | [diff] [blame] | 301 | |
jadmanski | 6bb32d7 | 2009-03-19 20:25:24 +0000 | [diff] [blame] | 302 | atrun.execute_control( |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 303 | timeout=timeout, |
| 304 | client_disconnect_timeout=client_disconnect_timeout) |
jadmanski | 23afbec | 2008-09-17 18:12:07 +0000 | [diff] [blame] | 305 | |
| 306 | |
jadmanski | c09fc15 | 2008-10-15 17:56:59 +0000 | [diff] [blame] | 307 | def _create_state_file(self, job, state_dict): |
| 308 | """ Create a state file from a dictionary. Returns the path of the |
| 309 | state file. """ |
| 310 | fd, path = tempfile.mkstemp(dir=job.tmpdir) |
| 311 | state_file = os.fdopen(fd, "w") |
| 312 | pickle.dump(state_dict, state_file) |
| 313 | state_file.close() |
| 314 | return path |
| 315 | |
| 316 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 317 | def run_timed_test(self, test_name, results_dir='.', host=None, |
jadmanski | c98c470 | 2009-01-05 15:50:06 +0000 | [diff] [blame] | 318 | timeout=None, *args, **dargs): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 319 | """ |
| 320 | Assemble a tiny little control file to just run one test, |
| 321 | and run it as an autotest client-side test |
| 322 | """ |
| 323 | if not host: |
| 324 | host = self.host |
| 325 | if not self.installed: |
| 326 | self.install(host) |
| 327 | opts = ["%s=%s" % (o[0], repr(o[1])) for o in dargs.items()] |
| 328 | cmd = ", ".join([repr(test_name)] + map(repr, args) + opts) |
| 329 | control = "job.run_test(%s)\n" % cmd |
jadmanski | c98c470 | 2009-01-05 15:50:06 +0000 | [diff] [blame] | 330 | self.run(control, results_dir, host, timeout=timeout) |
mbligh | 0e4613b | 2007-10-29 16:55:07 +0000 | [diff] [blame] | 331 | |
| 332 | |
jadmanski | c98c470 | 2009-01-05 15:50:06 +0000 | [diff] [blame] | 333 | def run_test(self, test_name, results_dir='.', host=None, *args, **dargs): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 334 | self.run_timed_test(test_name, results_dir, host, timeout=None, |
jadmanski | c98c470 | 2009-01-05 15:50:06 +0000 | [diff] [blame] | 335 | *args, **dargs) |
mbligh | d54832b | 2007-07-25 16:46:56 +0000 | [diff] [blame] | 336 | |
| 337 | |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 338 | class _Run(object): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 339 | """ |
| 340 | Represents a run of autotest control file. This class maintains |
| 341 | all the state necessary as an autotest control file is executed. |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 342 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 343 | It is not intended to be used directly, rather control files |
| 344 | should be run using the run method in Autotest. |
| 345 | """ |
mbligh | b3c0c91 | 2008-11-27 00:32:45 +0000 | [diff] [blame] | 346 | def __init__(self, host, results_dir, tag, parallel_flag, background): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 347 | self.host = host |
| 348 | self.results_dir = results_dir |
| 349 | self.env = host.env |
| 350 | self.tag = tag |
| 351 | self.parallel_flag = parallel_flag |
mbligh | b3c0c91 | 2008-11-27 00:32:45 +0000 | [diff] [blame] | 352 | self.background = background |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 353 | self.autodir = _get_autodir(self.host) |
mbligh | 78bf535 | 2008-07-11 20:27:36 +0000 | [diff] [blame] | 354 | control = os.path.join(self.autodir, 'control') |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 355 | if tag: |
mbligh | 78bf535 | 2008-07-11 20:27:36 +0000 | [diff] [blame] | 356 | control += '.' + tag |
| 357 | self.manual_control_file = control |
| 358 | self.remote_control_file = control + '.autoserv' |
mbligh | dc735a2 | 2007-08-02 16:54:37 +0000 | [diff] [blame] | 359 | |
| 360 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 361 | def verify_machine(self): |
| 362 | binary = os.path.join(self.autodir, 'bin/autotest') |
| 363 | try: |
| 364 | self.host.run('ls %s > /dev/null 2>&1' % binary) |
| 365 | except: |
| 366 | raise "Autotest does not appear to be installed" |
mbligh | dc735a2 | 2007-08-02 16:54:37 +0000 | [diff] [blame] | 367 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 368 | if not self.parallel_flag: |
| 369 | tmpdir = os.path.join(self.autodir, 'tmp') |
| 370 | download = os.path.join(self.autodir, 'tests/download') |
| 371 | self.host.run('umount %s' % tmpdir, ignore_status=True) |
| 372 | self.host.run('umount %s' % download, ignore_status=True) |
mbligh | dc735a2 | 2007-08-02 16:54:37 +0000 | [diff] [blame] | 373 | |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 374 | |
| 375 | def get_base_cmd_args(self, section): |
| 376 | args = [] |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 377 | if section > 0: |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 378 | args.append('-c') |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 379 | if self.tag: |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 380 | args.append('-t %s' % self.tag) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 381 | if self.host.job.use_external_logging(): |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 382 | args.append('-l') |
| 383 | args.append(self.remote_control_file) |
| 384 | return args |
| 385 | |
| 386 | |
| 387 | def get_background_cmd(self, section): |
| 388 | cmd = ['nohup', os.path.join(self.autodir, 'bin/autotest_client')] |
| 389 | cmd += self.get_base_cmd_args(section) |
| 390 | cmd.append('>/dev/null 2>/dev/null &') |
| 391 | return ' '.join(cmd) |
| 392 | |
| 393 | |
| 394 | def get_daemon_cmd(self, section, monitor_dir): |
| 395 | cmd = ['nohup', os.path.join(self.autodir, 'bin/autotestd'), |
| 396 | monitor_dir, '-H autoserv'] |
| 397 | cmd += self.get_base_cmd_args(section) |
| 398 | cmd.append('>/dev/null 2>/dev/null </dev/null &') |
| 399 | return ' '.join(cmd) |
| 400 | |
| 401 | |
| 402 | def get_monitor_cmd(self, monitor_dir, stdout_read, stderr_read): |
| 403 | cmd = [os.path.join(self.autodir, 'bin', 'autotestd_monitor'), |
| 404 | monitor_dir, str(stdout_read), str(stderr_read)] |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 405 | return ' '.join(cmd) |
mbligh | adf2aab | 2007-11-29 18:16:43 +0000 | [diff] [blame] | 406 | |
mbligh | d8b3925 | 2008-03-20 21:15:03 +0000 | [diff] [blame] | 407 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 408 | def get_client_log(self, section): |
jadmanski | e0c7fb6 | 2008-12-16 20:51:16 +0000 | [diff] [blame] | 409 | """ Find what the "next" client.log.* file should be and open it. """ |
| 410 | debug_dir = os.path.join(self.results_dir, "debug") |
| 411 | client_logs = glob.glob(os.path.join(debug_dir, "client.log.*")) |
| 412 | next_log = os.path.join(debug_dir, "client.log.%d" % len(client_logs)) |
| 413 | return open(next_log, "w", 0) |
mbligh | d8b3925 | 2008-03-20 21:15:03 +0000 | [diff] [blame] | 414 | |
| 415 | |
jadmanski | b264ed0 | 2009-01-12 23:54:27 +0000 | [diff] [blame] | 416 | @staticmethod |
| 417 | def is_client_job_finished(last_line): |
| 418 | return bool(re.match(r'^END .*\t----\t----\t.*$', last_line)) |
| 419 | |
| 420 | |
| 421 | @staticmethod |
| 422 | def is_client_job_rebooting(last_line): |
| 423 | return bool(re.match(r'^\t*GOOD\t----\treboot\.start.*$', last_line)) |
| 424 | |
| 425 | |
| 426 | def log_unexpected_abort(self): |
| 427 | msg = "Autotest client terminated unexpectedly" |
| 428 | self.host.job.record("END ABORT", None, None, msg) |
| 429 | |
| 430 | |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 431 | def _execute_in_background(self, section, timeout): |
| 432 | full_cmd = self.get_background_cmd(section) |
| 433 | devnull = open(os.devnull, "w") |
mbligh | d8b3925 | 2008-03-20 21:15:03 +0000 | [diff] [blame] | 434 | |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 435 | old_resultdir = self.host.job.resultdir |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 436 | try: |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 437 | self.host.job.resultdir = self.results_dir |
| 438 | result = self.host.run(full_cmd, ignore_status=True, |
| 439 | timeout=timeout, |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 440 | stdout_tee=devnull, |
| 441 | stderr_tee=devnull) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 442 | finally: |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 443 | self.host.job.resultdir = old_resultdir |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 444 | |
| 445 | return result |
| 446 | |
| 447 | |
| 448 | @staticmethod |
| 449 | def _strip_stderr_prologue(stderr): |
| 450 | """Strips the 'standard' prologue that get pre-pended to every |
| 451 | remote command and returns the text that was actually written to |
| 452 | stderr by the remote command.""" |
| 453 | stderr_lines = stderr.split("\n")[1:] |
| 454 | if not stderr_lines: |
| 455 | return "" |
| 456 | elif stderr_lines[0].startswith("NOTE: autotestd_monitor"): |
| 457 | del stderr_lines[0] |
| 458 | return "\n".join(stderr_lines) |
| 459 | |
| 460 | |
| 461 | def _execute_daemon(self, section, timeout, stderr_redirector, |
| 462 | client_disconnect_timeout): |
| 463 | monitor_dir = self.host.get_tmp_dir() |
| 464 | daemon_cmd = self.get_daemon_cmd(section, monitor_dir) |
| 465 | client_log = self.get_client_log(section) |
| 466 | |
| 467 | stdout_read = stderr_read = 0 |
| 468 | old_resultdir = self.host.job.resultdir |
| 469 | try: |
jadmanski | 29a4c70 | 2009-03-03 23:30:59 +0000 | [diff] [blame] | 470 | self.host.job.resultdir = self.results_dir |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 471 | self.host.run(daemon_cmd, ignore_status=True, timeout=timeout) |
| 472 | while True: |
| 473 | monitor_cmd = self.get_monitor_cmd(monitor_dir, stdout_read, |
| 474 | stderr_read) |
| 475 | try: |
| 476 | result = self.host.run(monitor_cmd, ignore_status=True, |
| 477 | timeout=timeout, |
| 478 | stdout_tee=client_log, |
| 479 | stderr_tee=stderr_redirector) |
| 480 | except error.AutoservRunError, e: |
| 481 | result = e.result_obj |
| 482 | result.exit_status = None |
| 483 | stderr_redirector.log_warning( |
| 484 | "Autotest client was disconnected: %s" % e.description) |
| 485 | except error.AutoservSSHTimeout: |
| 486 | result = utils.CmdResult(monitor_cmd, "", "", None, 0) |
| 487 | stderr_redirector.log_warning( |
| 488 | "Attempt to connect to Autotest client timed out") |
| 489 | |
| 490 | stdout_read += len(result.stdout) |
| 491 | stderr_read += len(self._strip_stderr_prologue(result.stderr)) |
| 492 | |
| 493 | if result.exit_status is not None: |
| 494 | return result |
| 495 | elif not self.host.wait_up(client_disconnect_timeout): |
| 496 | raise error.AutoservSSHTimeout( |
| 497 | "client was disconnected, reconnect timed out") |
| 498 | finally: |
| 499 | self.host.job.resultdir = old_resultdir |
| 500 | |
| 501 | |
| 502 | def execute_section(self, section, timeout, stderr_redirector, |
| 503 | client_disconnect_timeout): |
showard | b18134f | 2009-03-20 20:52:18 +0000 | [diff] [blame^] | 504 | logging.info("Executing %s/bin/autotest %s/control phase %d", |
| 505 | self.autodir, self.autodir, section) |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 506 | |
| 507 | if self.background: |
| 508 | result = self._execute_in_background(section, timeout) |
| 509 | else: |
| 510 | result = self._execute_daemon(section, timeout, stderr_redirector, |
| 511 | client_disconnect_timeout) |
| 512 | |
| 513 | last_line = stderr_redirector.last_line |
mbligh | 2bf2db6 | 2007-11-27 00:53:18 +0000 | [diff] [blame] | 514 | |
jadmanski | b264ed0 | 2009-01-12 23:54:27 +0000 | [diff] [blame] | 515 | # check if we failed hard enough to warrant an exception |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 516 | if result.exit_status == 1: |
jadmanski | b264ed0 | 2009-01-12 23:54:27 +0000 | [diff] [blame] | 517 | err = error.AutotestRunError("client job was aborted") |
| 518 | elif not self.background and not result.stderr: |
| 519 | err = error.AutotestRunError( |
jadmanski | e413053 | 2009-03-17 18:01:28 +0000 | [diff] [blame] | 520 | "execute_section %s failed to return anything\n" |
| 521 | "stdout:%s\n" % (section, result.stdout)) |
jadmanski | b264ed0 | 2009-01-12 23:54:27 +0000 | [diff] [blame] | 522 | else: |
| 523 | err = None |
mbligh | 0e4613b | 2007-10-29 16:55:07 +0000 | [diff] [blame] | 524 | |
jadmanski | b264ed0 | 2009-01-12 23:54:27 +0000 | [diff] [blame] | 525 | # log something if the client failed AND never finished logging |
| 526 | if err and not self.is_client_job_finished(last_line): |
| 527 | self.log_unexpected_abort() |
| 528 | |
| 529 | if err: |
| 530 | raise err |
| 531 | else: |
| 532 | return stderr_redirector.last_line |
jadmanski | 4600e34 | 2008-10-29 22:54:00 +0000 | [diff] [blame] | 533 | |
| 534 | |
| 535 | def _wait_for_reboot(self): |
showard | b18134f | 2009-03-20 20:52:18 +0000 | [diff] [blame^] | 536 | logging.info("Client is rebooting") |
| 537 | logging.info("Waiting for client to halt") |
jadmanski | 4600e34 | 2008-10-29 22:54:00 +0000 | [diff] [blame] | 538 | if not self.host.wait_down(HALT_TIME): |
| 539 | err = "%s failed to shutdown after %d" |
| 540 | err %= (self.host.hostname, HALT_TIME) |
| 541 | raise error.AutotestRunError(err) |
showard | b18134f | 2009-03-20 20:52:18 +0000 | [diff] [blame^] | 542 | logging.info("Client down, waiting for restart") |
jadmanski | 4600e34 | 2008-10-29 22:54:00 +0000 | [diff] [blame] | 543 | if not self.host.wait_up(BOOT_TIME): |
| 544 | # since reboot failed |
| 545 | # hardreset the machine once if possible |
| 546 | # before failing this control file |
| 547 | warning = "%s did not come back up, hard resetting" |
| 548 | warning %= self.host.hostname |
showard | b18134f | 2009-03-20 20:52:18 +0000 | [diff] [blame^] | 549 | logging.warning(warning) |
jadmanski | 4600e34 | 2008-10-29 22:54:00 +0000 | [diff] [blame] | 550 | try: |
| 551 | self.host.hardreset(wait=False) |
mbligh | d99d3b27 | 2008-12-22 14:41:27 +0000 | [diff] [blame] | 552 | except (AttributeError, error.AutoservUnsupportedError): |
jadmanski | 4600e34 | 2008-10-29 22:54:00 +0000 | [diff] [blame] | 553 | warning = "Hard reset unsupported on %s" |
| 554 | warning %= self.host.hostname |
showard | b18134f | 2009-03-20 20:52:18 +0000 | [diff] [blame^] | 555 | logging.warning(warning) |
jadmanski | 4600e34 | 2008-10-29 22:54:00 +0000 | [diff] [blame] | 556 | raise error.AutotestRunError("%s failed to boot after %ds" % |
| 557 | (self.host.hostname, BOOT_TIME)) |
| 558 | self.host.reboot_followup() |
mbligh | dc735a2 | 2007-08-02 16:54:37 +0000 | [diff] [blame] | 559 | |
| 560 | |
jadmanski | 6bb32d7 | 2009-03-19 20:25:24 +0000 | [diff] [blame] | 561 | def _process_client_state_file(self): |
| 562 | state_file = os.path.basename(self.remote_control_file) + ".state" |
| 563 | state_path = os.path.join(self.results_dir, state_file) |
| 564 | try: |
| 565 | state_dict = pickle.load(open(state_path)) |
| 566 | except Exception, e: |
| 567 | msg = "Ignoring error while loading client job state file: %s" % e |
| 568 | self.logger.warning(msg) |
| 569 | state_dict = {} |
| 570 | |
| 571 | # clear out the state file |
| 572 | # TODO: stash the file away somewhere useful instead |
| 573 | try: |
| 574 | os.remove(state_path) |
| 575 | except Exception: |
| 576 | pass |
| 577 | |
| 578 | msg = "Persistent state variables pulled back from %s: %s" |
| 579 | msg %= (self.host.hostname, state_dict) |
| 580 | print msg |
| 581 | |
| 582 | if "__run_test_cleanup" in state_dict: |
| 583 | if state_dict["__run_test_cleanup"]: |
| 584 | self.host.job.enable_test_cleanup() |
| 585 | else: |
| 586 | self.host.job.disable_test_cleanup() |
| 587 | |
| 588 | if "__last_boot_tag" in state_dict: |
| 589 | self.host.job.last_boot_tag = state_dict["__last_boot_tag"] |
| 590 | |
| 591 | if "__sysinfo" in state_dict: |
| 592 | self.host.job.sysinfo.deserialize(state_dict["__sysinfo"]) |
| 593 | |
| 594 | |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 595 | def execute_control(self, timeout=None, client_disconnect_timeout=None): |
jadmanski | 6bb32d7 | 2009-03-19 20:25:24 +0000 | [diff] [blame] | 596 | if not self.background: |
| 597 | collector = log_collector(self.host, self.tag, self.results_dir) |
| 598 | hostname = self.host.hostname |
| 599 | remote_results = collector.client_results_dir |
| 600 | local_results = collector.server_results_dir |
| 601 | self.host.job.add_client_log(hostname, remote_results, |
| 602 | local_results) |
| 603 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 604 | section = 0 |
jadmanski | 4600e34 | 2008-10-29 22:54:00 +0000 | [diff] [blame] | 605 | start_time = time.time() |
| 606 | |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 607 | logger = client_logger(self.host, self.tag, self.results_dir) |
jadmanski | 4600e34 | 2008-10-29 22:54:00 +0000 | [diff] [blame] | 608 | try: |
| 609 | while not timeout or time.time() < start_time + timeout: |
| 610 | if timeout: |
| 611 | section_timeout = start_time + timeout - time.time() |
| 612 | else: |
| 613 | section_timeout = None |
| 614 | last = self.execute_section(section, section_timeout, |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 615 | logger, client_disconnect_timeout) |
mbligh | b3c0c91 | 2008-11-27 00:32:45 +0000 | [diff] [blame] | 616 | if self.background: |
| 617 | return |
jadmanski | 4600e34 | 2008-10-29 22:54:00 +0000 | [diff] [blame] | 618 | section += 1 |
jadmanski | b264ed0 | 2009-01-12 23:54:27 +0000 | [diff] [blame] | 619 | if self.is_client_job_finished(last): |
showard | b18134f | 2009-03-20 20:52:18 +0000 | [diff] [blame^] | 620 | logging.info("Client complete") |
jadmanski | 4600e34 | 2008-10-29 22:54:00 +0000 | [diff] [blame] | 621 | return |
jadmanski | b264ed0 | 2009-01-12 23:54:27 +0000 | [diff] [blame] | 622 | elif self.is_client_job_rebooting(last): |
jadmanski | 79ab928 | 2008-11-11 17:53:12 +0000 | [diff] [blame] | 623 | try: |
| 624 | self._wait_for_reboot() |
| 625 | except error.AutotestRunError, e: |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 626 | self.host.job.record("ABORT", None, "reboot", str(e)) |
| 627 | self.host.job.record("END ABORT", None, None, str(e)) |
jadmanski | 79ab928 | 2008-11-11 17:53:12 +0000 | [diff] [blame] | 628 | raise |
jadmanski | 4600e34 | 2008-10-29 22:54:00 +0000 | [diff] [blame] | 629 | continue |
| 630 | |
| 631 | # if we reach here, something unexpected happened |
jadmanski | b264ed0 | 2009-01-12 23:54:27 +0000 | [diff] [blame] | 632 | self.log_unexpected_abort() |
jadmanski | 4600e34 | 2008-10-29 22:54:00 +0000 | [diff] [blame] | 633 | |
| 634 | # give the client machine a chance to recover from a crash |
| 635 | self.host.wait_up(CRASH_RECOVERY_TIME) |
| 636 | msg = ("Aborting - unexpected final status message from " |
| 637 | "client: %s\n") % last |
| 638 | raise error.AutotestRunError(msg) |
| 639 | finally: |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 640 | logger.close() |
jadmanski | 6bb32d7 | 2009-03-19 20:25:24 +0000 | [diff] [blame] | 641 | if not self.background: |
| 642 | collector.collect_client_job_results() |
| 643 | self._process_client_state_file() |
| 644 | self.host.job.remove_client_log(hostname, remote_results, |
| 645 | local_results) |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 646 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 647 | # should only get here if we timed out |
| 648 | assert timeout |
| 649 | raise error.AutotestTimeoutError() |
mbligh | 0e4613b | 2007-10-29 16:55:07 +0000 | [diff] [blame] | 650 | |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 651 | |
| 652 | def _get_autodir(host): |
mbligh | 3c7a150 | 2008-07-24 18:08:47 +0000 | [diff] [blame] | 653 | autodir = host.get_autodir() |
| 654 | if autodir: |
| 655 | return autodir |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 656 | try: |
| 657 | # There's no clean way to do this. readlink may not exist |
| 658 | cmd = "python -c 'import os,sys; print os.readlink(sys.argv[1])' /etc/autotest.conf 2> /dev/null" |
mbligh | 3c7a150 | 2008-07-24 18:08:47 +0000 | [diff] [blame] | 659 | autodir = os.path.dirname(host.run(cmd).stdout) |
| 660 | if autodir: |
| 661 | return autodir |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 662 | except error.AutoservRunError: |
| 663 | pass |
| 664 | for path in ['/usr/local/autotest', '/home/autotest']: |
| 665 | try: |
jadmanski | 169ecad | 2008-09-12 15:49:44 +0000 | [diff] [blame] | 666 | host.run('ls %s > /dev/null 2>&1' % |
| 667 | os.path.join(path, 'bin/autotest')) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 668 | return path |
| 669 | except error.AutoservRunError: |
| 670 | pass |
| 671 | raise error.AutotestRunError("Cannot figure out autotest directory") |
mbligh | d8b3925 | 2008-03-20 21:15:03 +0000 | [diff] [blame] | 672 | |
| 673 | |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 674 | class log_collector(object): |
| 675 | def __init__(self, host, client_tag, results_dir): |
| 676 | self.host = host |
| 677 | if not client_tag: |
| 678 | client_tag = "default" |
| 679 | self.client_results_dir = os.path.join(host.get_autodir(), "results", |
| 680 | client_tag) |
| 681 | self.server_results_dir = results_dir |
| 682 | |
| 683 | |
| 684 | def collect_client_job_results(self): |
| 685 | """ A method that collects all the current results of a running |
| 686 | client job into the results dir. By default does nothing as no |
| 687 | client job is running, but when running a client job you can override |
| 688 | this with something that will actually do something. """ |
| 689 | |
| 690 | # make an effort to wait for the machine to come up |
| 691 | try: |
| 692 | self.host.wait_up(timeout=30) |
| 693 | except error.AutoservError: |
| 694 | # don't worry about any errors, we'll try and |
| 695 | # get the results anyway |
| 696 | pass |
| 697 | |
| 698 | |
| 699 | # Copy all dirs in default to results_dir |
| 700 | try: |
| 701 | keyval_path = self._prepare_for_copying_logs() |
| 702 | self.host.get_file(self.client_results_dir + '/', |
| 703 | self.server_results_dir) |
| 704 | self._process_copied_logs(keyval_path) |
| 705 | self._postprocess_copied_logs() |
| 706 | except Exception: |
| 707 | # well, don't stop running just because we couldn't get logs |
showard | b18134f | 2009-03-20 20:52:18 +0000 | [diff] [blame^] | 708 | e_msg = "Unexpected error copying test result logs, continuing ..." |
| 709 | logging.error(e_msg) |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 710 | traceback.print_exc(file=sys.stdout) |
| 711 | |
| 712 | |
| 713 | def _prepare_for_copying_logs(self): |
| 714 | server_keyval = os.path.join(self.server_results_dir, 'keyval') |
| 715 | if not os.path.exists(server_keyval): |
| 716 | # Client-side keyval file can be copied directly |
| 717 | return |
| 718 | |
| 719 | # Copy client-side keyval to temporary location |
| 720 | suffix = '.keyval_%s' % self.host.hostname |
| 721 | fd, keyval_path = tempfile.mkstemp(suffix) |
| 722 | os.close(fd) |
| 723 | try: |
| 724 | client_keyval = os.path.join(self.client_results_dir, 'keyval') |
| 725 | try: |
| 726 | self.host.get_file(client_keyval, keyval_path) |
| 727 | finally: |
| 728 | # We will squirrel away the client side keyval |
| 729 | # away and move it back when we are done |
| 730 | remote_temp_dir = self.host.get_tmp_dir() |
| 731 | self.temp_keyval_path = os.path.join(remote_temp_dir, "keyval") |
| 732 | self.host.run('mv %s %s' % (client_keyval, |
| 733 | self.temp_keyval_path)) |
| 734 | except (error.AutoservRunError, error.AutoservSSHTimeout): |
showard | b18134f | 2009-03-20 20:52:18 +0000 | [diff] [blame^] | 735 | logging.error("Prepare for copying logs failed") |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 736 | return keyval_path |
| 737 | |
| 738 | |
| 739 | def _process_copied_logs(self, keyval_path): |
| 740 | if not keyval_path: |
| 741 | # Client-side keyval file was copied directly |
| 742 | return |
| 743 | |
| 744 | # Append contents of keyval_<host> file to keyval file |
| 745 | try: |
| 746 | # Read in new and old keyval files |
| 747 | new_keyval = utils.read_keyval(keyval_path) |
| 748 | old_keyval = utils.read_keyval(self.server_results_dir) |
| 749 | # 'Delete' from new keyval entries that are in both |
| 750 | tmp_keyval = {} |
| 751 | for key, val in new_keyval.iteritems(): |
| 752 | if key not in old_keyval: |
| 753 | tmp_keyval[key] = val |
| 754 | # Append new info to keyval file |
| 755 | utils.write_keyval(self.server_results_dir, tmp_keyval) |
| 756 | # Delete keyval_<host> file |
| 757 | os.remove(keyval_path) |
| 758 | except IOError: |
showard | b18134f | 2009-03-20 20:52:18 +0000 | [diff] [blame^] | 759 | logging.error("Process copied logs failed") |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 760 | |
| 761 | |
| 762 | def _postprocess_copied_logs(self): |
| 763 | # we can now put our keyval file back |
| 764 | client_keyval = os.path.join(self.client_results_dir, 'keyval') |
| 765 | try: |
| 766 | self.host.run('mv %s %s' % (self.temp_keyval_path, client_keyval)) |
| 767 | except Exception: |
| 768 | pass |
| 769 | |
| 770 | |
| 771 | |
| 772 | # a file-like object for catching stderr from an autotest client and |
| 773 | # extracting status logs from it |
| 774 | class client_logger(object): |
| 775 | """Partial file object to write to both stdout and |
| 776 | the status log file. We only implement those methods |
| 777 | utils.run() actually calls. |
| 778 | |
| 779 | Note that this class is fairly closely coupled with server_job, as it |
| 780 | uses special job._ methods to actually carry out the loggging. |
| 781 | """ |
| 782 | status_parser = re.compile(r"^AUTOTEST_STATUS:([^:]*):(.*)$") |
| 783 | test_complete_parser = re.compile(r"^AUTOTEST_TEST_COMPLETE:(.*)$") |
| 784 | extract_indent = re.compile(r"^(\t*).*$") |
| 785 | |
| 786 | def __init__(self, host, tag, server_results_dir): |
| 787 | self.host = host |
| 788 | self.job = host.job |
| 789 | self.log_collector = log_collector(host, tag, server_results_dir) |
| 790 | self.leftover = "" |
| 791 | self.last_line = "" |
| 792 | self.logs = {} |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 793 | self.server_warnings = [] |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 794 | |
| 795 | |
| 796 | def _process_log_dict(self, log_dict): |
| 797 | log_list = log_dict.pop("logs", []) |
| 798 | for key in sorted(log_dict.iterkeys()): |
| 799 | log_list += self._process_log_dict(log_dict.pop(key)) |
| 800 | return log_list |
| 801 | |
| 802 | |
| 803 | def _process_logs(self): |
| 804 | """Go through the accumulated logs in self.log and print them |
| 805 | out to stdout and the status log. Note that this processes |
| 806 | logs in an ordering where: |
| 807 | |
| 808 | 1) logs to different tags are never interleaved |
| 809 | 2) logs to x.y come before logs to x.y.z for all z |
| 810 | 3) logs to x.y come before x.z whenever y < z |
| 811 | |
| 812 | Note that this will in general not be the same as the |
| 813 | chronological ordering of the logs. However, if a chronological |
| 814 | ordering is desired that one can be reconstructed from the |
| 815 | status log by looking at timestamp lines.""" |
| 816 | log_list = self._process_log_dict(self.logs) |
| 817 | for line in log_list: |
| 818 | self.job._record_prerendered(line + '\n') |
| 819 | if log_list: |
| 820 | self.last_line = log_list[-1] |
| 821 | |
| 822 | |
| 823 | def _process_quoted_line(self, tag, line): |
| 824 | """Process a line quoted with an AUTOTEST_STATUS flag. If the |
| 825 | tag is blank then we want to push out all the data we've been |
| 826 | building up in self.logs, and then the newest line. If the |
| 827 | tag is not blank, then push the line into the logs for handling |
| 828 | later.""" |
showard | b18134f | 2009-03-20 20:52:18 +0000 | [diff] [blame^] | 829 | logging.info(line) |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 830 | if tag == "": |
| 831 | self._process_logs() |
| 832 | self.job._record_prerendered(line + '\n') |
| 833 | self.last_line = line |
| 834 | else: |
| 835 | tag_parts = [int(x) for x in tag.split(".")] |
| 836 | log_dict = self.logs |
| 837 | for part in tag_parts: |
| 838 | log_dict = log_dict.setdefault(part, {}) |
| 839 | log_list = log_dict.setdefault("logs", []) |
| 840 | log_list.append(line) |
| 841 | |
| 842 | |
jadmanski | f37df84 | 2009-02-11 00:03:26 +0000 | [diff] [blame] | 843 | def _process_info_line(self, line): |
| 844 | """Check if line is an INFO line, and if it is, interpret any control |
| 845 | messages (e.g. enabling/disabling warnings) that it may contain.""" |
| 846 | match = re.search(r"^\t*INFO\t----\t----(.*)\t[^\t]*$", line) |
| 847 | if not match: |
| 848 | return # not an INFO line |
| 849 | for field in match.group(1).split('\t'): |
| 850 | if field.startswith("warnings.enable="): |
| 851 | func = self.job.enable_warnings |
| 852 | elif field.startswith("warnings.disable="): |
| 853 | func = self.job.disable_warnings |
| 854 | else: |
| 855 | continue |
| 856 | warning_type = field.split("=", 1)[1] |
| 857 | func(warning_type, record=False) |
| 858 | |
| 859 | |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 860 | def _process_line(self, line): |
| 861 | """Write out a line of data to the appropriate stream. Status |
| 862 | lines sent by autotest will be prepended with |
| 863 | "AUTOTEST_STATUS", and all other lines are ssh error |
| 864 | messages.""" |
| 865 | status_match = self.status_parser.search(line) |
| 866 | test_complete_match = self.test_complete_parser.search(line) |
| 867 | if status_match: |
| 868 | tag, line = status_match.groups() |
jadmanski | f37df84 | 2009-02-11 00:03:26 +0000 | [diff] [blame] | 869 | self._process_info_line(line) |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 870 | self._process_quoted_line(tag, line) |
| 871 | elif test_complete_match: |
jadmanski | fcc0d5d | 2009-02-12 21:52:54 +0000 | [diff] [blame] | 872 | self._process_logs() |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 873 | fifo_path, = test_complete_match.groups() |
| 874 | self.log_collector.collect_client_job_results() |
| 875 | self.host.run("echo A > %s" % fifo_path) |
| 876 | else: |
showard | b18134f | 2009-03-20 20:52:18 +0000 | [diff] [blame^] | 877 | logging.info(line) |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 878 | |
| 879 | |
| 880 | def _format_warnings(self, last_line, warnings): |
| 881 | # use the indentation of whatever the last log line was |
| 882 | indent = self.extract_indent.match(last_line).group(1) |
| 883 | # if the last line starts a new group, add an extra indent |
| 884 | if last_line.lstrip('\t').startswith("START\t"): |
| 885 | indent += '\t' |
| 886 | return [self.job._render_record("WARN", None, None, msg, |
| 887 | timestamp, indent).rstrip('\n') |
| 888 | for timestamp, msg in warnings] |
| 889 | |
| 890 | |
| 891 | def _process_warnings(self, last_line, log_dict, warnings): |
| 892 | if log_dict.keys() in ([], ["logs"]): |
| 893 | # there are no sub-jobs, just append the warnings here |
| 894 | warnings = self._format_warnings(last_line, warnings) |
| 895 | log_list = log_dict.setdefault("logs", []) |
| 896 | log_list += warnings |
| 897 | for warning in warnings: |
| 898 | sys.stdout.write(warning + '\n') |
| 899 | else: |
| 900 | # there are sub-jobs, so put the warnings in there |
| 901 | log_list = log_dict.get("logs", []) |
| 902 | if log_list: |
| 903 | last_line = log_list[-1] |
| 904 | for key in sorted(log_dict.iterkeys()): |
| 905 | if key != "logs": |
| 906 | self._process_warnings(last_line, |
| 907 | log_dict[key], |
| 908 | warnings) |
| 909 | |
jadmanski | f37df84 | 2009-02-11 00:03:26 +0000 | [diff] [blame] | 910 | |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 911 | def log_warning(self, msg): |
| 912 | """Injects a WARN message into the current status logging stream.""" |
| 913 | self.server_warnings.append((int(time.time()), msg)) |
| 914 | |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 915 | |
| 916 | def write(self, data): |
| 917 | # first check for any new console warnings |
jadmanski | 6dadd83 | 2009-02-05 23:39:27 +0000 | [diff] [blame] | 918 | warnings = self.job._read_warnings() + self.server_warnings |
| 919 | warnings.sort() # sort into timestamp order |
| 920 | self.server_warnings = [] |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 921 | self._process_warnings(self.last_line, self.logs, warnings) |
| 922 | # now process the newest data written out |
| 923 | data = self.leftover + data |
| 924 | lines = data.split("\n") |
| 925 | # process every line but the last one |
| 926 | for line in lines[:-1]: |
| 927 | self._process_line(line) |
| 928 | # save the last line for later processing |
| 929 | # since we may not have the whole line yet |
| 930 | self.leftover = lines[-1] |
| 931 | |
| 932 | |
| 933 | def flush(self): |
| 934 | sys.stdout.flush() |
| 935 | |
| 936 | |
| 937 | def close(self): |
| 938 | if self.leftover: |
| 939 | self._process_line(self.leftover) |
| 940 | self._process_logs() |
| 941 | self.flush() |
| 942 | |
| 943 | |
mbligh | a700772 | 2009-01-13 00:37:11 +0000 | [diff] [blame] | 944 | SiteAutotest = client_utils.import_site_class( |
| 945 | __file__, "autotest_lib.server.site_autotest", "SiteAutotest", |
| 946 | BaseAutotest) |
mbligh | d8b3925 | 2008-03-20 21:15:03 +0000 | [diff] [blame] | 947 | |
| 948 | class Autotest(SiteAutotest): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 949 | pass |