mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 1 | #!/usr/bin/python |
| 2 | # |
| 3 | # Copyright 2007 Google Inc. Released under the GPL v2 |
| 4 | |
mbligh | dc735a2 | 2007-08-02 16:54:37 +0000 | [diff] [blame] | 5 | """ |
| 6 | This module defines the Autotest class |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 7 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 8 | Autotest: software to run tests automatically |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 9 | """ |
| 10 | |
mbligh | dc735a2 | 2007-08-02 16:54:37 +0000 | [diff] [blame] | 11 | __author__ = """ |
| 12 | mbligh@google.com (Martin J. Bligh), |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 13 | poirier@google.com (Benjamin Poirier), |
mbligh | dc735a2 | 2007-08-02 16:54:37 +0000 | [diff] [blame] | 14 | stutsman@google.com (Ryan Stutsman) |
| 15 | """ |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 16 | |
mbligh | 02ff2d5 | 2008-06-03 15:00:21 +0000 | [diff] [blame] | 17 | import re, os, sys, traceback, subprocess, tempfile, shutil, time |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 18 | |
mbligh | ccb9e18 | 2008-04-17 15:42:10 +0000 | [diff] [blame] | 19 | from autotest_lib.server import installable_object, utils, server_job |
| 20 | from autotest_lib.client.common_lib import logging |
mbligh | c5ddfd1 | 2008-08-04 17:15:00 +0000 | [diff] [blame^] | 21 | from autotest_lib.client.common_lib import error, global_config, packages |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 22 | |
| 23 | |
mbligh | 3c7a150 | 2008-07-24 18:08:47 +0000 | [diff] [blame] | 24 | |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 25 | AUTOTEST_SVN = 'svn://test.kernel.org/autotest/trunk/client' |
| 26 | AUTOTEST_HTTP = 'http://test.kernel.org/svn/autotest/trunk/client' |
| 27 | |
| 28 | # Timeouts for powering down and up respectively |
| 29 | HALT_TIME = 300 |
mbligh | 07c1eac | 2007-11-05 18:39:29 +0000 | [diff] [blame] | 30 | BOOT_TIME = 1800 |
jadmanski | ec85914 | 2008-05-29 21:33:39 +0000 | [diff] [blame] | 31 | CRASH_RECOVERY_TIME = 9000 |
mbligh | 0e4613b | 2007-10-29 16:55:07 +0000 | [diff] [blame] | 32 | |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 33 | |
mbligh | d8b3925 | 2008-03-20 21:15:03 +0000 | [diff] [blame] | 34 | class BaseAutotest(installable_object.InstallableObject): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 35 | """ |
| 36 | This class represents the Autotest program. |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 37 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 38 | Autotest is used to run tests automatically and collect the results. |
| 39 | It also supports profilers. |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 40 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 41 | Implementation details: |
| 42 | This is a leaf class in an abstract class hierarchy, it must |
| 43 | implement the unimplemented methods in parent classes. |
| 44 | """ |
| 45 | job = None |
mbligh | 119c12a | 2007-11-12 22:13:44 +0000 | [diff] [blame] | 46 | |
| 47 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 48 | def __init__(self, host = None): |
| 49 | self.host = host |
| 50 | self.got = False |
| 51 | self.installed = False |
| 52 | self.serverdir = utils.get_server_dir() |
| 53 | super(BaseAutotest, self).__init__() |
mbligh | c8949b8 | 2007-07-23 16:33:58 +0000 | [diff] [blame] | 54 | |
mbligh | dc735a2 | 2007-08-02 16:54:37 +0000 | [diff] [blame] | 55 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 56 | @logging.record |
| 57 | def install(self, host = None): |
| 58 | """ |
| 59 | Install autotest. If get() was not called previously, an |
| 60 | attempt will be made to install from the autotest svn |
| 61 | repository. |
mbligh | 9a3f5e5 | 2008-05-28 21:21:43 +0000 | [diff] [blame] | 62 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 63 | Args: |
| 64 | host: a Host instance on which autotest will be |
| 65 | installed |
mbligh | 9a3f5e5 | 2008-05-28 21:21:43 +0000 | [diff] [blame] | 66 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 67 | Raises: |
| 68 | AutoservError: if a tarball was not specified and |
| 69 | the target host does not have svn installed in its path |
mbligh | 9a3f5e5 | 2008-05-28 21:21:43 +0000 | [diff] [blame] | 70 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 71 | TODO(poirier): check dependencies |
| 72 | autotest needs: |
| 73 | bzcat |
| 74 | liboptdev (oprofile) |
| 75 | binutils-dev (oprofile) |
| 76 | make |
| 77 | psutils (netperf) |
| 78 | """ |
| 79 | if not host: |
| 80 | host = self.host |
| 81 | if not self.got: |
| 82 | self.get() |
| 83 | host.wait_up(timeout=30) |
| 84 | host.setup() |
| 85 | print "Installing autotest on %s" % host.hostname |
mbligh | 40f122a | 2007-11-03 23:08:46 +0000 | [diff] [blame] | 86 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 87 | # Let's try to figure out where autotest is installed. If we can't, |
| 88 | # (autotest not installed) just assume '/usr/local/autotest' and |
| 89 | # proceed. |
| 90 | try: |
| 91 | autodir = _get_autodir(host) |
| 92 | except error.AutotestRunError: |
| 93 | autodir = '/usr/local/autotest' |
mbligh | 2974230 | 2008-05-09 16:06:37 +0000 | [diff] [blame] | 94 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 95 | host.run('mkdir -p "%s"' % utils.sh_escape(autodir)) |
mbligh | 40f122a | 2007-11-03 23:08:46 +0000 | [diff] [blame] | 96 | |
mbligh | c5ddfd1 | 2008-08-04 17:15:00 +0000 | [diff] [blame^] | 97 | # Fetch the autotest client from the nearest repository |
| 98 | try: |
| 99 | c = global_config.global_config |
| 100 | repos = c.get_config_value("PACKAGES", 'fetch_location', type=list) |
| 101 | pkgmgr = packages.PackageManager( |
| 102 | autodir, repo_urls=repos, do_locking=False, |
| 103 | run_function=host.run, |
| 104 | run_function_dargs=dict(timeout=600)) |
| 105 | # The packages dir is used to store all the packages that |
| 106 | # are fetched on that client. (for the tests,deps etc. |
| 107 | # too apart from the client) |
| 108 | pkg_dir = os.path.join(autodir, 'packages') |
| 109 | # clean up the autodir except for the packages directory |
| 110 | host.run('cd %s && ls | grep -v "^packages$"' |
| 111 | ' | xargs rm -rf && rm -rf .[^.]*' % autodir) |
| 112 | pkgmgr.install_pkg('autotest', 'client', pkg_dir, autodir, |
| 113 | preserve_install_dir=True) |
| 114 | self.installed = True |
| 115 | return |
| 116 | except global_config.ConfigError, e: |
| 117 | print ("Could not install autotest using the" |
| 118 | " packaging system %s" % e) |
| 119 | except (packages.PackageInstallError, error.AutoservRunError), e: |
| 120 | print "Could not install autotest from %s : %s " % (repos, e) |
| 121 | |
mbligh | 40f122a | 2007-11-03 23:08:46 +0000 | [diff] [blame] | 122 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 123 | # try to install from file or directory |
| 124 | if self.source_material: |
| 125 | if os.path.isdir(self.source_material): |
| 126 | # Copy autotest recursively |
| 127 | host.send_file(self.source_material, autodir) |
| 128 | else: |
| 129 | # Copy autotest via tarball |
| 130 | e_msg = 'Installation method not yet implemented!' |
| 131 | raise NotImplementedError(e_msg) |
| 132 | print "Installation of autotest completed" |
| 133 | self.installed = True |
| 134 | return |
mbligh | 9133490 | 2007-09-28 01:47:59 +0000 | [diff] [blame] | 135 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 136 | # if that fails try to install using svn |
| 137 | if utils.run('which svn').exit_status: |
mbligh | 78bf535 | 2008-07-11 20:27:36 +0000 | [diff] [blame] | 138 | raise error.AutoservError('svn not found on target machine: %s' |
| 139 | % host.name) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 140 | try: |
mbligh | 78bf535 | 2008-07-11 20:27:36 +0000 | [diff] [blame] | 141 | host.run('svn checkout %s %s' % (AUTOTEST_SVN, autodir)) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 142 | except error.AutoservRunError, e: |
mbligh | 78bf535 | 2008-07-11 20:27:36 +0000 | [diff] [blame] | 143 | host.run('svn checkout %s %s' % (AUTOTEST_HTTP, autodir)) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 144 | print "Installation of autotest completed" |
| 145 | self.installed = True |
mbligh | 9133490 | 2007-09-28 01:47:59 +0000 | [diff] [blame] | 146 | |
| 147 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 148 | def get(self, location = None): |
| 149 | if not location: |
| 150 | location = os.path.join(self.serverdir, '../client') |
| 151 | location = os.path.abspath(location) |
| 152 | # If there's stuff run on our client directory already, it |
| 153 | # can cause problems. Try giving it a quick clean first. |
| 154 | cwd = os.getcwd() |
| 155 | os.chdir(location) |
| 156 | os.system('tools/make_clean') |
| 157 | os.chdir(cwd) |
| 158 | super(BaseAutotest, self).get(location) |
| 159 | self.got = True |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 160 | |
| 161 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 162 | def run(self, control_file, results_dir = '.', host = None, |
| 163 | timeout=None, tag=None, parallel_flag=False): |
| 164 | """ |
| 165 | Run an autotest job on the remote machine. |
mbligh | 9a3f5e5 | 2008-05-28 21:21:43 +0000 | [diff] [blame] | 166 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 167 | Args: |
| 168 | control_file: an open file-like-obj of the control file |
| 169 | results_dir: a str path where the results should be stored |
| 170 | on the local filesystem |
| 171 | host: a Host instance on which the control file should |
| 172 | be run |
| 173 | tag: tag name for the client side instance of autotest |
| 174 | parallel_flag: flag set when multiple jobs are run at the |
| 175 | same time |
| 176 | Raises: |
| 177 | AutotestRunError: if there is a problem executing |
| 178 | the control file |
| 179 | """ |
| 180 | host = self._get_host_and_setup(host) |
| 181 | results_dir = os.path.abspath(results_dir) |
mbligh | c1cbc99 | 2008-05-27 20:01:45 +0000 | [diff] [blame] | 182 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 183 | if tag: |
| 184 | results_dir = os.path.join(results_dir, tag) |
mbligh | c1cbc99 | 2008-05-27 20:01:45 +0000 | [diff] [blame] | 185 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 186 | atrun = _Run(host, results_dir, tag, parallel_flag) |
| 187 | self._do_run(control_file, results_dir, host, atrun, timeout) |
mbligh | d8b3925 | 2008-03-20 21:15:03 +0000 | [diff] [blame] | 188 | |
| 189 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 190 | def _get_host_and_setup(self, host): |
| 191 | if not host: |
| 192 | host = self.host |
| 193 | if not self.installed: |
| 194 | self.install(host) |
mbligh | 9133490 | 2007-09-28 01:47:59 +0000 | [diff] [blame] | 195 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 196 | host.wait_up(timeout=30) |
| 197 | return host |
mbligh | d8b3925 | 2008-03-20 21:15:03 +0000 | [diff] [blame] | 198 | |
| 199 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 200 | def prepare_for_copying_logs(self, src, dest, host): |
| 201 | keyval_path = '' |
| 202 | if not os.path.exists(os.path.join(dest, 'keyval')): |
| 203 | # Client-side keyval file can be copied directly |
| 204 | return keyval_path |
| 205 | # Copy client-side keyval to temporary location |
| 206 | try: |
| 207 | try: |
| 208 | # Create temp file |
mbligh | 78bf535 | 2008-07-11 20:27:36 +0000 | [diff] [blame] | 209 | fd, keyval_path = tempfile.mkstemp('.keyval_%s' % host.hostname) |
| 210 | host.get_file(os.path.join(src, 'keyval'), keyval_path) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 211 | finally: |
| 212 | # We will squirrel away the client side keyval |
| 213 | # away and move it back when we are done |
| 214 | self.temp_keyval_path = tempfile.mktemp() |
| 215 | host.run('mv %s %s' % |
mbligh | 78bf535 | 2008-07-11 20:27:36 +0000 | [diff] [blame] | 216 | (os.path.join(src, 'keyval'), self.temp_keyval_path)) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 217 | except (error.AutoservRunError, error.AutoservSSHTimeout): |
| 218 | print "Prepare for copying logs failed" |
| 219 | return keyval_path |
mbligh | caa62c2 | 2008-04-07 21:51:17 +0000 | [diff] [blame] | 220 | |
| 221 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 222 | def process_copied_logs(self, dest, host, keyval_path): |
| 223 | if not os.path.exists(os.path.join(dest, 'keyval')): |
| 224 | # Client-side keyval file was copied directly |
| 225 | return |
| 226 | # Append contents of keyval_<host> file to keyval file |
| 227 | try: |
| 228 | # Read in new and old keyval files |
| 229 | new_keyval = utils.read_keyval(keyval_path) |
| 230 | old_keyval = utils.read_keyval(dest) |
| 231 | # 'Delete' from new keyval entries that are in both |
| 232 | tmp_keyval = {} |
| 233 | for key, val in new_keyval.iteritems(): |
| 234 | if key not in old_keyval: |
| 235 | tmp_keyval[key] = val |
| 236 | # Append new info to keyval file |
| 237 | utils.write_keyval(dest, tmp_keyval) |
| 238 | # Delete keyval_<host> file |
| 239 | os.remove(keyval_path) |
| 240 | except IOError: |
| 241 | print "Process copied logs failed" |
mbligh | caa62c2 | 2008-04-07 21:51:17 +0000 | [diff] [blame] | 242 | |
| 243 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 244 | def postprocess_copied_logs(self, src, host): |
| 245 | # we can now put our keyval file back |
| 246 | try: |
| 247 | host.run('mv %s %s' % (self.temp_keyval_path, |
| 248 | os.path.join(src, 'keyval'))) |
| 249 | except: |
| 250 | pass |
mbligh | 9a3f5e5 | 2008-05-28 21:21:43 +0000 | [diff] [blame] | 251 | |
| 252 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 253 | def _do_run(self, control_file, results_dir, host, atrun, timeout): |
| 254 | try: |
| 255 | atrun.verify_machine() |
| 256 | except: |
mbligh | 78bf535 | 2008-07-11 20:27:36 +0000 | [diff] [blame] | 257 | print "Verify machine failed on %s. Reinstalling" % host.hostname |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 258 | self.install(host) |
| 259 | atrun.verify_machine() |
| 260 | debug = os.path.join(results_dir, 'debug') |
| 261 | try: |
| 262 | os.makedirs(debug) |
| 263 | except: |
| 264 | pass |
mbligh | 9a3f5e5 | 2008-05-28 21:21:43 +0000 | [diff] [blame] | 265 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 266 | # Ready .... Aim .... |
| 267 | for control in [atrun.remote_control_file, |
| 268 | atrun.remote_control_file + '.state', |
| 269 | atrun.manual_control_file, |
| 270 | atrun.manual_control_file + '.state']: |
| 271 | host.run('rm -f ' + control) |
mbligh | 9a3f5e5 | 2008-05-28 21:21:43 +0000 | [diff] [blame] | 272 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 273 | tmppath = utils.get(control_file) |
mbligh | c5ddfd1 | 2008-08-04 17:15:00 +0000 | [diff] [blame^] | 274 | |
| 275 | # Insert the job.add_repository() lines in the control file |
| 276 | # if there are any repos defined in global_config.ini |
| 277 | try: |
| 278 | cfile = open(tmppath, 'r') |
| 279 | cfile_orig = cfile.read() |
| 280 | cfile.close() |
| 281 | c = global_config.global_config |
| 282 | repos = c.get_config_value("PACKAGES", 'fetch_location', type=list) |
| 283 | control_file_new = [] |
| 284 | control_file_new.append('job.add_repository(%s)\n' % repos) |
| 285 | control_file_new.append(cfile_orig) |
| 286 | |
| 287 | # Overwrite the control file with the new one |
| 288 | cfile = open(tmppath, 'w') |
| 289 | cfile.write('\n'.join(control_file_new)) |
| 290 | cfile.close() |
| 291 | except global_config.ConfigError, e: |
| 292 | pass |
| 293 | |
| 294 | |
| 295 | # Copy control_file to remote_control_file on the host |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 296 | host.send_file(tmppath, atrun.remote_control_file) |
| 297 | if os.path.abspath(tmppath) != os.path.abspath(control_file): |
| 298 | os.remove(tmppath) |
mbligh | 0e4613b | 2007-10-29 16:55:07 +0000 | [diff] [blame] | 299 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 300 | try: |
| 301 | atrun.execute_control(timeout=timeout) |
| 302 | finally: |
| 303 | # make an effort to wait for the machine to come up |
| 304 | try: |
| 305 | host.wait_up(timeout=30) |
| 306 | except error.AutoservError: |
| 307 | # don't worry about any errors, we'll try and |
| 308 | # get the results anyway |
| 309 | pass |
mbligh | 1f19490 | 2007-12-20 01:31:43 +0000 | [diff] [blame] | 310 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 311 | # get the results |
| 312 | if not atrun.tag: |
mbligh | 78bf535 | 2008-07-11 20:27:36 +0000 | [diff] [blame] | 313 | results = os.path.join(atrun.autodir, 'results', 'default') |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 314 | else: |
mbligh | 78bf535 | 2008-07-11 20:27:36 +0000 | [diff] [blame] | 315 | results = os.path.join(atrun.autodir, 'results', atrun.tag) |
mbligh | c1cbc99 | 2008-05-27 20:01:45 +0000 | [diff] [blame] | 316 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 317 | # Copy all dirs in default to results_dir |
mbligh | 78bf535 | 2008-07-11 20:27:36 +0000 | [diff] [blame] | 318 | keyval_path = self.prepare_for_copying_logs(results, results_dir, |
| 319 | host) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 320 | host.get_file(results + '/', results_dir) |
| 321 | self.process_copied_logs(results_dir, host, keyval_path) |
| 322 | self.postprocess_copied_logs(results, host) |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 323 | |
mbligh | 0e4613b | 2007-10-29 16:55:07 +0000 | [diff] [blame] | 324 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 325 | def run_timed_test(self, test_name, results_dir='.', host=None, |
| 326 | timeout=None, tag=None, *args, **dargs): |
| 327 | """ |
| 328 | Assemble a tiny little control file to just run one test, |
| 329 | and run it as an autotest client-side test |
| 330 | """ |
| 331 | if not host: |
| 332 | host = self.host |
| 333 | if not self.installed: |
| 334 | self.install(host) |
| 335 | opts = ["%s=%s" % (o[0], repr(o[1])) for o in dargs.items()] |
| 336 | cmd = ", ".join([repr(test_name)] + map(repr, args) + opts) |
| 337 | control = "job.run_test(%s)\n" % cmd |
| 338 | self.run(control, results_dir, host, timeout=timeout, tag=tag) |
mbligh | 0e4613b | 2007-10-29 16:55:07 +0000 | [diff] [blame] | 339 | |
| 340 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 341 | def run_test(self, test_name, results_dir='.', host=None, |
mbligh | 78bf535 | 2008-07-11 20:27:36 +0000 | [diff] [blame] | 342 | tag=None, *args, **dargs): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 343 | self.run_timed_test(test_name, results_dir, host, timeout=None, |
| 344 | tag=tag, *args, **dargs) |
mbligh | d54832b | 2007-07-25 16:46:56 +0000 | [diff] [blame] | 345 | |
| 346 | |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 347 | class _Run(object): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 348 | """ |
| 349 | Represents a run of autotest control file. This class maintains |
| 350 | all the state necessary as an autotest control file is executed. |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 351 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 352 | It is not intended to be used directly, rather control files |
| 353 | should be run using the run method in Autotest. |
| 354 | """ |
| 355 | def __init__(self, host, results_dir, tag, parallel_flag): |
| 356 | self.host = host |
| 357 | self.results_dir = results_dir |
| 358 | self.env = host.env |
| 359 | self.tag = tag |
| 360 | self.parallel_flag = parallel_flag |
| 361 | self.autodir = _get_autodir(self.host) |
mbligh | 78bf535 | 2008-07-11 20:27:36 +0000 | [diff] [blame] | 362 | control = os.path.join(self.autodir, 'control') |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 363 | if tag: |
mbligh | 78bf535 | 2008-07-11 20:27:36 +0000 | [diff] [blame] | 364 | control += '.' + tag |
| 365 | self.manual_control_file = control |
| 366 | self.remote_control_file = control + '.autoserv' |
mbligh | dc735a2 | 2007-08-02 16:54:37 +0000 | [diff] [blame] | 367 | |
| 368 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 369 | def verify_machine(self): |
| 370 | binary = os.path.join(self.autodir, 'bin/autotest') |
| 371 | try: |
| 372 | self.host.run('ls %s > /dev/null 2>&1' % binary) |
| 373 | except: |
| 374 | raise "Autotest does not appear to be installed" |
mbligh | dc735a2 | 2007-08-02 16:54:37 +0000 | [diff] [blame] | 375 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 376 | if not self.parallel_flag: |
| 377 | tmpdir = os.path.join(self.autodir, 'tmp') |
| 378 | download = os.path.join(self.autodir, 'tests/download') |
| 379 | self.host.run('umount %s' % tmpdir, ignore_status=True) |
| 380 | self.host.run('umount %s' % download, ignore_status=True) |
mbligh | dc735a2 | 2007-08-02 16:54:37 +0000 | [diff] [blame] | 381 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 382 | def get_full_cmd(self, section): |
| 383 | # build up the full command we want to run over the host |
| 384 | cmd = [os.path.join(self.autodir, 'bin/autotest_client')] |
| 385 | if section > 0: |
| 386 | cmd.append('-c') |
| 387 | if self.tag: |
| 388 | cmd.append('-t %s' % self.tag) |
| 389 | if self.host.job.use_external_logging(): |
| 390 | cmd.append('-l') |
| 391 | cmd.append(self.remote_control_file) |
| 392 | return ' '.join(cmd) |
mbligh | adf2aab | 2007-11-29 18:16:43 +0000 | [diff] [blame] | 393 | |
mbligh | d8b3925 | 2008-03-20 21:15:03 +0000 | [diff] [blame] | 394 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 395 | def get_client_log(self, section): |
| 396 | # open up the files we need for our logging |
| 397 | client_log_file = os.path.join(self.results_dir, 'debug', |
| 398 | 'client.log.%d' % section) |
| 399 | return open(client_log_file, 'w', 0) |
mbligh | d8b3925 | 2008-03-20 21:15:03 +0000 | [diff] [blame] | 400 | |
| 401 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 402 | def execute_section(self, section, timeout): |
mbligh | 4e4961c | 2008-07-11 21:08:10 +0000 | [diff] [blame] | 403 | print "Executing %s/bin/autotest %s/control phase %d" % \ |
| 404 | (self.autodir, self.autodir, section) |
mbligh | d8b3925 | 2008-03-20 21:15:03 +0000 | [diff] [blame] | 405 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 406 | full_cmd = self.get_full_cmd(section) |
| 407 | client_log = self.get_client_log(section) |
| 408 | redirector = server_job.client_logger(self.host.job) |
mbligh | d528d30 | 2007-12-19 16:19:05 +0000 | [diff] [blame] | 409 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 410 | try: |
| 411 | old_resultdir = self.host.job.resultdir |
| 412 | self.host.job.resultdir = self.results_dir |
| 413 | result = self.host.run(full_cmd, ignore_status=True, |
| 414 | timeout=timeout, |
| 415 | stdout_tee=client_log, |
| 416 | stderr_tee=redirector) |
| 417 | finally: |
| 418 | redirector.close() |
| 419 | self.host.job.resultdir = old_resultdir |
mbligh | 2bf2db6 | 2007-11-27 00:53:18 +0000 | [diff] [blame] | 420 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 421 | if result.exit_status == 1: |
| 422 | self.host.job.aborted = True |
jadmanski | 8947606 | 2008-07-01 23:57:21 +0000 | [diff] [blame] | 423 | raise error.AutotestRunError("client job was aborted") |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 424 | if not result.stderr: |
| 425 | raise error.AutotestRunError( |
| 426 | "execute_section: %s failed to return anything\n" |
| 427 | "stdout:%s\n" % (full_cmd, result.stdout)) |
mbligh | 0e4613b | 2007-10-29 16:55:07 +0000 | [diff] [blame] | 428 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 429 | return redirector.last_line |
mbligh | dc735a2 | 2007-08-02 16:54:37 +0000 | [diff] [blame] | 430 | |
| 431 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 432 | def execute_control(self, timeout=None): |
| 433 | section = 0 |
| 434 | time_left = None |
| 435 | if timeout: |
| 436 | end_time = time.time() + timeout |
| 437 | time_left = end_time - time.time() |
| 438 | while not timeout or time_left > 0: |
| 439 | last = self.execute_section(section, time_left) |
| 440 | if timeout: |
| 441 | time_left = end_time - time.time() |
| 442 | if time_left <= 0: |
| 443 | break |
| 444 | section += 1 |
| 445 | if re.match(r'^END .*\t----\t----\t.*$', last): |
| 446 | print "Client complete" |
| 447 | return |
| 448 | elif re.match('^\t*GOOD\t----\treboot\.start.*$', last): |
| 449 | print "Client is rebooting" |
| 450 | print "Waiting for client to halt" |
| 451 | if not self.host.wait_down(HALT_TIME): |
mbligh | 78bf535 | 2008-07-11 20:27:36 +0000 | [diff] [blame] | 452 | err = "%s failed to shutdown after %d" % \ |
| 453 | (self.host.hostname, HALT_TIME) |
| 454 | raise error.AutotestRunError(err) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 455 | print "Client down, waiting for restart" |
| 456 | if not self.host.wait_up(BOOT_TIME): |
| 457 | # since reboot failed |
| 458 | # hardreset the machine once if possible |
| 459 | # before failing this control file |
mbligh | 78bf535 | 2008-07-11 20:27:36 +0000 | [diff] [blame] | 460 | print "Hardresetting %s" % self.host.hostname |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 461 | try: |
| 462 | self.host.hardreset(wait=False) |
| 463 | except error.AutoservUnsupportedError: |
mbligh | 78bf535 | 2008-07-11 20:27:36 +0000 | [diff] [blame] | 464 | print "Hardreset unsupported on %s" % self.host.hostname |
| 465 | raise error.AutotestRunError("%s failed to boot after %ds" % |
| 466 | (self.host.hostname, BOOT_TIME)) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 467 | self.host.reboot_followup() |
| 468 | continue |
jadmanski | c66e93c | 2008-07-29 21:25:22 +0000 | [diff] [blame] | 469 | self.host.job.record("END ABORT", None, None, |
mbligh | 78bf535 | 2008-07-11 20:27:36 +0000 | [diff] [blame] | 470 | "Autotest client terminated unexpectedly") |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 471 | # give the client machine a chance to recover from |
| 472 | # possible crash |
| 473 | self.host.wait_up(CRASH_RECOVERY_TIME) |
mbligh | 78bf535 | 2008-07-11 20:27:36 +0000 | [diff] [blame] | 474 | raise error.AutotestRunError("Aborting - unexpected final status " |
| 475 | "message from client: %s\n" % last) |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 476 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 477 | # should only get here if we timed out |
| 478 | assert timeout |
| 479 | raise error.AutotestTimeoutError() |
mbligh | 0e4613b | 2007-10-29 16:55:07 +0000 | [diff] [blame] | 480 | |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 481 | |
| 482 | def _get_autodir(host): |
mbligh | 3c7a150 | 2008-07-24 18:08:47 +0000 | [diff] [blame] | 483 | autodir = host.get_autodir() |
| 484 | if autodir: |
| 485 | return autodir |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 486 | try: |
| 487 | # There's no clean way to do this. readlink may not exist |
| 488 | cmd = "python -c 'import os,sys; print os.readlink(sys.argv[1])' /etc/autotest.conf 2> /dev/null" |
mbligh | 3c7a150 | 2008-07-24 18:08:47 +0000 | [diff] [blame] | 489 | autodir = os.path.dirname(host.run(cmd).stdout) |
| 490 | if autodir: |
| 491 | return autodir |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 492 | except error.AutoservRunError: |
| 493 | pass |
| 494 | for path in ['/usr/local/autotest', '/home/autotest']: |
| 495 | try: |
| 496 | host.run('ls %s > /dev/null 2>&1' % \ |
mbligh | 78bf535 | 2008-07-11 20:27:36 +0000 | [diff] [blame] | 497 | os.path.join(path, 'bin/autotest')) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 498 | return path |
| 499 | except error.AutoservRunError: |
| 500 | pass |
| 501 | raise error.AutotestRunError("Cannot figure out autotest directory") |
mbligh | d8b3925 | 2008-03-20 21:15:03 +0000 | [diff] [blame] | 502 | |
| 503 | |
| 504 | # site_autotest.py may be non-existant or empty, make sure that an appropriate |
| 505 | # SiteAutotest class is created nevertheless |
| 506 | try: |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 507 | from site_autotest import SiteAutotest |
mbligh | d8b3925 | 2008-03-20 21:15:03 +0000 | [diff] [blame] | 508 | except ImportError: |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 509 | class SiteAutotest(BaseAutotest): |
| 510 | pass |
mbligh | d8b3925 | 2008-03-20 21:15:03 +0000 | [diff] [blame] | 511 | |
| 512 | |
| 513 | class Autotest(SiteAutotest): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 514 | pass |