mbligh | 57e7866 | 2008-06-17 19:53:49 +0000 | [diff] [blame] | 1 | """ |
| 2 | The main job wrapper for the server side. |
| 3 | |
| 4 | This is the core infrastructure. Derived from the client side job.py |
| 5 | |
| 6 | Copyright Martin J. Bligh, Andy Whitcroft 2007 |
| 7 | """ |
| 8 | |
jadmanski | 025099d | 2008-09-23 14:13:48 +0000 | [diff] [blame] | 9 | import getpass, os, sys, re, stat, tempfile, time, select, subprocess, traceback |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 10 | import shutil, warnings |
jadmanski | c09fc15 | 2008-10-15 17:56:59 +0000 | [diff] [blame] | 11 | from autotest_lib.client.bin import fd_stack, sysinfo |
mbligh | 0910844 | 2008-10-15 16:27:38 +0000 | [diff] [blame] | 12 | from autotest_lib.client.common_lib import error, log, utils, packages |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 13 | from autotest_lib.server import test, subcommand, profilers |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 14 | from autotest_lib.tko import db as tko_db, status_lib, utils as tko_utils |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 15 | |
| 16 | |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 17 | def _control_segment_path(name): |
| 18 | """Get the pathname of the named control segment file.""" |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 19 | server_dir = os.path.dirname(os.path.abspath(__file__)) |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 20 | return os.path.join(server_dir, "control_segments", name) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 21 | |
| 22 | |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 23 | CLIENT_CONTROL_FILENAME = 'control' |
| 24 | SERVER_CONTROL_FILENAME = 'control.srv' |
| 25 | MACHINES_FILENAME = '.machines' |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 26 | |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 27 | CLIENT_WRAPPER_CONTROL_FILE = _control_segment_path('client_wrapper') |
| 28 | CRASHDUMPS_CONTROL_FILE = _control_segment_path('crashdumps') |
| 29 | CRASHINFO_CONTROL_FILE = _control_segment_path('crashinfo') |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 30 | INSTALL_CONTROL_FILE = _control_segment_path('install') |
showard | 45ae819 | 2008-11-05 19:32:53 +0000 | [diff] [blame] | 31 | CLEANUP_CONTROL_FILE = _control_segment_path('cleanup') |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 32 | |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 33 | VERIFY_CONTROL_FILE = _control_segment_path('verify') |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 34 | REPAIR_CONTROL_FILE = _control_segment_path('repair') |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 35 | |
| 36 | |
mbligh | 062ed15 | 2009-01-13 00:57:14 +0000 | [diff] [blame^] | 37 | # by default provide a stub that generates no site data |
| 38 | def _get_site_job_data_dummy(job): |
| 39 | return {} |
| 40 | |
| 41 | |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 42 | # load up site-specific code for generating site-specific job data |
mbligh | 062ed15 | 2009-01-13 00:57:14 +0000 | [diff] [blame^] | 43 | get_site_job_data = utils.import_site_function(__file__, |
| 44 | "autotest_lib.server.site_job", "get_site_job_data", |
| 45 | _get_site_job_data_dummy) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 46 | |
| 47 | |
| 48 | class base_server_job(object): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 49 | """ |
| 50 | The actual job against which we do everything. |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 51 | |
| 52 | Properties: |
| 53 | autodir |
| 54 | The top level autotest directory (/usr/local/autotest). |
| 55 | serverdir |
| 56 | <autodir>/server/ |
| 57 | clientdir |
| 58 | <autodir>/client/ |
| 59 | conmuxdir |
| 60 | <autodir>/conmux/ |
| 61 | testdir |
| 62 | <autodir>/server/tests/ |
| 63 | site_testdir |
| 64 | <autodir>/server/site_tests/ |
| 65 | control |
| 66 | the control file for this job |
mbligh | b5dac43 | 2008-11-27 00:38:44 +0000 | [diff] [blame] | 67 | drop_caches_between_iterations |
| 68 | drop the pagecache between each iteration |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 69 | """ |
| 70 | |
| 71 | STATUS_VERSION = 1 |
| 72 | |
| 73 | |
| 74 | def __init__(self, control, args, resultdir, label, user, machines, |
| 75 | client=False, parse_job='', |
| 76 | ssh_user='root', ssh_port=22, ssh_pass=''): |
| 77 | """ |
mbligh | b5dac43 | 2008-11-27 00:38:44 +0000 | [diff] [blame] | 78 | Server side job object. |
| 79 | |
| 80 | Parameters: |
| 81 | control: The control file (pathname of) |
| 82 | args: args to pass to the control file |
| 83 | resultdir: where to throw the results |
| 84 | label: label for the job |
| 85 | user: Username for the job (email address) |
| 86 | client: True if a client-side control file |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 87 | """ |
| 88 | path = os.path.dirname(__file__) |
| 89 | self.autodir = os.path.abspath(os.path.join(path, '..')) |
| 90 | self.serverdir = os.path.join(self.autodir, 'server') |
| 91 | self.testdir = os.path.join(self.serverdir, 'tests') |
| 92 | self.site_testdir = os.path.join(self.serverdir, 'site_tests') |
| 93 | self.tmpdir = os.path.join(self.serverdir, 'tmp') |
| 94 | self.conmuxdir = os.path.join(self.autodir, 'conmux') |
| 95 | self.clientdir = os.path.join(self.autodir, 'client') |
| 96 | self.toolsdir = os.path.join(self.autodir, 'client/tools') |
| 97 | if control: |
| 98 | self.control = open(control, 'r').read() |
| 99 | self.control = re.sub('\r', '', self.control) |
| 100 | else: |
showard | 45ae819 | 2008-11-05 19:32:53 +0000 | [diff] [blame] | 101 | self.control = '' |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 102 | self.resultdir = resultdir |
mbligh | 80e1eba | 2008-11-19 00:26:18 +0000 | [diff] [blame] | 103 | if resultdir: |
| 104 | if not os.path.exists(resultdir): |
| 105 | os.mkdir(resultdir) |
| 106 | self.debugdir = os.path.join(resultdir, 'debug') |
| 107 | if not os.path.exists(self.debugdir): |
| 108 | os.mkdir(self.debugdir) |
| 109 | self.status = os.path.join(resultdir, 'status') |
| 110 | else: |
| 111 | self.status = None |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 112 | self.label = label |
| 113 | self.user = user |
| 114 | self.args = args |
| 115 | self.machines = machines |
| 116 | self.client = client |
| 117 | self.record_prefix = '' |
| 118 | self.warning_loggers = set() |
| 119 | self.ssh_user = ssh_user |
| 120 | self.ssh_port = ssh_port |
| 121 | self.ssh_pass = ssh_pass |
jadmanski | 23afbec | 2008-09-17 18:12:07 +0000 | [diff] [blame] | 122 | self.run_test_cleanup = True |
mbligh | 0910844 | 2008-10-15 16:27:38 +0000 | [diff] [blame] | 123 | self.last_boot_tag = None |
jadmanski | 53aaf38 | 2008-11-17 16:22:31 +0000 | [diff] [blame] | 124 | self.hosts = set() |
mbligh | b5dac43 | 2008-11-27 00:38:44 +0000 | [diff] [blame] | 125 | self.drop_caches_between_iterations = False |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 126 | |
| 127 | self.stdout = fd_stack.fd_stack(1, sys.stdout) |
| 128 | self.stderr = fd_stack.fd_stack(2, sys.stderr) |
| 129 | |
mbligh | 80e1eba | 2008-11-19 00:26:18 +0000 | [diff] [blame] | 130 | if resultdir: |
| 131 | self.sysinfo = sysinfo.sysinfo(self.resultdir) |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 132 | self.profilers = profilers.profilers(self) |
jadmanski | c09fc15 | 2008-10-15 17:56:59 +0000 | [diff] [blame] | 133 | |
jadmanski | 025099d | 2008-09-23 14:13:48 +0000 | [diff] [blame] | 134 | if not os.access(self.tmpdir, os.W_OK): |
| 135 | try: |
| 136 | os.makedirs(self.tmpdir, 0700) |
| 137 | except os.error, e: |
| 138 | # Thrown if the directory already exists, which it may. |
| 139 | pass |
| 140 | |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 141 | if not (os.access(self.tmpdir, os.W_OK) and os.path.isdir(self.tmpdir)): |
jadmanski | 025099d | 2008-09-23 14:13:48 +0000 | [diff] [blame] | 142 | self.tmpdir = os.path.join(tempfile.gettempdir(), |
| 143 | 'autotest-' + getpass.getuser()) |
| 144 | try: |
| 145 | os.makedirs(self.tmpdir, 0700) |
| 146 | except os.error, e: |
| 147 | # Thrown if the directory already exists, which it may. |
| 148 | # If the problem was something other than the |
| 149 | # directory already existing, this chmod should throw as well |
| 150 | # exception. |
| 151 | os.chmod(self.tmpdir, stat.S_IRWXU) |
| 152 | |
mbligh | 80e1eba | 2008-11-19 00:26:18 +0000 | [diff] [blame] | 153 | if self.status and os.path.exists(self.status): |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 154 | os.unlink(self.status) |
| 155 | job_data = {'label' : label, 'user' : user, |
| 156 | 'hostname' : ','.join(machines), |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 157 | 'status_version' : str(self.STATUS_VERSION), |
| 158 | 'job_started' : str(int(time.time()))} |
mbligh | 80e1eba | 2008-11-19 00:26:18 +0000 | [diff] [blame] | 159 | if self.resultdir: |
| 160 | job_data.update(get_site_job_data(self)) |
| 161 | utils.write_keyval(self.resultdir, job_data) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 162 | |
| 163 | self.parse_job = parse_job |
| 164 | if self.parse_job and len(machines) == 1: |
| 165 | self.using_parser = True |
| 166 | self.init_parser(resultdir) |
| 167 | else: |
| 168 | self.using_parser = False |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 169 | self.pkgmgr = packages.PackageManager(self.autodir, |
| 170 | run_function_dargs={'timeout':600}) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 171 | self.pkgdir = os.path.join(self.autodir, 'packages') |
| 172 | |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 173 | self.num_tests_run = 0 |
| 174 | self.num_tests_failed = 0 |
| 175 | |
jadmanski | 550fdc2 | 2008-11-20 16:32:08 +0000 | [diff] [blame] | 176 | self._register_subcommand_hooks() |
| 177 | |
| 178 | |
| 179 | def _register_subcommand_hooks(self): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 180 | """ |
| 181 | Register some hooks into the subcommand modules that allow us |
| 182 | to properly clean up self.hosts created in forked subprocesses. |
| 183 | """ |
jadmanski | 550fdc2 | 2008-11-20 16:32:08 +0000 | [diff] [blame] | 184 | def on_fork(cmd): |
| 185 | self._existing_hosts_on_fork = set(self.hosts) |
| 186 | def on_join(cmd): |
| 187 | new_hosts = self.hosts - self._existing_hosts_on_fork |
| 188 | for host in new_hosts: |
| 189 | host.close() |
| 190 | subcommand.subcommand.register_fork_hook(on_fork) |
| 191 | subcommand.subcommand.register_join_hook(on_join) |
| 192 | |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 193 | |
| 194 | def init_parser(self, resultdir): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 195 | """ |
| 196 | Start the continuous parsing of resultdir. This sets up |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 197 | the database connection and inserts the basic job object into |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 198 | the database if necessary. |
| 199 | """ |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 200 | # redirect parser debugging to .parse.log |
| 201 | parse_log = os.path.join(resultdir, '.parse.log') |
| 202 | parse_log = open(parse_log, 'w', 0) |
| 203 | tko_utils.redirect_parser_debugging(parse_log) |
| 204 | # create a job model object and set up the db |
| 205 | self.results_db = tko_db.db(autocommit=True) |
| 206 | self.parser = status_lib.parser(self.STATUS_VERSION) |
| 207 | self.job_model = self.parser.make_job(resultdir) |
| 208 | self.parser.start(self.job_model) |
| 209 | # check if a job already exists in the db and insert it if |
| 210 | # it does not |
| 211 | job_idx = self.results_db.find_job(self.parse_job) |
| 212 | if job_idx is None: |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 213 | self.results_db.insert_job(self.parse_job, self.job_model) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 214 | else: |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 215 | machine_idx = self.results_db.lookup_machine(self.job_model.machine) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 216 | self.job_model.index = job_idx |
| 217 | self.job_model.machine_idx = machine_idx |
| 218 | |
| 219 | |
| 220 | def cleanup_parser(self): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 221 | """ |
| 222 | This should be called after the server job is finished |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 223 | to carry out any remaining cleanup (e.g. flushing any |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 224 | remaining test results to the results db) |
| 225 | """ |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 226 | if not self.using_parser: |
| 227 | return |
| 228 | final_tests = self.parser.end() |
| 229 | for test in final_tests: |
| 230 | self.__insert_test(test) |
| 231 | self.using_parser = False |
| 232 | |
| 233 | |
| 234 | def verify(self): |
| 235 | if not self.machines: |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 236 | raise error.AutoservError('No machines specified to verify') |
mbligh | 0fce411 | 2008-11-27 00:37:17 +0000 | [diff] [blame] | 237 | if self.resultdir: |
| 238 | os.chdir(self.resultdir) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 239 | try: |
jadmanski | cdd0c40 | 2008-09-19 21:21:31 +0000 | [diff] [blame] | 240 | namespace = {'machines' : self.machines, 'job' : self, |
| 241 | 'ssh_user' : self.ssh_user, |
| 242 | 'ssh_port' : self.ssh_port, |
| 243 | 'ssh_pass' : self.ssh_pass} |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 244 | self._execute_code(VERIFY_CONTROL_FILE, namespace, protect=False) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 245 | except Exception, e: |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 246 | msg = ('Verify failed\n' + str(e) + '\n' + traceback.format_exc()) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 247 | self.record('ABORT', None, None, msg) |
| 248 | raise |
| 249 | |
| 250 | |
| 251 | def repair(self, host_protection): |
| 252 | if not self.machines: |
| 253 | raise error.AutoservError('No machines specified to repair') |
mbligh | 0fce411 | 2008-11-27 00:37:17 +0000 | [diff] [blame] | 254 | if self.resultdir: |
| 255 | os.chdir(self.resultdir) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 256 | namespace = {'machines': self.machines, 'job': self, |
| 257 | 'ssh_user': self.ssh_user, 'ssh_port': self.ssh_port, |
| 258 | 'ssh_pass': self.ssh_pass, |
| 259 | 'protection_level': host_protection} |
| 260 | # no matter what happens during repair, go on to try to reverify |
| 261 | try: |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 262 | self._execute_code(REPAIR_CONTROL_FILE, namespace, protect=False) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 263 | except Exception, exc: |
| 264 | print 'Exception occured during repair' |
| 265 | traceback.print_exc() |
| 266 | self.verify() |
| 267 | |
| 268 | |
| 269 | def precheck(self): |
| 270 | """ |
| 271 | perform any additional checks in derived classes. |
| 272 | """ |
| 273 | pass |
| 274 | |
| 275 | |
| 276 | def enable_external_logging(self): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 277 | """ |
| 278 | Start or restart external logging mechanism. |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 279 | """ |
| 280 | pass |
| 281 | |
| 282 | |
| 283 | def disable_external_logging(self): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 284 | """ |
| 285 | Pause or stop external logging mechanism. |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 286 | """ |
| 287 | pass |
| 288 | |
| 289 | |
jadmanski | 23afbec | 2008-09-17 18:12:07 +0000 | [diff] [blame] | 290 | def enable_test_cleanup(self): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 291 | """ |
| 292 | By default tests run test.cleanup |
| 293 | """ |
jadmanski | 23afbec | 2008-09-17 18:12:07 +0000 | [diff] [blame] | 294 | self.run_test_cleanup = True |
| 295 | |
| 296 | |
| 297 | def disable_test_cleanup(self): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 298 | """ |
| 299 | By default tests do not run test.cleanup |
| 300 | """ |
jadmanski | 23afbec | 2008-09-17 18:12:07 +0000 | [diff] [blame] | 301 | self.run_test_cleanup = False |
| 302 | |
| 303 | |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 304 | def use_external_logging(self): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 305 | """ |
| 306 | Return True if external logging should be used. |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 307 | """ |
| 308 | return False |
| 309 | |
| 310 | |
| 311 | def parallel_simple(self, function, machines, log=True, timeout=None): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 312 | """ |
| 313 | Run 'function' using parallel_simple, with an extra wrapper to handle |
| 314 | the necessary setup for continuous parsing, if possible. If continuous |
| 315 | parsing is already properly initialized then this should just work. |
| 316 | """ |
| 317 | is_forking = not (len(machines) == 1 and self.machines == machines) |
jadmanski | 4dd1a00 | 2008-09-05 20:27:30 +0000 | [diff] [blame] | 318 | if self.parse_job and is_forking and log: |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 319 | def wrapper(machine): |
| 320 | self.parse_job += "/" + machine |
| 321 | self.using_parser = True |
| 322 | self.machines = [machine] |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 323 | self.resultdir = os.path.join(self.resultdir, machine) |
jadmanski | 609a5f4 | 2008-08-26 20:52:42 +0000 | [diff] [blame] | 324 | os.chdir(self.resultdir) |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 325 | utils.write_keyval(self.resultdir, {"hostname": machine}) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 326 | self.init_parser(self.resultdir) |
| 327 | result = function(machine) |
| 328 | self.cleanup_parser() |
| 329 | return result |
jadmanski | 4dd1a00 | 2008-09-05 20:27:30 +0000 | [diff] [blame] | 330 | elif len(machines) > 1 and log: |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 331 | def wrapper(machine): |
| 332 | self.resultdir = os.path.join(self.resultdir, machine) |
jadmanski | 609a5f4 | 2008-08-26 20:52:42 +0000 | [diff] [blame] | 333 | os.chdir(self.resultdir) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 334 | result = function(machine) |
| 335 | return result |
| 336 | else: |
| 337 | wrapper = function |
| 338 | subcommand.parallel_simple(wrapper, machines, log, timeout) |
| 339 | |
| 340 | |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 341 | def run(self, cleanup=False, install_before=False, install_after=False, |
| 342 | collect_crashdumps=True, namespace={}): |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 343 | # use a copy so changes don't affect the original dictionary |
| 344 | namespace = namespace.copy() |
| 345 | machines = self.machines |
| 346 | |
| 347 | self.aborted = False |
| 348 | namespace['machines'] = machines |
| 349 | namespace['args'] = self.args |
| 350 | namespace['job'] = self |
| 351 | namespace['ssh_user'] = self.ssh_user |
| 352 | namespace['ssh_port'] = self.ssh_port |
| 353 | namespace['ssh_pass'] = self.ssh_pass |
| 354 | test_start_time = int(time.time()) |
| 355 | |
mbligh | 80e1eba | 2008-11-19 00:26:18 +0000 | [diff] [blame] | 356 | if self.resultdir: |
| 357 | os.chdir(self.resultdir) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 358 | |
mbligh | 80e1eba | 2008-11-19 00:26:18 +0000 | [diff] [blame] | 359 | self.enable_external_logging() |
| 360 | status_log = os.path.join(self.resultdir, 'status.log') |
jadmanski | cdd0c40 | 2008-09-19 21:21:31 +0000 | [diff] [blame] | 361 | collect_crashinfo = True |
mbligh | aebe3b6 | 2008-12-22 14:45:40 +0000 | [diff] [blame] | 362 | temp_control_file_dir = None |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 363 | try: |
| 364 | if install_before and machines: |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 365 | self._execute_code(INSTALL_CONTROL_FILE, namespace) |
mbligh | aebe3b6 | 2008-12-22 14:45:40 +0000 | [diff] [blame] | 366 | if self.resultdir: |
| 367 | server_control_file = SERVER_CONTROL_FILENAME |
| 368 | client_control_file = CLIENT_CONTROL_FILENAME |
| 369 | else: |
mbligh | ac367cc | 2008-12-22 14:54:00 +0000 | [diff] [blame] | 370 | temp_control_file_dir = tempfile.mkdtemp() |
mbligh | aebe3b6 | 2008-12-22 14:45:40 +0000 | [diff] [blame] | 371 | server_control_file = os.path.join(temp_control_file_dir, |
| 372 | SERVER_CONTROL_FILENAME) |
| 373 | client_control_file = os.path.join(temp_control_file_dir, |
| 374 | CLIENT_CONTROL_FILENAME) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 375 | if self.client: |
| 376 | namespace['control'] = self.control |
mbligh | aebe3b6 | 2008-12-22 14:45:40 +0000 | [diff] [blame] | 377 | utils.open_write_close(client_control_file, self.control) |
| 378 | shutil.copy(CLIENT_WRAPPER_CONTROL_FILE, server_control_file) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 379 | else: |
mbligh | 181b7c2 | 2008-11-22 14:22:08 +0000 | [diff] [blame] | 380 | namespace['utils'] = utils |
mbligh | aebe3b6 | 2008-12-22 14:45:40 +0000 | [diff] [blame] | 381 | utils.open_write_close(server_control_file, self.control) |
| 382 | self._execute_code(server_control_file, namespace) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 383 | |
jadmanski | cdd0c40 | 2008-09-19 21:21:31 +0000 | [diff] [blame] | 384 | # disable crashinfo collection if we get this far without error |
| 385 | collect_crashinfo = False |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 386 | finally: |
mbligh | aebe3b6 | 2008-12-22 14:45:40 +0000 | [diff] [blame] | 387 | if temp_control_file_dir: |
| 388 | # Clean up temp. directory used for copies of the control files. |
| 389 | try: |
| 390 | shutil.rmtree(temp_control_file_dir) |
| 391 | except Exception, e: |
| 392 | print 'Error', e, 'removing dir', temp_control_file_dir |
jadmanski | cdd0c40 | 2008-09-19 21:21:31 +0000 | [diff] [blame] | 393 | if machines and (collect_crashdumps or collect_crashinfo): |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 394 | namespace['test_start_time'] = test_start_time |
jadmanski | cdd0c40 | 2008-09-19 21:21:31 +0000 | [diff] [blame] | 395 | if collect_crashinfo: |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 396 | # includes crashdumps |
| 397 | self._execute_code(CRASHINFO_CONTROL_FILE, namespace) |
jadmanski | cdd0c40 | 2008-09-19 21:21:31 +0000 | [diff] [blame] | 398 | else: |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 399 | self._execute_code(CRASHDUMPS_CONTROL_FILE, namespace) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 400 | self.disable_external_logging() |
showard | 45ae819 | 2008-11-05 19:32:53 +0000 | [diff] [blame] | 401 | if cleanup and machines: |
| 402 | self._execute_code(CLEANUP_CONTROL_FILE, namespace) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 403 | if install_after and machines: |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 404 | self._execute_code(INSTALL_CONTROL_FILE, namespace) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 405 | |
| 406 | |
| 407 | def run_test(self, url, *args, **dargs): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 408 | """ |
| 409 | Summon a test object and run it. |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 410 | |
| 411 | tag |
| 412 | tag to add to testname |
| 413 | url |
| 414 | url of the test to run |
| 415 | """ |
| 416 | |
| 417 | (group, testname) = self.pkgmgr.get_package_name(url, 'test') |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 418 | |
| 419 | tag = dargs.pop('tag', None) |
| 420 | if tag: |
mbligh | 8ad2420 | 2009-01-07 16:49:36 +0000 | [diff] [blame] | 421 | testname += '.' + str(tag) |
jadmanski | de292df | 2008-08-26 20:51:14 +0000 | [diff] [blame] | 422 | subdir = testname |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 423 | |
| 424 | outputdir = os.path.join(self.resultdir, subdir) |
| 425 | if os.path.exists(outputdir): |
| 426 | msg = ("%s already exists, test <%s> may have" |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 427 | " already run with tag <%s>" % (outputdir, testname, tag)) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 428 | raise error.TestError(msg) |
| 429 | os.mkdir(outputdir) |
| 430 | |
| 431 | def group_func(): |
| 432 | try: |
| 433 | test.runtest(self, url, tag, args, dargs) |
| 434 | except error.TestBaseException, e: |
| 435 | self.record(e.exit_status, subdir, testname, str(e)) |
| 436 | raise |
| 437 | except Exception, e: |
| 438 | info = str(e) + "\n" + traceback.format_exc() |
| 439 | self.record('FAIL', subdir, testname, info) |
| 440 | raise |
| 441 | else: |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 442 | self.record('GOOD', subdir, testname, 'completed successfully') |
jadmanski | de292df | 2008-08-26 20:51:14 +0000 | [diff] [blame] | 443 | |
| 444 | result, exc_info = self._run_group(testname, subdir, group_func) |
| 445 | if exc_info and isinstance(exc_info[1], error.TestBaseException): |
| 446 | return False |
| 447 | elif exc_info: |
| 448 | raise exc_info[0], exc_info[1], exc_info[2] |
| 449 | else: |
| 450 | return True |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 451 | |
| 452 | |
| 453 | def _run_group(self, name, subdir, function, *args, **dargs): |
| 454 | """\ |
| 455 | Underlying method for running something inside of a group. |
| 456 | """ |
jadmanski | de292df | 2008-08-26 20:51:14 +0000 | [diff] [blame] | 457 | result, exc_info = None, None |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 458 | old_record_prefix = self.record_prefix |
| 459 | try: |
| 460 | self.record('START', subdir, name) |
| 461 | self.record_prefix += '\t' |
| 462 | try: |
| 463 | result = function(*args, **dargs) |
| 464 | finally: |
| 465 | self.record_prefix = old_record_prefix |
| 466 | except error.TestBaseException, e: |
jadmanski | b88d6dc | 2009-01-10 00:33:18 +0000 | [diff] [blame] | 467 | self.record("END %s" % e.exit_status, subdir, name) |
jadmanski | de292df | 2008-08-26 20:51:14 +0000 | [diff] [blame] | 468 | exc_info = sys.exc_info() |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 469 | except Exception, e: |
| 470 | err_msg = str(e) + '\n' |
| 471 | err_msg += traceback.format_exc() |
| 472 | self.record('END ABORT', subdir, name, err_msg) |
| 473 | raise error.JobError(name + ' failed\n' + traceback.format_exc()) |
| 474 | else: |
| 475 | self.record('END GOOD', subdir, name) |
| 476 | |
jadmanski | de292df | 2008-08-26 20:51:14 +0000 | [diff] [blame] | 477 | return result, exc_info |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 478 | |
| 479 | |
| 480 | def run_group(self, function, *args, **dargs): |
| 481 | """\ |
| 482 | function: |
| 483 | subroutine to run |
| 484 | *args: |
| 485 | arguments for the function |
| 486 | """ |
| 487 | |
| 488 | name = function.__name__ |
| 489 | |
| 490 | # Allow the tag for the group to be specified. |
| 491 | tag = dargs.pop('tag', None) |
| 492 | if tag: |
| 493 | name = tag |
| 494 | |
jadmanski | de292df | 2008-08-26 20:51:14 +0000 | [diff] [blame] | 495 | return self._run_group(name, None, function, *args, **dargs)[0] |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 496 | |
| 497 | |
| 498 | def run_reboot(self, reboot_func, get_kernel_func): |
| 499 | """\ |
| 500 | A specialization of run_group meant specifically for handling |
| 501 | a reboot. Includes support for capturing the kernel version |
| 502 | after the reboot. |
| 503 | |
| 504 | reboot_func: a function that carries out the reboot |
| 505 | |
| 506 | get_kernel_func: a function that returns a string |
| 507 | representing the kernel version. |
| 508 | """ |
| 509 | |
| 510 | old_record_prefix = self.record_prefix |
| 511 | try: |
| 512 | self.record('START', None, 'reboot') |
| 513 | self.record_prefix += '\t' |
| 514 | reboot_func() |
| 515 | except Exception, e: |
| 516 | self.record_prefix = old_record_prefix |
| 517 | err_msg = str(e) + '\n' + traceback.format_exc() |
| 518 | self.record('END FAIL', None, 'reboot', err_msg) |
| 519 | else: |
| 520 | kernel = get_kernel_func() |
| 521 | self.record_prefix = old_record_prefix |
| 522 | self.record('END GOOD', None, 'reboot', |
| 523 | optional_fields={"kernel": kernel}) |
| 524 | |
| 525 | |
jadmanski | c09fc15 | 2008-10-15 17:56:59 +0000 | [diff] [blame] | 526 | def add_sysinfo_command(self, command, logfile=None, on_every_test=False): |
| 527 | self._add_sysinfo_loggable(sysinfo.command(command, logfile), |
| 528 | on_every_test) |
| 529 | |
| 530 | |
| 531 | def add_sysinfo_logfile(self, file, on_every_test=False): |
| 532 | self._add_sysinfo_loggable(sysinfo.logfile(file), on_every_test) |
| 533 | |
| 534 | |
| 535 | def _add_sysinfo_loggable(self, loggable, on_every_test): |
| 536 | if on_every_test: |
| 537 | self.sysinfo.test_loggables.add(loggable) |
| 538 | else: |
| 539 | self.sysinfo.boot_loggables.add(loggable) |
| 540 | |
| 541 | |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 542 | def record(self, status_code, subdir, operation, status='', |
| 543 | optional_fields=None): |
| 544 | """ |
| 545 | Record job-level status |
| 546 | |
| 547 | The intent is to make this file both machine parseable and |
| 548 | human readable. That involves a little more complexity, but |
| 549 | really isn't all that bad ;-) |
| 550 | |
| 551 | Format is <status code>\t<subdir>\t<operation>\t<status> |
| 552 | |
mbligh | 1b3b376 | 2008-09-25 02:46:34 +0000 | [diff] [blame] | 553 | status code: see common_lib.log.is_valid_status() |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 554 | for valid status definition |
| 555 | |
| 556 | subdir: MUST be a relevant subdirectory in the results, |
| 557 | or None, which will be represented as '----' |
| 558 | |
| 559 | operation: description of what you ran (e.g. "dbench", or |
| 560 | "mkfs -t foobar /dev/sda9") |
| 561 | |
| 562 | status: error message or "completed sucessfully" |
| 563 | |
| 564 | ------------------------------------------------------------ |
| 565 | |
| 566 | Initial tabs indicate indent levels for grouping, and is |
| 567 | governed by self.record_prefix |
| 568 | |
| 569 | multiline messages have secondary lines prefaced by a double |
| 570 | space (' ') |
| 571 | |
| 572 | Executing this method will trigger the logging of all new |
| 573 | warnings to date from the various console loggers. |
| 574 | """ |
| 575 | # poll all our warning loggers for new warnings |
| 576 | warnings = self._read_warnings() |
| 577 | for timestamp, msg in warnings: |
| 578 | self._record("WARN", None, None, msg, timestamp) |
| 579 | |
| 580 | # write out the actual status log line |
| 581 | self._record(status_code, subdir, operation, status, |
| 582 | optional_fields=optional_fields) |
| 583 | |
| 584 | |
| 585 | def _read_warnings(self): |
| 586 | warnings = [] |
| 587 | while True: |
| 588 | # pull in a line of output from every logger that has |
| 589 | # output ready to be read |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 590 | loggers, _, _ = select.select(self.warning_loggers, [], [], 0) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 591 | closed_loggers = set() |
| 592 | for logger in loggers: |
| 593 | line = logger.readline() |
| 594 | # record any broken pipes (aka line == empty) |
| 595 | if len(line) == 0: |
| 596 | closed_loggers.add(logger) |
| 597 | continue |
| 598 | timestamp, msg = line.split('\t', 1) |
| 599 | warnings.append((int(timestamp), msg.strip())) |
| 600 | |
| 601 | # stop listening to loggers that are closed |
| 602 | self.warning_loggers -= closed_loggers |
| 603 | |
| 604 | # stop if none of the loggers have any output left |
| 605 | if not loggers: |
| 606 | break |
| 607 | |
| 608 | # sort into timestamp order |
| 609 | warnings.sort() |
| 610 | return warnings |
| 611 | |
| 612 | |
| 613 | def _render_record(self, status_code, subdir, operation, status='', |
| 614 | epoch_time=None, record_prefix=None, |
| 615 | optional_fields=None): |
| 616 | """ |
| 617 | Internal Function to generate a record to be written into a |
| 618 | status log. For use by server_job.* classes only. |
| 619 | """ |
| 620 | if subdir: |
| 621 | if re.match(r'[\n\t]', subdir): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 622 | raise ValueError('Invalid character in subdir string') |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 623 | substr = subdir |
| 624 | else: |
| 625 | substr = '----' |
| 626 | |
mbligh | 1b3b376 | 2008-09-25 02:46:34 +0000 | [diff] [blame] | 627 | if not log.is_valid_status(status_code): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 628 | raise ValueError('Invalid status code supplied: %s' % status_code) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 629 | if not operation: |
| 630 | operation = '----' |
| 631 | if re.match(r'[\n\t]', operation): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 632 | raise ValueError('Invalid character in operation string') |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 633 | operation = operation.rstrip() |
| 634 | status = status.rstrip() |
| 635 | status = re.sub(r"\t", " ", status) |
| 636 | # Ensure any continuation lines are marked so we can |
| 637 | # detect them in the status file to ensure it is parsable. |
| 638 | status = re.sub(r"\n", "\n" + self.record_prefix + " ", status) |
| 639 | |
| 640 | if not optional_fields: |
| 641 | optional_fields = {} |
| 642 | |
| 643 | # Generate timestamps for inclusion in the logs |
| 644 | if epoch_time is None: |
| 645 | epoch_time = int(time.time()) |
| 646 | local_time = time.localtime(epoch_time) |
| 647 | optional_fields["timestamp"] = str(epoch_time) |
| 648 | optional_fields["localtime"] = time.strftime("%b %d %H:%M:%S", |
| 649 | local_time) |
| 650 | |
| 651 | fields = [status_code, substr, operation] |
| 652 | fields += ["%s=%s" % x for x in optional_fields.iteritems()] |
| 653 | fields.append(status) |
| 654 | |
| 655 | if record_prefix is None: |
| 656 | record_prefix = self.record_prefix |
| 657 | |
| 658 | msg = '\t'.join(str(x) for x in fields) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 659 | return record_prefix + msg + '\n' |
| 660 | |
| 661 | |
| 662 | def _record_prerendered(self, msg): |
| 663 | """ |
| 664 | Record a pre-rendered msg into the status logs. The only |
| 665 | change this makes to the message is to add on the local |
| 666 | indentation. Should not be called outside of server_job.* |
| 667 | classes. Unlike _record, this does not write the message |
| 668 | to standard output. |
| 669 | """ |
| 670 | lines = [] |
| 671 | status_file = os.path.join(self.resultdir, 'status.log') |
| 672 | status_log = open(status_file, 'a') |
| 673 | for line in msg.splitlines(): |
| 674 | line = self.record_prefix + line + '\n' |
| 675 | lines.append(line) |
| 676 | status_log.write(line) |
| 677 | status_log.close() |
| 678 | self.__parse_status(lines) |
| 679 | |
| 680 | |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 681 | def _fill_server_control_namespace(self, namespace, protect=True): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 682 | """ |
| 683 | Prepare a namespace to be used when executing server control files. |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 684 | |
| 685 | This sets up the control file API by importing modules and making them |
| 686 | available under the appropriate names within namespace. |
| 687 | |
| 688 | For use by _execute_code(). |
| 689 | |
| 690 | Args: |
| 691 | namespace: The namespace dictionary to fill in. |
| 692 | protect: Boolean. If True (the default) any operation that would |
| 693 | clobber an existing entry in namespace will cause an error. |
| 694 | Raises: |
| 695 | error.AutoservError: When a name would be clobbered by import. |
| 696 | """ |
| 697 | def _import_names(module_name, names=()): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 698 | """ |
| 699 | Import a module and assign named attributes into namespace. |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 700 | |
| 701 | Args: |
| 702 | module_name: The string module name. |
| 703 | names: A limiting list of names to import from module_name. If |
| 704 | empty (the default), all names are imported from the module |
| 705 | similar to a "from foo.bar import *" statement. |
| 706 | Raises: |
| 707 | error.AutoservError: When a name being imported would clobber |
| 708 | a name already in namespace. |
| 709 | """ |
| 710 | module = __import__(module_name, {}, {}, names) |
| 711 | |
| 712 | # No names supplied? Import * from the lowest level module. |
| 713 | # (Ugh, why do I have to implement this part myself?) |
| 714 | if not names: |
| 715 | for submodule_name in module_name.split('.')[1:]: |
| 716 | module = getattr(module, submodule_name) |
| 717 | if hasattr(module, '__all__'): |
| 718 | names = getattr(module, '__all__') |
| 719 | else: |
| 720 | names = dir(module) |
| 721 | |
| 722 | # Install each name into namespace, checking to make sure it |
| 723 | # doesn't override anything that already exists. |
| 724 | for name in names: |
| 725 | # Check for conflicts to help prevent future problems. |
| 726 | if name in namespace and protect: |
| 727 | if namespace[name] is not getattr(module, name): |
| 728 | raise error.AutoservError('importing name ' |
| 729 | '%s from %s %r would override %r' % |
| 730 | (name, module_name, getattr(module, name), |
| 731 | namespace[name])) |
| 732 | else: |
| 733 | # Encourage cleanliness and the use of __all__ for a |
| 734 | # more concrete API with less surprises on '*' imports. |
| 735 | warnings.warn('%s (%r) being imported from %s for use ' |
| 736 | 'in server control files is not the ' |
| 737 | 'first occurrance of that import.' % |
| 738 | (name, namespace[name], module_name)) |
| 739 | |
| 740 | namespace[name] = getattr(module, name) |
| 741 | |
| 742 | |
| 743 | # This is the equivalent of prepending a bunch of import statements to |
| 744 | # the front of the control script. |
| 745 | namespace.update(os=os, sys=sys) |
| 746 | _import_names('autotest_lib.server', |
| 747 | ('hosts', 'autotest', 'kvm', 'git', 'standalone_profiler', |
| 748 | 'source_kernel', 'rpm_kernel', 'deb_kernel', 'git_kernel')) |
| 749 | _import_names('autotest_lib.server.subcommand', |
| 750 | ('parallel', 'parallel_simple', 'subcommand')) |
| 751 | _import_names('autotest_lib.server.utils', |
| 752 | ('run', 'get_tmp_dir', 'sh_escape', 'parse_machine')) |
| 753 | _import_names('autotest_lib.client.common_lib.error') |
| 754 | _import_names('autotest_lib.client.common_lib.barrier', ('barrier',)) |
| 755 | |
| 756 | # Inject ourself as the job object into other classes within the API. |
| 757 | # (Yuck, this injection is a gross thing be part of a public API. -gps) |
| 758 | # |
| 759 | # XXX Base & SiteAutotest do not appear to use .job. Who does? |
| 760 | namespace['autotest'].Autotest.job = self |
| 761 | # server.hosts.base_classes.Host uses .job. |
| 762 | namespace['hosts'].Host.job = self |
| 763 | |
| 764 | |
| 765 | def _execute_code(self, code_file, namespace, protect=True): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 766 | """ |
| 767 | Execute code using a copy of namespace as a server control script. |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 768 | |
| 769 | Unless protect_namespace is explicitly set to False, the dict will not |
| 770 | be modified. |
| 771 | |
| 772 | Args: |
| 773 | code_file: The filename of the control file to execute. |
| 774 | namespace: A dict containing names to make available during execution. |
| 775 | protect: Boolean. If True (the default) a copy of the namespace dict |
| 776 | is used during execution to prevent the code from modifying its |
| 777 | contents outside of this function. If False the raw dict is |
| 778 | passed in and modifications will be allowed. |
| 779 | """ |
| 780 | if protect: |
| 781 | namespace = namespace.copy() |
| 782 | self._fill_server_control_namespace(namespace, protect=protect) |
| 783 | # TODO: Simplify and get rid of the special cases for only 1 machine. |
showard | 3e66e8c | 2008-10-27 19:20:51 +0000 | [diff] [blame] | 784 | if len(self.machines) > 1: |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 785 | machines_text = '\n'.join(self.machines) + '\n' |
| 786 | # Only rewrite the file if it does not match our machine list. |
| 787 | try: |
| 788 | machines_f = open(MACHINES_FILENAME, 'r') |
| 789 | existing_machines_text = machines_f.read() |
| 790 | machines_f.close() |
| 791 | except EnvironmentError: |
| 792 | existing_machines_text = None |
| 793 | if machines_text != existing_machines_text: |
| 794 | utils.open_write_close(MACHINES_FILENAME, machines_text) |
| 795 | execfile(code_file, namespace, namespace) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 796 | |
| 797 | |
| 798 | def _record(self, status_code, subdir, operation, status='', |
| 799 | epoch_time=None, optional_fields=None): |
| 800 | """ |
| 801 | Actual function for recording a single line into the status |
| 802 | logs. Should never be called directly, only by job.record as |
| 803 | this would bypass the console monitor logging. |
| 804 | """ |
| 805 | |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 806 | msg = self._render_record(status_code, subdir, operation, status, |
| 807 | epoch_time, optional_fields=optional_fields) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 808 | |
| 809 | |
| 810 | status_file = os.path.join(self.resultdir, 'status.log') |
| 811 | sys.stdout.write(msg) |
| 812 | open(status_file, "a").write(msg) |
| 813 | if subdir: |
| 814 | test_dir = os.path.join(self.resultdir, subdir) |
jadmanski | 5ff5535 | 2008-09-18 19:43:46 +0000 | [diff] [blame] | 815 | status_file = os.path.join(test_dir, 'status.log') |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 816 | open(status_file, "a").write(msg) |
| 817 | self.__parse_status(msg.splitlines()) |
| 818 | |
| 819 | |
| 820 | def __parse_status(self, new_lines): |
| 821 | if not self.using_parser: |
| 822 | return |
| 823 | new_tests = self.parser.process_lines(new_lines) |
| 824 | for test in new_tests: |
| 825 | self.__insert_test(test) |
| 826 | |
| 827 | |
| 828 | def __insert_test(self, test): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 829 | """ |
| 830 | An internal method to insert a new test result into the |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 831 | database. This method will not raise an exception, even if an |
| 832 | error occurs during the insert, to avoid failing a test |
| 833 | simply because of unexpected database issues.""" |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 834 | self.num_tests_run += 1 |
| 835 | if status_lib.is_worse_than_or_equal_to(test.status, 'FAIL'): |
| 836 | self.num_tests_failed += 1 |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 837 | try: |
| 838 | self.results_db.insert_test(self.job_model, test) |
| 839 | except Exception: |
| 840 | msg = ("WARNING: An unexpected error occured while " |
| 841 | "inserting test results into the database. " |
| 842 | "Ignoring error.\n" + traceback.format_exc()) |
| 843 | print >> sys.stderr, msg |
| 844 | |
mbligh | caa62c2 | 2008-04-07 21:51:17 +0000 | [diff] [blame] | 845 | |
mbligh | a700772 | 2009-01-13 00:37:11 +0000 | [diff] [blame] | 846 | site_server_job = utils.import_site_class( |
| 847 | __file__, "autotest_lib.server.site_server_job", "site_server_job", |
| 848 | base_server_job) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 849 | |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 850 | class server_job(site_server_job, base_server_job): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 851 | pass |