mbligh | 57e7866 | 2008-06-17 19:53:49 +0000 | [diff] [blame] | 1 | """ |
| 2 | The main job wrapper for the server side. |
| 3 | |
| 4 | This is the core infrastructure. Derived from the client side job.py |
| 5 | |
| 6 | Copyright Martin J. Bligh, Andy Whitcroft 2007 |
| 7 | """ |
| 8 | |
jadmanski | 025099d | 2008-09-23 14:13:48 +0000 | [diff] [blame] | 9 | import getpass, os, sys, re, stat, tempfile, time, select, subprocess, traceback |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 10 | import shutil, warnings |
jadmanski | c09fc15 | 2008-10-15 17:56:59 +0000 | [diff] [blame] | 11 | from autotest_lib.client.bin import fd_stack, sysinfo |
mbligh | 0910844 | 2008-10-15 16:27:38 +0000 | [diff] [blame] | 12 | from autotest_lib.client.common_lib import error, log, utils, packages |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 13 | from autotest_lib.server import test, subcommand, profilers |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 14 | from autotest_lib.tko import db as tko_db, status_lib, utils as tko_utils |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 15 | |
| 16 | |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 17 | def _control_segment_path(name): |
| 18 | """Get the pathname of the named control segment file.""" |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 19 | server_dir = os.path.dirname(os.path.abspath(__file__)) |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 20 | return os.path.join(server_dir, "control_segments", name) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 21 | |
| 22 | |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 23 | CLIENT_CONTROL_FILENAME = 'control' |
| 24 | SERVER_CONTROL_FILENAME = 'control.srv' |
| 25 | MACHINES_FILENAME = '.machines' |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 26 | |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 27 | CLIENT_WRAPPER_CONTROL_FILE = _control_segment_path('client_wrapper') |
| 28 | CRASHDUMPS_CONTROL_FILE = _control_segment_path('crashdumps') |
| 29 | CRASHINFO_CONTROL_FILE = _control_segment_path('crashinfo') |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 30 | INSTALL_CONTROL_FILE = _control_segment_path('install') |
showard | 45ae819 | 2008-11-05 19:32:53 +0000 | [diff] [blame] | 31 | CLEANUP_CONTROL_FILE = _control_segment_path('cleanup') |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 32 | |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 33 | VERIFY_CONTROL_FILE = _control_segment_path('verify') |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 34 | REPAIR_CONTROL_FILE = _control_segment_path('repair') |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 35 | |
| 36 | |
mbligh | 062ed15 | 2009-01-13 00:57:14 +0000 | [diff] [blame] | 37 | # by default provide a stub that generates no site data |
| 38 | def _get_site_job_data_dummy(job): |
| 39 | return {} |
| 40 | |
| 41 | |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 42 | # load up site-specific code for generating site-specific job data |
mbligh | 062ed15 | 2009-01-13 00:57:14 +0000 | [diff] [blame] | 43 | get_site_job_data = utils.import_site_function(__file__, |
| 44 | "autotest_lib.server.site_job", "get_site_job_data", |
| 45 | _get_site_job_data_dummy) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 46 | |
| 47 | |
| 48 | class base_server_job(object): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 49 | """ |
| 50 | The actual job against which we do everything. |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 51 | |
| 52 | Properties: |
| 53 | autodir |
| 54 | The top level autotest directory (/usr/local/autotest). |
| 55 | serverdir |
| 56 | <autodir>/server/ |
| 57 | clientdir |
| 58 | <autodir>/client/ |
| 59 | conmuxdir |
| 60 | <autodir>/conmux/ |
| 61 | testdir |
| 62 | <autodir>/server/tests/ |
| 63 | site_testdir |
| 64 | <autodir>/server/site_tests/ |
| 65 | control |
| 66 | the control file for this job |
mbligh | b5dac43 | 2008-11-27 00:38:44 +0000 | [diff] [blame] | 67 | drop_caches_between_iterations |
| 68 | drop the pagecache between each iteration |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 69 | """ |
| 70 | |
| 71 | STATUS_VERSION = 1 |
| 72 | |
| 73 | |
| 74 | def __init__(self, control, args, resultdir, label, user, machines, |
| 75 | client=False, parse_job='', |
| 76 | ssh_user='root', ssh_port=22, ssh_pass=''): |
| 77 | """ |
mbligh | b5dac43 | 2008-11-27 00:38:44 +0000 | [diff] [blame] | 78 | Server side job object. |
| 79 | |
| 80 | Parameters: |
| 81 | control: The control file (pathname of) |
| 82 | args: args to pass to the control file |
| 83 | resultdir: where to throw the results |
| 84 | label: label for the job |
| 85 | user: Username for the job (email address) |
| 86 | client: True if a client-side control file |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 87 | """ |
| 88 | path = os.path.dirname(__file__) |
| 89 | self.autodir = os.path.abspath(os.path.join(path, '..')) |
| 90 | self.serverdir = os.path.join(self.autodir, 'server') |
| 91 | self.testdir = os.path.join(self.serverdir, 'tests') |
| 92 | self.site_testdir = os.path.join(self.serverdir, 'site_tests') |
| 93 | self.tmpdir = os.path.join(self.serverdir, 'tmp') |
| 94 | self.conmuxdir = os.path.join(self.autodir, 'conmux') |
| 95 | self.clientdir = os.path.join(self.autodir, 'client') |
| 96 | self.toolsdir = os.path.join(self.autodir, 'client/tools') |
| 97 | if control: |
jadmanski | e432dd2 | 2009-01-30 15:04:51 +0000 | [diff] [blame] | 98 | self.control = self._load_control_file(control) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 99 | else: |
showard | 45ae819 | 2008-11-05 19:32:53 +0000 | [diff] [blame] | 100 | self.control = '' |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 101 | self.resultdir = resultdir |
mbligh | 80e1eba | 2008-11-19 00:26:18 +0000 | [diff] [blame] | 102 | if resultdir: |
| 103 | if not os.path.exists(resultdir): |
| 104 | os.mkdir(resultdir) |
| 105 | self.debugdir = os.path.join(resultdir, 'debug') |
| 106 | if not os.path.exists(self.debugdir): |
| 107 | os.mkdir(self.debugdir) |
| 108 | self.status = os.path.join(resultdir, 'status') |
| 109 | else: |
| 110 | self.status = None |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 111 | self.label = label |
| 112 | self.user = user |
| 113 | self.args = args |
| 114 | self.machines = machines |
| 115 | self.client = client |
| 116 | self.record_prefix = '' |
| 117 | self.warning_loggers = set() |
jadmanski | f37df84 | 2009-02-11 00:03:26 +0000 | [diff] [blame^] | 118 | self.warning_manager = warning_manager() |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 119 | self.ssh_user = ssh_user |
| 120 | self.ssh_port = ssh_port |
| 121 | self.ssh_pass = ssh_pass |
jadmanski | 23afbec | 2008-09-17 18:12:07 +0000 | [diff] [blame] | 122 | self.run_test_cleanup = True |
mbligh | 0910844 | 2008-10-15 16:27:38 +0000 | [diff] [blame] | 123 | self.last_boot_tag = None |
jadmanski | 53aaf38 | 2008-11-17 16:22:31 +0000 | [diff] [blame] | 124 | self.hosts = set() |
mbligh | b5dac43 | 2008-11-27 00:38:44 +0000 | [diff] [blame] | 125 | self.drop_caches_between_iterations = False |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 126 | |
| 127 | self.stdout = fd_stack.fd_stack(1, sys.stdout) |
| 128 | self.stderr = fd_stack.fd_stack(2, sys.stderr) |
| 129 | |
mbligh | 80e1eba | 2008-11-19 00:26:18 +0000 | [diff] [blame] | 130 | if resultdir: |
| 131 | self.sysinfo = sysinfo.sysinfo(self.resultdir) |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 132 | self.profilers = profilers.profilers(self) |
jadmanski | c09fc15 | 2008-10-15 17:56:59 +0000 | [diff] [blame] | 133 | |
jadmanski | 025099d | 2008-09-23 14:13:48 +0000 | [diff] [blame] | 134 | if not os.access(self.tmpdir, os.W_OK): |
| 135 | try: |
| 136 | os.makedirs(self.tmpdir, 0700) |
| 137 | except os.error, e: |
| 138 | # Thrown if the directory already exists, which it may. |
| 139 | pass |
| 140 | |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 141 | if not (os.access(self.tmpdir, os.W_OK) and os.path.isdir(self.tmpdir)): |
jadmanski | 025099d | 2008-09-23 14:13:48 +0000 | [diff] [blame] | 142 | self.tmpdir = os.path.join(tempfile.gettempdir(), |
| 143 | 'autotest-' + getpass.getuser()) |
| 144 | try: |
| 145 | os.makedirs(self.tmpdir, 0700) |
| 146 | except os.error, e: |
| 147 | # Thrown if the directory already exists, which it may. |
| 148 | # If the problem was something other than the |
| 149 | # directory already existing, this chmod should throw as well |
| 150 | # exception. |
| 151 | os.chmod(self.tmpdir, stat.S_IRWXU) |
| 152 | |
mbligh | 80e1eba | 2008-11-19 00:26:18 +0000 | [diff] [blame] | 153 | if self.status and os.path.exists(self.status): |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 154 | os.unlink(self.status) |
| 155 | job_data = {'label' : label, 'user' : user, |
| 156 | 'hostname' : ','.join(machines), |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 157 | 'status_version' : str(self.STATUS_VERSION), |
| 158 | 'job_started' : str(int(time.time()))} |
mbligh | 80e1eba | 2008-11-19 00:26:18 +0000 | [diff] [blame] | 159 | if self.resultdir: |
| 160 | job_data.update(get_site_job_data(self)) |
| 161 | utils.write_keyval(self.resultdir, job_data) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 162 | |
| 163 | self.parse_job = parse_job |
| 164 | if self.parse_job and len(machines) == 1: |
| 165 | self.using_parser = True |
| 166 | self.init_parser(resultdir) |
| 167 | else: |
| 168 | self.using_parser = False |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 169 | self.pkgmgr = packages.PackageManager(self.autodir, |
| 170 | run_function_dargs={'timeout':600}) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 171 | self.pkgdir = os.path.join(self.autodir, 'packages') |
| 172 | |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 173 | self.num_tests_run = 0 |
| 174 | self.num_tests_failed = 0 |
| 175 | |
jadmanski | 550fdc2 | 2008-11-20 16:32:08 +0000 | [diff] [blame] | 176 | self._register_subcommand_hooks() |
| 177 | |
| 178 | |
jadmanski | e432dd2 | 2009-01-30 15:04:51 +0000 | [diff] [blame] | 179 | @staticmethod |
| 180 | def _load_control_file(path): |
| 181 | f = open(path) |
| 182 | try: |
| 183 | control_file = f.read() |
| 184 | finally: |
| 185 | f.close() |
| 186 | return re.sub('\r', '', control_file) |
| 187 | |
| 188 | |
jadmanski | 550fdc2 | 2008-11-20 16:32:08 +0000 | [diff] [blame] | 189 | def _register_subcommand_hooks(self): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 190 | """ |
| 191 | Register some hooks into the subcommand modules that allow us |
| 192 | to properly clean up self.hosts created in forked subprocesses. |
| 193 | """ |
jadmanski | 550fdc2 | 2008-11-20 16:32:08 +0000 | [diff] [blame] | 194 | def on_fork(cmd): |
| 195 | self._existing_hosts_on_fork = set(self.hosts) |
| 196 | def on_join(cmd): |
| 197 | new_hosts = self.hosts - self._existing_hosts_on_fork |
| 198 | for host in new_hosts: |
| 199 | host.close() |
| 200 | subcommand.subcommand.register_fork_hook(on_fork) |
| 201 | subcommand.subcommand.register_join_hook(on_join) |
| 202 | |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 203 | |
| 204 | def init_parser(self, resultdir): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 205 | """ |
| 206 | Start the continuous parsing of resultdir. This sets up |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 207 | the database connection and inserts the basic job object into |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 208 | the database if necessary. |
| 209 | """ |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 210 | # redirect parser debugging to .parse.log |
| 211 | parse_log = os.path.join(resultdir, '.parse.log') |
| 212 | parse_log = open(parse_log, 'w', 0) |
| 213 | tko_utils.redirect_parser_debugging(parse_log) |
| 214 | # create a job model object and set up the db |
| 215 | self.results_db = tko_db.db(autocommit=True) |
| 216 | self.parser = status_lib.parser(self.STATUS_VERSION) |
| 217 | self.job_model = self.parser.make_job(resultdir) |
| 218 | self.parser.start(self.job_model) |
| 219 | # check if a job already exists in the db and insert it if |
| 220 | # it does not |
| 221 | job_idx = self.results_db.find_job(self.parse_job) |
| 222 | if job_idx is None: |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 223 | self.results_db.insert_job(self.parse_job, self.job_model) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 224 | else: |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 225 | machine_idx = self.results_db.lookup_machine(self.job_model.machine) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 226 | self.job_model.index = job_idx |
| 227 | self.job_model.machine_idx = machine_idx |
| 228 | |
| 229 | |
| 230 | def cleanup_parser(self): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 231 | """ |
| 232 | This should be called after the server job is finished |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 233 | to carry out any remaining cleanup (e.g. flushing any |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 234 | remaining test results to the results db) |
| 235 | """ |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 236 | if not self.using_parser: |
| 237 | return |
| 238 | final_tests = self.parser.end() |
| 239 | for test in final_tests: |
| 240 | self.__insert_test(test) |
| 241 | self.using_parser = False |
| 242 | |
| 243 | |
| 244 | def verify(self): |
| 245 | if not self.machines: |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 246 | raise error.AutoservError('No machines specified to verify') |
mbligh | 0fce411 | 2008-11-27 00:37:17 +0000 | [diff] [blame] | 247 | if self.resultdir: |
| 248 | os.chdir(self.resultdir) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 249 | try: |
jadmanski | cdd0c40 | 2008-09-19 21:21:31 +0000 | [diff] [blame] | 250 | namespace = {'machines' : self.machines, 'job' : self, |
| 251 | 'ssh_user' : self.ssh_user, |
| 252 | 'ssh_port' : self.ssh_port, |
| 253 | 'ssh_pass' : self.ssh_pass} |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 254 | self._execute_code(VERIFY_CONTROL_FILE, namespace, protect=False) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 255 | except Exception, e: |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 256 | msg = ('Verify failed\n' + str(e) + '\n' + traceback.format_exc()) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 257 | self.record('ABORT', None, None, msg) |
| 258 | raise |
| 259 | |
| 260 | |
| 261 | def repair(self, host_protection): |
| 262 | if not self.machines: |
| 263 | raise error.AutoservError('No machines specified to repair') |
mbligh | 0fce411 | 2008-11-27 00:37:17 +0000 | [diff] [blame] | 264 | if self.resultdir: |
| 265 | os.chdir(self.resultdir) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 266 | namespace = {'machines': self.machines, 'job': self, |
| 267 | 'ssh_user': self.ssh_user, 'ssh_port': self.ssh_port, |
| 268 | 'ssh_pass': self.ssh_pass, |
| 269 | 'protection_level': host_protection} |
mbligh | 25c0b8c | 2009-01-24 01:44:17 +0000 | [diff] [blame] | 270 | # no matter what happens during repair (except if it succeeded in |
| 271 | # initiating hardware repair procedure), go on to try to reverify |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 272 | try: |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 273 | self._execute_code(REPAIR_CONTROL_FILE, namespace, protect=False) |
mbligh | 25c0b8c | 2009-01-24 01:44:17 +0000 | [diff] [blame] | 274 | except error.AutoservHardwareRepairRequestedError: |
| 275 | raise |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 276 | except Exception, exc: |
| 277 | print 'Exception occured during repair' |
| 278 | traceback.print_exc() |
mbligh | 25c0b8c | 2009-01-24 01:44:17 +0000 | [diff] [blame] | 279 | |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 280 | self.verify() |
| 281 | |
| 282 | |
| 283 | def precheck(self): |
| 284 | """ |
| 285 | perform any additional checks in derived classes. |
| 286 | """ |
| 287 | pass |
| 288 | |
| 289 | |
| 290 | def enable_external_logging(self): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 291 | """ |
| 292 | Start or restart external logging mechanism. |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 293 | """ |
| 294 | pass |
| 295 | |
| 296 | |
| 297 | def disable_external_logging(self): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 298 | """ |
| 299 | Pause or stop external logging mechanism. |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 300 | """ |
| 301 | pass |
| 302 | |
| 303 | |
jadmanski | 23afbec | 2008-09-17 18:12:07 +0000 | [diff] [blame] | 304 | def enable_test_cleanup(self): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 305 | """ |
| 306 | By default tests run test.cleanup |
| 307 | """ |
jadmanski | 23afbec | 2008-09-17 18:12:07 +0000 | [diff] [blame] | 308 | self.run_test_cleanup = True |
| 309 | |
| 310 | |
| 311 | def disable_test_cleanup(self): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 312 | """ |
| 313 | By default tests do not run test.cleanup |
| 314 | """ |
jadmanski | 23afbec | 2008-09-17 18:12:07 +0000 | [diff] [blame] | 315 | self.run_test_cleanup = False |
| 316 | |
| 317 | |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 318 | def use_external_logging(self): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 319 | """ |
| 320 | Return True if external logging should be used. |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 321 | """ |
| 322 | return False |
| 323 | |
| 324 | |
| 325 | def parallel_simple(self, function, machines, log=True, timeout=None): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 326 | """ |
| 327 | Run 'function' using parallel_simple, with an extra wrapper to handle |
| 328 | the necessary setup for continuous parsing, if possible. If continuous |
| 329 | parsing is already properly initialized then this should just work. |
| 330 | """ |
| 331 | is_forking = not (len(machines) == 1 and self.machines == machines) |
jadmanski | 4dd1a00 | 2008-09-05 20:27:30 +0000 | [diff] [blame] | 332 | if self.parse_job and is_forking and log: |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 333 | def wrapper(machine): |
| 334 | self.parse_job += "/" + machine |
| 335 | self.using_parser = True |
| 336 | self.machines = [machine] |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 337 | self.resultdir = os.path.join(self.resultdir, machine) |
jadmanski | 609a5f4 | 2008-08-26 20:52:42 +0000 | [diff] [blame] | 338 | os.chdir(self.resultdir) |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 339 | utils.write_keyval(self.resultdir, {"hostname": machine}) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 340 | self.init_parser(self.resultdir) |
| 341 | result = function(machine) |
| 342 | self.cleanup_parser() |
| 343 | return result |
jadmanski | 4dd1a00 | 2008-09-05 20:27:30 +0000 | [diff] [blame] | 344 | elif len(machines) > 1 and log: |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 345 | def wrapper(machine): |
| 346 | self.resultdir = os.path.join(self.resultdir, machine) |
jadmanski | 609a5f4 | 2008-08-26 20:52:42 +0000 | [diff] [blame] | 347 | os.chdir(self.resultdir) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 348 | result = function(machine) |
| 349 | return result |
| 350 | else: |
| 351 | wrapper = function |
| 352 | subcommand.parallel_simple(wrapper, machines, log, timeout) |
| 353 | |
| 354 | |
jadmanski | e432dd2 | 2009-01-30 15:04:51 +0000 | [diff] [blame] | 355 | USE_TEMP_DIR = object() |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 356 | def run(self, cleanup=False, install_before=False, install_after=False, |
jadmanski | e432dd2 | 2009-01-30 15:04:51 +0000 | [diff] [blame] | 357 | collect_crashdumps=True, namespace={}, control=None, |
| 358 | control_file_dir=None): |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 359 | # use a copy so changes don't affect the original dictionary |
| 360 | namespace = namespace.copy() |
| 361 | machines = self.machines |
jadmanski | e432dd2 | 2009-01-30 15:04:51 +0000 | [diff] [blame] | 362 | if control is None: |
| 363 | control = self.control |
| 364 | if control_file_dir is None: |
| 365 | control_file_dir = self.resultdir |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 366 | |
| 367 | self.aborted = False |
| 368 | namespace['machines'] = machines |
| 369 | namespace['args'] = self.args |
| 370 | namespace['job'] = self |
| 371 | namespace['ssh_user'] = self.ssh_user |
| 372 | namespace['ssh_port'] = self.ssh_port |
| 373 | namespace['ssh_pass'] = self.ssh_pass |
| 374 | test_start_time = int(time.time()) |
| 375 | |
mbligh | 80e1eba | 2008-11-19 00:26:18 +0000 | [diff] [blame] | 376 | if self.resultdir: |
| 377 | os.chdir(self.resultdir) |
mbligh | 80e1eba | 2008-11-19 00:26:18 +0000 | [diff] [blame] | 378 | self.enable_external_logging() |
jadmanski | e432dd2 | 2009-01-30 15:04:51 +0000 | [diff] [blame] | 379 | |
jadmanski | cdd0c40 | 2008-09-19 21:21:31 +0000 | [diff] [blame] | 380 | collect_crashinfo = True |
mbligh | aebe3b6 | 2008-12-22 14:45:40 +0000 | [diff] [blame] | 381 | temp_control_file_dir = None |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 382 | try: |
| 383 | if install_before and machines: |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 384 | self._execute_code(INSTALL_CONTROL_FILE, namespace) |
jadmanski | e432dd2 | 2009-01-30 15:04:51 +0000 | [diff] [blame] | 385 | |
| 386 | # determine the dir to write the control files to |
| 387 | if control_file_dir and control_file_dir is not self.USE_TEMP_DIR: |
| 388 | temp_control_file_dir = None |
mbligh | aebe3b6 | 2008-12-22 14:45:40 +0000 | [diff] [blame] | 389 | else: |
jadmanski | e432dd2 | 2009-01-30 15:04:51 +0000 | [diff] [blame] | 390 | temp_control_file_dir = control_file_dir = tempfile.mkdtemp( |
| 391 | suffix='temp_control_file_dir') |
| 392 | server_control_file = os.path.join(control_file_dir, |
| 393 | SERVER_CONTROL_FILENAME) |
| 394 | client_control_file = os.path.join(control_file_dir, |
| 395 | CLIENT_CONTROL_FILENAME) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 396 | if self.client: |
jadmanski | e432dd2 | 2009-01-30 15:04:51 +0000 | [diff] [blame] | 397 | namespace['control'] = control |
| 398 | utils.open_write_close(client_control_file, control) |
mbligh | aebe3b6 | 2008-12-22 14:45:40 +0000 | [diff] [blame] | 399 | shutil.copy(CLIENT_WRAPPER_CONTROL_FILE, server_control_file) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 400 | else: |
mbligh | 181b7c2 | 2008-11-22 14:22:08 +0000 | [diff] [blame] | 401 | namespace['utils'] = utils |
jadmanski | e432dd2 | 2009-01-30 15:04:51 +0000 | [diff] [blame] | 402 | utils.open_write_close(server_control_file, control) |
mbligh | aebe3b6 | 2008-12-22 14:45:40 +0000 | [diff] [blame] | 403 | self._execute_code(server_control_file, namespace) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 404 | |
jadmanski | cdd0c40 | 2008-09-19 21:21:31 +0000 | [diff] [blame] | 405 | # disable crashinfo collection if we get this far without error |
| 406 | collect_crashinfo = False |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 407 | finally: |
mbligh | aebe3b6 | 2008-12-22 14:45:40 +0000 | [diff] [blame] | 408 | if temp_control_file_dir: |
jadmanski | e432dd2 | 2009-01-30 15:04:51 +0000 | [diff] [blame] | 409 | # Clean up temp directory used for copies of the control files |
mbligh | aebe3b6 | 2008-12-22 14:45:40 +0000 | [diff] [blame] | 410 | try: |
| 411 | shutil.rmtree(temp_control_file_dir) |
| 412 | except Exception, e: |
jadmanski | e432dd2 | 2009-01-30 15:04:51 +0000 | [diff] [blame] | 413 | print 'Error %s removing dir %s' % (e, |
| 414 | temp_control_file_dir) |
| 415 | |
jadmanski | cdd0c40 | 2008-09-19 21:21:31 +0000 | [diff] [blame] | 416 | if machines and (collect_crashdumps or collect_crashinfo): |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 417 | namespace['test_start_time'] = test_start_time |
jadmanski | cdd0c40 | 2008-09-19 21:21:31 +0000 | [diff] [blame] | 418 | if collect_crashinfo: |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 419 | # includes crashdumps |
| 420 | self._execute_code(CRASHINFO_CONTROL_FILE, namespace) |
jadmanski | cdd0c40 | 2008-09-19 21:21:31 +0000 | [diff] [blame] | 421 | else: |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 422 | self._execute_code(CRASHDUMPS_CONTROL_FILE, namespace) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 423 | self.disable_external_logging() |
showard | 45ae819 | 2008-11-05 19:32:53 +0000 | [diff] [blame] | 424 | if cleanup and machines: |
| 425 | self._execute_code(CLEANUP_CONTROL_FILE, namespace) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 426 | if install_after and machines: |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 427 | self._execute_code(INSTALL_CONTROL_FILE, namespace) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 428 | |
| 429 | |
| 430 | def run_test(self, url, *args, **dargs): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 431 | """ |
| 432 | Summon a test object and run it. |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 433 | |
| 434 | tag |
| 435 | tag to add to testname |
| 436 | url |
| 437 | url of the test to run |
| 438 | """ |
| 439 | |
| 440 | (group, testname) = self.pkgmgr.get_package_name(url, 'test') |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 441 | |
| 442 | tag = dargs.pop('tag', None) |
| 443 | if tag: |
mbligh | 8ad2420 | 2009-01-07 16:49:36 +0000 | [diff] [blame] | 444 | testname += '.' + str(tag) |
jadmanski | de292df | 2008-08-26 20:51:14 +0000 | [diff] [blame] | 445 | subdir = testname |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 446 | |
| 447 | outputdir = os.path.join(self.resultdir, subdir) |
| 448 | if os.path.exists(outputdir): |
| 449 | msg = ("%s already exists, test <%s> may have" |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 450 | " already run with tag <%s>" % (outputdir, testname, tag)) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 451 | raise error.TestError(msg) |
| 452 | os.mkdir(outputdir) |
| 453 | |
| 454 | def group_func(): |
| 455 | try: |
| 456 | test.runtest(self, url, tag, args, dargs) |
| 457 | except error.TestBaseException, e: |
| 458 | self.record(e.exit_status, subdir, testname, str(e)) |
| 459 | raise |
| 460 | except Exception, e: |
| 461 | info = str(e) + "\n" + traceback.format_exc() |
| 462 | self.record('FAIL', subdir, testname, info) |
| 463 | raise |
| 464 | else: |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 465 | self.record('GOOD', subdir, testname, 'completed successfully') |
jadmanski | de292df | 2008-08-26 20:51:14 +0000 | [diff] [blame] | 466 | |
| 467 | result, exc_info = self._run_group(testname, subdir, group_func) |
| 468 | if exc_info and isinstance(exc_info[1], error.TestBaseException): |
| 469 | return False |
| 470 | elif exc_info: |
| 471 | raise exc_info[0], exc_info[1], exc_info[2] |
| 472 | else: |
| 473 | return True |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 474 | |
| 475 | |
| 476 | def _run_group(self, name, subdir, function, *args, **dargs): |
| 477 | """\ |
| 478 | Underlying method for running something inside of a group. |
| 479 | """ |
jadmanski | de292df | 2008-08-26 20:51:14 +0000 | [diff] [blame] | 480 | result, exc_info = None, None |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 481 | old_record_prefix = self.record_prefix |
| 482 | try: |
| 483 | self.record('START', subdir, name) |
| 484 | self.record_prefix += '\t' |
| 485 | try: |
| 486 | result = function(*args, **dargs) |
| 487 | finally: |
| 488 | self.record_prefix = old_record_prefix |
| 489 | except error.TestBaseException, e: |
jadmanski | b88d6dc | 2009-01-10 00:33:18 +0000 | [diff] [blame] | 490 | self.record("END %s" % e.exit_status, subdir, name) |
jadmanski | de292df | 2008-08-26 20:51:14 +0000 | [diff] [blame] | 491 | exc_info = sys.exc_info() |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 492 | except Exception, e: |
| 493 | err_msg = str(e) + '\n' |
| 494 | err_msg += traceback.format_exc() |
| 495 | self.record('END ABORT', subdir, name, err_msg) |
| 496 | raise error.JobError(name + ' failed\n' + traceback.format_exc()) |
| 497 | else: |
| 498 | self.record('END GOOD', subdir, name) |
| 499 | |
jadmanski | de292df | 2008-08-26 20:51:14 +0000 | [diff] [blame] | 500 | return result, exc_info |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 501 | |
| 502 | |
| 503 | def run_group(self, function, *args, **dargs): |
| 504 | """\ |
| 505 | function: |
| 506 | subroutine to run |
| 507 | *args: |
| 508 | arguments for the function |
| 509 | """ |
| 510 | |
| 511 | name = function.__name__ |
| 512 | |
| 513 | # Allow the tag for the group to be specified. |
| 514 | tag = dargs.pop('tag', None) |
| 515 | if tag: |
| 516 | name = tag |
| 517 | |
jadmanski | de292df | 2008-08-26 20:51:14 +0000 | [diff] [blame] | 518 | return self._run_group(name, None, function, *args, **dargs)[0] |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 519 | |
| 520 | |
| 521 | def run_reboot(self, reboot_func, get_kernel_func): |
| 522 | """\ |
| 523 | A specialization of run_group meant specifically for handling |
| 524 | a reboot. Includes support for capturing the kernel version |
| 525 | after the reboot. |
| 526 | |
| 527 | reboot_func: a function that carries out the reboot |
| 528 | |
| 529 | get_kernel_func: a function that returns a string |
| 530 | representing the kernel version. |
| 531 | """ |
| 532 | |
| 533 | old_record_prefix = self.record_prefix |
| 534 | try: |
| 535 | self.record('START', None, 'reboot') |
| 536 | self.record_prefix += '\t' |
| 537 | reboot_func() |
| 538 | except Exception, e: |
| 539 | self.record_prefix = old_record_prefix |
| 540 | err_msg = str(e) + '\n' + traceback.format_exc() |
| 541 | self.record('END FAIL', None, 'reboot', err_msg) |
| 542 | else: |
| 543 | kernel = get_kernel_func() |
| 544 | self.record_prefix = old_record_prefix |
| 545 | self.record('END GOOD', None, 'reboot', |
| 546 | optional_fields={"kernel": kernel}) |
| 547 | |
| 548 | |
jadmanski | e432dd2 | 2009-01-30 15:04:51 +0000 | [diff] [blame] | 549 | def run_control(self, path): |
| 550 | """Execute a control file found at path (relative to the autotest |
| 551 | path). Intended for executing a control file within a control file, |
| 552 | not for running the top-level job control file.""" |
| 553 | path = os.path.join(self.autodir, path) |
| 554 | control_file = self._load_control_file(path) |
| 555 | self.run(control=control_file, control_file_dir=self.USE_TEMP_DIR) |
| 556 | |
| 557 | |
jadmanski | c09fc15 | 2008-10-15 17:56:59 +0000 | [diff] [blame] | 558 | def add_sysinfo_command(self, command, logfile=None, on_every_test=False): |
| 559 | self._add_sysinfo_loggable(sysinfo.command(command, logfile), |
| 560 | on_every_test) |
| 561 | |
| 562 | |
| 563 | def add_sysinfo_logfile(self, file, on_every_test=False): |
| 564 | self._add_sysinfo_loggable(sysinfo.logfile(file), on_every_test) |
| 565 | |
| 566 | |
| 567 | def _add_sysinfo_loggable(self, loggable, on_every_test): |
| 568 | if on_every_test: |
| 569 | self.sysinfo.test_loggables.add(loggable) |
| 570 | else: |
| 571 | self.sysinfo.boot_loggables.add(loggable) |
| 572 | |
| 573 | |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 574 | def record(self, status_code, subdir, operation, status='', |
| 575 | optional_fields=None): |
| 576 | """ |
| 577 | Record job-level status |
| 578 | |
| 579 | The intent is to make this file both machine parseable and |
| 580 | human readable. That involves a little more complexity, but |
| 581 | really isn't all that bad ;-) |
| 582 | |
| 583 | Format is <status code>\t<subdir>\t<operation>\t<status> |
| 584 | |
mbligh | 1b3b376 | 2008-09-25 02:46:34 +0000 | [diff] [blame] | 585 | status code: see common_lib.log.is_valid_status() |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 586 | for valid status definition |
| 587 | |
| 588 | subdir: MUST be a relevant subdirectory in the results, |
| 589 | or None, which will be represented as '----' |
| 590 | |
| 591 | operation: description of what you ran (e.g. "dbench", or |
| 592 | "mkfs -t foobar /dev/sda9") |
| 593 | |
| 594 | status: error message or "completed sucessfully" |
| 595 | |
| 596 | ------------------------------------------------------------ |
| 597 | |
| 598 | Initial tabs indicate indent levels for grouping, and is |
| 599 | governed by self.record_prefix |
| 600 | |
| 601 | multiline messages have secondary lines prefaced by a double |
| 602 | space (' ') |
| 603 | |
| 604 | Executing this method will trigger the logging of all new |
| 605 | warnings to date from the various console loggers. |
| 606 | """ |
| 607 | # poll all our warning loggers for new warnings |
| 608 | warnings = self._read_warnings() |
| 609 | for timestamp, msg in warnings: |
| 610 | self._record("WARN", None, None, msg, timestamp) |
| 611 | |
| 612 | # write out the actual status log line |
| 613 | self._record(status_code, subdir, operation, status, |
| 614 | optional_fields=optional_fields) |
| 615 | |
| 616 | |
| 617 | def _read_warnings(self): |
jadmanski | f37df84 | 2009-02-11 00:03:26 +0000 | [diff] [blame^] | 618 | """Poll all the warning loggers and extract any new warnings that have |
| 619 | been logged. If the warnings belong to a category that is currently |
| 620 | disabled, this method will discard them and they will no longer be |
| 621 | retrievable. |
| 622 | |
| 623 | Returns a list of (timestamp, message) tuples, where timestamp is an |
| 624 | integer epoch timestamp.""" |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 625 | warnings = [] |
| 626 | while True: |
| 627 | # pull in a line of output from every logger that has |
| 628 | # output ready to be read |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 629 | loggers, _, _ = select.select(self.warning_loggers, [], [], 0) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 630 | closed_loggers = set() |
| 631 | for logger in loggers: |
| 632 | line = logger.readline() |
| 633 | # record any broken pipes (aka line == empty) |
| 634 | if len(line) == 0: |
| 635 | closed_loggers.add(logger) |
| 636 | continue |
jadmanski | f37df84 | 2009-02-11 00:03:26 +0000 | [diff] [blame^] | 637 | # parse out the warning |
| 638 | timestamp, msgtype, msg = line.split('\t', 2) |
| 639 | timestamp = int(timestamp) |
| 640 | # if the warning is valid, add it to the results |
| 641 | if self.warning_manager.is_valid(timestamp, msgtype): |
| 642 | warnings.append((timestamp, msg.strip())) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 643 | |
| 644 | # stop listening to loggers that are closed |
| 645 | self.warning_loggers -= closed_loggers |
| 646 | |
| 647 | # stop if none of the loggers have any output left |
| 648 | if not loggers: |
| 649 | break |
| 650 | |
| 651 | # sort into timestamp order |
| 652 | warnings.sort() |
| 653 | return warnings |
| 654 | |
| 655 | |
jadmanski | f37df84 | 2009-02-11 00:03:26 +0000 | [diff] [blame^] | 656 | def disable_warnings(self, warning_type, record=True): |
| 657 | self.warning_manager.disable_warnings(warning_type) |
| 658 | if record: |
| 659 | self.record("INFO", None, None, |
| 660 | "disabling %s warnings" % warning_type, |
| 661 | {"warnings.disable": warning_type}) |
| 662 | |
| 663 | |
| 664 | def enable_warnings(self, warning_type, record=True): |
| 665 | self.warning_manager.enable_warnings(warning_type) |
| 666 | if record: |
| 667 | self.record("INFO", None, None, |
| 668 | "enabling %s warnings" % warning_type, |
| 669 | {"warnings.enable": warning_type}) |
| 670 | |
| 671 | |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 672 | def _render_record(self, status_code, subdir, operation, status='', |
| 673 | epoch_time=None, record_prefix=None, |
| 674 | optional_fields=None): |
| 675 | """ |
| 676 | Internal Function to generate a record to be written into a |
| 677 | status log. For use by server_job.* classes only. |
| 678 | """ |
| 679 | if subdir: |
| 680 | if re.match(r'[\n\t]', subdir): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 681 | raise ValueError('Invalid character in subdir string') |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 682 | substr = subdir |
| 683 | else: |
| 684 | substr = '----' |
| 685 | |
mbligh | 1b3b376 | 2008-09-25 02:46:34 +0000 | [diff] [blame] | 686 | if not log.is_valid_status(status_code): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 687 | raise ValueError('Invalid status code supplied: %s' % status_code) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 688 | if not operation: |
| 689 | operation = '----' |
| 690 | if re.match(r'[\n\t]', operation): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 691 | raise ValueError('Invalid character in operation string') |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 692 | operation = operation.rstrip() |
| 693 | status = status.rstrip() |
| 694 | status = re.sub(r"\t", " ", status) |
| 695 | # Ensure any continuation lines are marked so we can |
| 696 | # detect them in the status file to ensure it is parsable. |
| 697 | status = re.sub(r"\n", "\n" + self.record_prefix + " ", status) |
| 698 | |
| 699 | if not optional_fields: |
| 700 | optional_fields = {} |
| 701 | |
| 702 | # Generate timestamps for inclusion in the logs |
| 703 | if epoch_time is None: |
| 704 | epoch_time = int(time.time()) |
| 705 | local_time = time.localtime(epoch_time) |
| 706 | optional_fields["timestamp"] = str(epoch_time) |
| 707 | optional_fields["localtime"] = time.strftime("%b %d %H:%M:%S", |
| 708 | local_time) |
| 709 | |
| 710 | fields = [status_code, substr, operation] |
| 711 | fields += ["%s=%s" % x for x in optional_fields.iteritems()] |
| 712 | fields.append(status) |
| 713 | |
| 714 | if record_prefix is None: |
| 715 | record_prefix = self.record_prefix |
| 716 | |
| 717 | msg = '\t'.join(str(x) for x in fields) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 718 | return record_prefix + msg + '\n' |
| 719 | |
| 720 | |
| 721 | def _record_prerendered(self, msg): |
| 722 | """ |
| 723 | Record a pre-rendered msg into the status logs. The only |
| 724 | change this makes to the message is to add on the local |
| 725 | indentation. Should not be called outside of server_job.* |
| 726 | classes. Unlike _record, this does not write the message |
| 727 | to standard output. |
| 728 | """ |
| 729 | lines = [] |
| 730 | status_file = os.path.join(self.resultdir, 'status.log') |
| 731 | status_log = open(status_file, 'a') |
| 732 | for line in msg.splitlines(): |
| 733 | line = self.record_prefix + line + '\n' |
| 734 | lines.append(line) |
| 735 | status_log.write(line) |
| 736 | status_log.close() |
| 737 | self.__parse_status(lines) |
| 738 | |
| 739 | |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 740 | def _fill_server_control_namespace(self, namespace, protect=True): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 741 | """ |
| 742 | Prepare a namespace to be used when executing server control files. |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 743 | |
| 744 | This sets up the control file API by importing modules and making them |
| 745 | available under the appropriate names within namespace. |
| 746 | |
| 747 | For use by _execute_code(). |
| 748 | |
| 749 | Args: |
| 750 | namespace: The namespace dictionary to fill in. |
| 751 | protect: Boolean. If True (the default) any operation that would |
| 752 | clobber an existing entry in namespace will cause an error. |
| 753 | Raises: |
| 754 | error.AutoservError: When a name would be clobbered by import. |
| 755 | """ |
| 756 | def _import_names(module_name, names=()): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 757 | """ |
| 758 | Import a module and assign named attributes into namespace. |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 759 | |
| 760 | Args: |
| 761 | module_name: The string module name. |
| 762 | names: A limiting list of names to import from module_name. If |
| 763 | empty (the default), all names are imported from the module |
| 764 | similar to a "from foo.bar import *" statement. |
| 765 | Raises: |
| 766 | error.AutoservError: When a name being imported would clobber |
| 767 | a name already in namespace. |
| 768 | """ |
| 769 | module = __import__(module_name, {}, {}, names) |
| 770 | |
| 771 | # No names supplied? Import * from the lowest level module. |
| 772 | # (Ugh, why do I have to implement this part myself?) |
| 773 | if not names: |
| 774 | for submodule_name in module_name.split('.')[1:]: |
| 775 | module = getattr(module, submodule_name) |
| 776 | if hasattr(module, '__all__'): |
| 777 | names = getattr(module, '__all__') |
| 778 | else: |
| 779 | names = dir(module) |
| 780 | |
| 781 | # Install each name into namespace, checking to make sure it |
| 782 | # doesn't override anything that already exists. |
| 783 | for name in names: |
| 784 | # Check for conflicts to help prevent future problems. |
| 785 | if name in namespace and protect: |
| 786 | if namespace[name] is not getattr(module, name): |
| 787 | raise error.AutoservError('importing name ' |
| 788 | '%s from %s %r would override %r' % |
| 789 | (name, module_name, getattr(module, name), |
| 790 | namespace[name])) |
| 791 | else: |
| 792 | # Encourage cleanliness and the use of __all__ for a |
| 793 | # more concrete API with less surprises on '*' imports. |
| 794 | warnings.warn('%s (%r) being imported from %s for use ' |
| 795 | 'in server control files is not the ' |
| 796 | 'first occurrance of that import.' % |
| 797 | (name, namespace[name], module_name)) |
| 798 | |
| 799 | namespace[name] = getattr(module, name) |
| 800 | |
| 801 | |
| 802 | # This is the equivalent of prepending a bunch of import statements to |
| 803 | # the front of the control script. |
| 804 | namespace.update(os=os, sys=sys) |
| 805 | _import_names('autotest_lib.server', |
| 806 | ('hosts', 'autotest', 'kvm', 'git', 'standalone_profiler', |
| 807 | 'source_kernel', 'rpm_kernel', 'deb_kernel', 'git_kernel')) |
| 808 | _import_names('autotest_lib.server.subcommand', |
| 809 | ('parallel', 'parallel_simple', 'subcommand')) |
| 810 | _import_names('autotest_lib.server.utils', |
| 811 | ('run', 'get_tmp_dir', 'sh_escape', 'parse_machine')) |
| 812 | _import_names('autotest_lib.client.common_lib.error') |
| 813 | _import_names('autotest_lib.client.common_lib.barrier', ('barrier',)) |
| 814 | |
| 815 | # Inject ourself as the job object into other classes within the API. |
| 816 | # (Yuck, this injection is a gross thing be part of a public API. -gps) |
| 817 | # |
| 818 | # XXX Base & SiteAutotest do not appear to use .job. Who does? |
| 819 | namespace['autotest'].Autotest.job = self |
| 820 | # server.hosts.base_classes.Host uses .job. |
| 821 | namespace['hosts'].Host.job = self |
| 822 | |
| 823 | |
| 824 | def _execute_code(self, code_file, namespace, protect=True): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 825 | """ |
| 826 | Execute code using a copy of namespace as a server control script. |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 827 | |
| 828 | Unless protect_namespace is explicitly set to False, the dict will not |
| 829 | be modified. |
| 830 | |
| 831 | Args: |
| 832 | code_file: The filename of the control file to execute. |
| 833 | namespace: A dict containing names to make available during execution. |
| 834 | protect: Boolean. If True (the default) a copy of the namespace dict |
| 835 | is used during execution to prevent the code from modifying its |
| 836 | contents outside of this function. If False the raw dict is |
| 837 | passed in and modifications will be allowed. |
| 838 | """ |
| 839 | if protect: |
| 840 | namespace = namespace.copy() |
| 841 | self._fill_server_control_namespace(namespace, protect=protect) |
| 842 | # TODO: Simplify and get rid of the special cases for only 1 machine. |
showard | 3e66e8c | 2008-10-27 19:20:51 +0000 | [diff] [blame] | 843 | if len(self.machines) > 1: |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 844 | machines_text = '\n'.join(self.machines) + '\n' |
| 845 | # Only rewrite the file if it does not match our machine list. |
| 846 | try: |
| 847 | machines_f = open(MACHINES_FILENAME, 'r') |
| 848 | existing_machines_text = machines_f.read() |
| 849 | machines_f.close() |
| 850 | except EnvironmentError: |
| 851 | existing_machines_text = None |
| 852 | if machines_text != existing_machines_text: |
| 853 | utils.open_write_close(MACHINES_FILENAME, machines_text) |
| 854 | execfile(code_file, namespace, namespace) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 855 | |
| 856 | |
| 857 | def _record(self, status_code, subdir, operation, status='', |
| 858 | epoch_time=None, optional_fields=None): |
| 859 | """ |
| 860 | Actual function for recording a single line into the status |
| 861 | logs. Should never be called directly, only by job.record as |
| 862 | this would bypass the console monitor logging. |
| 863 | """ |
| 864 | |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 865 | msg = self._render_record(status_code, subdir, operation, status, |
| 866 | epoch_time, optional_fields=optional_fields) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 867 | |
| 868 | |
| 869 | status_file = os.path.join(self.resultdir, 'status.log') |
| 870 | sys.stdout.write(msg) |
| 871 | open(status_file, "a").write(msg) |
| 872 | if subdir: |
| 873 | test_dir = os.path.join(self.resultdir, subdir) |
jadmanski | 5ff5535 | 2008-09-18 19:43:46 +0000 | [diff] [blame] | 874 | status_file = os.path.join(test_dir, 'status.log') |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 875 | open(status_file, "a").write(msg) |
| 876 | self.__parse_status(msg.splitlines()) |
| 877 | |
| 878 | |
| 879 | def __parse_status(self, new_lines): |
| 880 | if not self.using_parser: |
| 881 | return |
| 882 | new_tests = self.parser.process_lines(new_lines) |
| 883 | for test in new_tests: |
| 884 | self.__insert_test(test) |
| 885 | |
| 886 | |
| 887 | def __insert_test(self, test): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 888 | """ |
| 889 | An internal method to insert a new test result into the |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 890 | database. This method will not raise an exception, even if an |
| 891 | error occurs during the insert, to avoid failing a test |
| 892 | simply because of unexpected database issues.""" |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 893 | self.num_tests_run += 1 |
| 894 | if status_lib.is_worse_than_or_equal_to(test.status, 'FAIL'): |
| 895 | self.num_tests_failed += 1 |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 896 | try: |
| 897 | self.results_db.insert_test(self.job_model, test) |
| 898 | except Exception: |
| 899 | msg = ("WARNING: An unexpected error occured while " |
| 900 | "inserting test results into the database. " |
| 901 | "Ignoring error.\n" + traceback.format_exc()) |
| 902 | print >> sys.stderr, msg |
| 903 | |
mbligh | caa62c2 | 2008-04-07 21:51:17 +0000 | [diff] [blame] | 904 | |
mbligh | a700772 | 2009-01-13 00:37:11 +0000 | [diff] [blame] | 905 | site_server_job = utils.import_site_class( |
| 906 | __file__, "autotest_lib.server.site_server_job", "site_server_job", |
| 907 | base_server_job) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 908 | |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 909 | class server_job(site_server_job, base_server_job): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 910 | pass |
jadmanski | f37df84 | 2009-02-11 00:03:26 +0000 | [diff] [blame^] | 911 | |
| 912 | |
| 913 | class warning_manager(object): |
| 914 | """Class for controlling warning logs. Manages the enabling and disabling |
| 915 | of warnings.""" |
| 916 | def __init__(self): |
| 917 | # a map of warning types to a list of disabled time intervals |
| 918 | self.disabled_warnings = {} |
| 919 | |
| 920 | |
| 921 | def is_valid(self, timestamp, warning_type): |
| 922 | """Indicates if a warning (based on the time it occured and its type) |
| 923 | is a valid warning. A warning is considered "invalid" if this type of |
| 924 | warning was marked as "disabled" at the time the warning occured.""" |
| 925 | disabled_intervals = self.disabled_warnings.get(warning_type, []) |
| 926 | for start, end in disabled_intervals: |
| 927 | if timestamp >= start and (end is None or timestamp < end): |
| 928 | return False |
| 929 | return True |
| 930 | |
| 931 | |
| 932 | def disable_warnings(self, warning_type, current_time_func=time.time): |
| 933 | """As of now, disables all further warnings of this type.""" |
| 934 | intervals = self.disabled_warnings.setdefault(warning_type, []) |
| 935 | if not intervals or intervals[-1][1] is not None: |
| 936 | intervals.append((current_time_func(), None)) |
| 937 | |
| 938 | |
| 939 | def enable_warnings(self, warning_type, current_time_func=time.time): |
| 940 | """As of now, enables all further warnings of this type.""" |
| 941 | intervals = self.disabled_warnings.get(warning_type, []) |
| 942 | if intervals and intervals[-1][1] is None: |
| 943 | intervals[-1] = (intervals[-1][0], current_time_func()) |