Dan Shi | 07e09af | 2013-04-12 09:31:29 -0700 | [diff] [blame] | 1 | # pylint: disable-msg=C0111 |
| 2 | |
Paul Pendlebury | 7c1fdcf | 2011-05-04 12:39:15 -0700 | [diff] [blame] | 3 | # Copyright (c) 2011 The Chromium OS Authors. All rights reserved. |
| 4 | # Use of this source code is governed by a BSD-style license that can be |
| 5 | # found in the LICENSE file. |
mbligh | 57e7866 | 2008-06-17 19:53:49 +0000 | [diff] [blame] | 6 | """ |
| 7 | The main job wrapper for the server side. |
| 8 | |
| 9 | This is the core infrastructure. Derived from the client side job.py |
| 10 | |
| 11 | Copyright Martin J. Bligh, Andy Whitcroft 2007 |
| 12 | """ |
| 13 | |
Scott Zawalski | 91493c8 | 2013-01-25 16:15:20 -0500 | [diff] [blame] | 14 | import getpass, os, sys, re, tempfile, time, select, platform |
mbligh | fc3da5b | 2010-01-06 18:37:22 +0000 | [diff] [blame] | 15 | import traceback, shutil, warnings, fcntl, pickle, logging, itertools, errno |
showard | 75cdfee | 2009-06-10 17:40:41 +0000 | [diff] [blame] | 16 | from autotest_lib.client.bin import sysinfo |
Alex Miller | 44ae923 | 2014-06-20 17:24:25 -0700 | [diff] [blame] | 17 | from autotest_lib.client.common_lib import base_job, global_config |
Scott Zawalski | 91493c8 | 2013-01-25 16:15:20 -0500 | [diff] [blame] | 18 | from autotest_lib.client.common_lib import error, utils, packages |
showard | 75cdfee | 2009-06-10 17:40:41 +0000 | [diff] [blame] | 19 | from autotest_lib.client.common_lib import logging_manager |
Paul Pendlebury | 5759356 | 2011-06-15 10:45:49 -0700 | [diff] [blame] | 20 | from autotest_lib.server import test, subcommand, profilers |
beeps | d067268 | 2013-09-16 17:32:16 -0700 | [diff] [blame] | 21 | from autotest_lib.server.hosts import abstract_ssh, factory as host_factory |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 22 | from autotest_lib.tko import db as tko_db, status_lib, utils as tko_utils |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 23 | |
| 24 | |
Alex Miller | 44ae923 | 2014-06-20 17:24:25 -0700 | [diff] [blame] | 25 | INCREMENTAL_TKO_PARSING = global_config.global_config.get_config_value( |
| 26 | 'autoserv', 'incremental_tko_parsing', type=bool, default=False) |
| 27 | |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 28 | def _control_segment_path(name): |
| 29 | """Get the pathname of the named control segment file.""" |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 30 | server_dir = os.path.dirname(os.path.abspath(__file__)) |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 31 | return os.path.join(server_dir, "control_segments", name) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 32 | |
| 33 | |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 34 | CLIENT_CONTROL_FILENAME = 'control' |
| 35 | SERVER_CONTROL_FILENAME = 'control.srv' |
| 36 | MACHINES_FILENAME = '.machines' |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 37 | |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 38 | CLIENT_WRAPPER_CONTROL_FILE = _control_segment_path('client_wrapper') |
| 39 | CRASHDUMPS_CONTROL_FILE = _control_segment_path('crashdumps') |
| 40 | CRASHINFO_CONTROL_FILE = _control_segment_path('crashinfo') |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 41 | INSTALL_CONTROL_FILE = _control_segment_path('install') |
showard | 45ae819 | 2008-11-05 19:32:53 +0000 | [diff] [blame] | 42 | CLEANUP_CONTROL_FILE = _control_segment_path('cleanup') |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 43 | VERIFY_CONTROL_FILE = _control_segment_path('verify') |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 44 | REPAIR_CONTROL_FILE = _control_segment_path('repair') |
Alex Miller | cb79ba7 | 2013-05-29 14:43:00 -0700 | [diff] [blame] | 45 | PROVISION_CONTROL_FILE = _control_segment_path('provision') |
beeps | cb6f1e2 | 2013-06-28 19:14:10 -0700 | [diff] [blame] | 46 | VERIFY_JOB_REPO_URL_CONTROL_FILE = _control_segment_path('verify_job_repo_url') |
Dan Shi | 07e09af | 2013-04-12 09:31:29 -0700 | [diff] [blame] | 47 | RESET_CONTROL_FILE = _control_segment_path('reset') |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 48 | |
mbligh | 062ed15 | 2009-01-13 00:57:14 +0000 | [diff] [blame] | 49 | # by default provide a stub that generates no site data |
| 50 | def _get_site_job_data_dummy(job): |
| 51 | return {} |
| 52 | |
| 53 | |
jadmanski | 2a89dac | 2010-06-11 14:32:58 +0000 | [diff] [blame] | 54 | class status_indenter(base_job.status_indenter): |
| 55 | """Provide a simple integer-backed status indenter.""" |
| 56 | def __init__(self): |
| 57 | self._indent = 0 |
| 58 | |
| 59 | |
| 60 | @property |
| 61 | def indent(self): |
| 62 | return self._indent |
| 63 | |
| 64 | |
| 65 | def increment(self): |
| 66 | self._indent += 1 |
| 67 | |
| 68 | |
| 69 | def decrement(self): |
| 70 | self._indent -= 1 |
| 71 | |
| 72 | |
jadmanski | 5205363 | 2010-06-11 21:08:10 +0000 | [diff] [blame] | 73 | def get_context(self): |
| 74 | """Returns a context object for use by job.get_record_context.""" |
| 75 | class context(object): |
| 76 | def __init__(self, indenter, indent): |
| 77 | self._indenter = indenter |
| 78 | self._indent = indent |
| 79 | def restore(self): |
| 80 | self._indenter._indent = self._indent |
| 81 | return context(self, self._indent) |
| 82 | |
| 83 | |
jadmanski | 2a89dac | 2010-06-11 14:32:58 +0000 | [diff] [blame] | 84 | class server_job_record_hook(object): |
| 85 | """The job.record hook for server job. Used to inject WARN messages from |
| 86 | the console or vlm whenever new logs are written, and to echo any logs |
| 87 | to INFO level logging. Implemented as a class so that it can use state to |
| 88 | block recursive calls, so that the hook can call job.record itself to |
| 89 | log WARN messages. |
| 90 | |
| 91 | Depends on job._read_warnings and job._logger. |
| 92 | """ |
| 93 | def __init__(self, job): |
| 94 | self._job = job |
| 95 | self._being_called = False |
| 96 | |
| 97 | |
| 98 | def __call__(self, entry): |
| 99 | """A wrapper around the 'real' record hook, the _hook method, which |
| 100 | prevents recursion. This isn't making any effort to be threadsafe, |
| 101 | the intent is to outright block infinite recursion via a |
| 102 | job.record->_hook->job.record->_hook->job.record... chain.""" |
| 103 | if self._being_called: |
| 104 | return |
| 105 | self._being_called = True |
| 106 | try: |
| 107 | self._hook(self._job, entry) |
| 108 | finally: |
| 109 | self._being_called = False |
| 110 | |
| 111 | |
| 112 | @staticmethod |
| 113 | def _hook(job, entry): |
| 114 | """The core hook, which can safely call job.record.""" |
| 115 | entries = [] |
| 116 | # poll all our warning loggers for new warnings |
| 117 | for timestamp, msg in job._read_warnings(): |
| 118 | warning_entry = base_job.status_log_entry( |
| 119 | 'WARN', None, None, msg, {}, timestamp=timestamp) |
| 120 | entries.append(warning_entry) |
| 121 | job.record_entry(warning_entry) |
| 122 | # echo rendered versions of all the status logs to info |
| 123 | entries.append(entry) |
| 124 | for entry in entries: |
| 125 | rendered_entry = job._logger.render_entry(entry) |
| 126 | logging.info(rendered_entry) |
jadmanski | e29d0e4 | 2010-06-17 16:06:52 +0000 | [diff] [blame] | 127 | job._parse_status(rendered_entry) |
jadmanski | 2a89dac | 2010-06-11 14:32:58 +0000 | [diff] [blame] | 128 | |
| 129 | |
mbligh | 0d0f67d | 2009-11-06 03:15:03 +0000 | [diff] [blame] | 130 | class base_server_job(base_job.base_job): |
| 131 | """The server-side concrete implementation of base_job. |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 132 | |
mbligh | 0d0f67d | 2009-11-06 03:15:03 +0000 | [diff] [blame] | 133 | Optional properties provided by this implementation: |
| 134 | serverdir |
| 135 | conmuxdir |
| 136 | |
| 137 | num_tests_run |
| 138 | num_tests_failed |
| 139 | |
| 140 | warning_manager |
| 141 | warning_loggers |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 142 | """ |
| 143 | |
mbligh | 0d0f67d | 2009-11-06 03:15:03 +0000 | [diff] [blame] | 144 | _STATUS_VERSION = 1 |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 145 | |
Aviv Keshet | c5947fa | 2013-09-04 14:06:29 -0700 | [diff] [blame] | 146 | # TODO crbug.com/285395 eliminate ssh_verbosity_flag |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 147 | def __init__(self, control, args, resultdir, label, user, machines, |
| 148 | client=False, parse_job='', |
beeps | d067268 | 2013-09-16 17:32:16 -0700 | [diff] [blame] | 149 | ssh_user=host_factory.DEFAULT_SSH_USER, |
| 150 | ssh_port=host_factory.DEFAULT_SSH_PORT, |
| 151 | ssh_pass=host_factory.DEFAULT_SSH_PASS, |
| 152 | ssh_verbosity_flag=host_factory.DEFAULT_SSH_VERBOSITY, |
| 153 | ssh_options=host_factory.DEFAULT_SSH_OPTIONS, |
Aviv Keshet | c5947fa | 2013-09-04 14:06:29 -0700 | [diff] [blame] | 154 | test_retry=0, group_name='', |
Fang Deng | d1c2b73 | 2013-08-20 12:59:46 -0700 | [diff] [blame] | 155 | tag='', disable_sysinfo=False, |
mbligh | e0cbc91 | 2010-03-11 18:03:07 +0000 | [diff] [blame] | 156 | control_filename=SERVER_CONTROL_FILENAME): |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 157 | """ |
mbligh | 374f341 | 2009-05-13 21:29:45 +0000 | [diff] [blame] | 158 | Create a server side job object. |
mbligh | b5dac43 | 2008-11-27 00:38:44 +0000 | [diff] [blame] | 159 | |
mbligh | e7d9c60 | 2009-07-02 19:02:33 +0000 | [diff] [blame] | 160 | @param control: The pathname of the control file. |
| 161 | @param args: Passed to the control file. |
| 162 | @param resultdir: Where to throw the results. |
| 163 | @param label: Description of the job. |
| 164 | @param user: Username for the job (email address). |
| 165 | @param client: True if this is a client-side control file. |
| 166 | @param parse_job: string, if supplied it is the job execution tag that |
| 167 | the results will be passed through to the TKO parser with. |
| 168 | @param ssh_user: The SSH username. [root] |
| 169 | @param ssh_port: The SSH port number. [22] |
| 170 | @param ssh_pass: The SSH passphrase, if needed. |
Fang Deng | d1c2b73 | 2013-08-20 12:59:46 -0700 | [diff] [blame] | 171 | @param ssh_verbosity_flag: The SSH verbosity flag, '-v', '-vv', |
| 172 | '-vvv', or an empty string if not needed. |
Aviv Keshet | c5947fa | 2013-09-04 14:06:29 -0700 | [diff] [blame] | 173 | @param ssh_options: A string giving additional options that will be |
| 174 | included in ssh commands. |
Scott Zawalski | 91493c8 | 2013-01-25 16:15:20 -0500 | [diff] [blame] | 175 | @param test_retry: The number of times to retry a test if the test did |
| 176 | not complete successfully. |
mbligh | e7d9c60 | 2009-07-02 19:02:33 +0000 | [diff] [blame] | 177 | @param group_name: If supplied, this will be written out as |
mbligh | 374f341 | 2009-05-13 21:29:45 +0000 | [diff] [blame] | 178 | host_group_name in the keyvals file for the parser. |
mbligh | e7d9c60 | 2009-07-02 19:02:33 +0000 | [diff] [blame] | 179 | @param tag: The job execution tag from the scheduler. [optional] |
Christopher Wiley | 8a91f23 | 2013-07-09 11:02:27 -0700 | [diff] [blame] | 180 | @param disable_sysinfo: Whether we should disable the sysinfo step of |
| 181 | tests for a modest shortening of test time. [optional] |
mbligh | e0cbc91 | 2010-03-11 18:03:07 +0000 | [diff] [blame] | 182 | @param control_filename: The filename where the server control file |
| 183 | should be written in the results directory. |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 184 | """ |
Scott Zawalski | 91493c8 | 2013-01-25 16:15:20 -0500 | [diff] [blame] | 185 | super(base_server_job, self).__init__(resultdir=resultdir, |
| 186 | test_retry=test_retry) |
mbligh | 0d0f67d | 2009-11-06 03:15:03 +0000 | [diff] [blame] | 187 | path = os.path.dirname(__file__) |
Scott Zawalski | 91493c8 | 2013-01-25 16:15:20 -0500 | [diff] [blame] | 188 | self.test_retry = test_retry |
mbligh | 0d0f67d | 2009-11-06 03:15:03 +0000 | [diff] [blame] | 189 | self.control = control |
| 190 | self._uncollected_log_file = os.path.join(self.resultdir, |
| 191 | 'uncollected_logs') |
| 192 | debugdir = os.path.join(self.resultdir, 'debug') |
| 193 | if not os.path.exists(debugdir): |
| 194 | os.mkdir(debugdir) |
| 195 | |
| 196 | if user: |
| 197 | self.user = user |
| 198 | else: |
| 199 | self.user = getpass.getuser() |
| 200 | |
jadmanski | 808f4b1 | 2010-04-09 22:30:31 +0000 | [diff] [blame] | 201 | self.args = args |
Peter Mayo | 7a87576 | 2012-06-13 14:38:15 -0400 | [diff] [blame] | 202 | self.label = label |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 203 | self.machines = machines |
mbligh | 0d0f67d | 2009-11-06 03:15:03 +0000 | [diff] [blame] | 204 | self._client = client |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 205 | self.warning_loggers = set() |
jadmanski | f37df84 | 2009-02-11 00:03:26 +0000 | [diff] [blame] | 206 | self.warning_manager = warning_manager() |
mbligh | 0d0f67d | 2009-11-06 03:15:03 +0000 | [diff] [blame] | 207 | self._ssh_user = ssh_user |
| 208 | self._ssh_port = ssh_port |
| 209 | self._ssh_pass = ssh_pass |
Fang Deng | d1c2b73 | 2013-08-20 12:59:46 -0700 | [diff] [blame] | 210 | self._ssh_verbosity_flag = ssh_verbosity_flag |
Aviv Keshet | c5947fa | 2013-09-04 14:06:29 -0700 | [diff] [blame] | 211 | self._ssh_options = ssh_options |
mbligh | e7d9c60 | 2009-07-02 19:02:33 +0000 | [diff] [blame] | 212 | self.tag = tag |
mbligh | 0910844 | 2008-10-15 16:27:38 +0000 | [diff] [blame] | 213 | self.last_boot_tag = None |
jadmanski | 53aaf38 | 2008-11-17 16:22:31 +0000 | [diff] [blame] | 214 | self.hosts = set() |
mbligh | 0d0f67d | 2009-11-06 03:15:03 +0000 | [diff] [blame] | 215 | self.drop_caches = False |
mbligh | b5dac43 | 2008-11-27 00:38:44 +0000 | [diff] [blame] | 216 | self.drop_caches_between_iterations = False |
mbligh | e0cbc91 | 2010-03-11 18:03:07 +0000 | [diff] [blame] | 217 | self._control_filename = control_filename |
Christopher Wiley | 8a91f23 | 2013-07-09 11:02:27 -0700 | [diff] [blame] | 218 | self._disable_sysinfo = disable_sysinfo |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 219 | |
showard | 75cdfee | 2009-06-10 17:40:41 +0000 | [diff] [blame] | 220 | self.logging = logging_manager.get_logging_manager( |
| 221 | manage_stdout_and_stderr=True, redirect_fds=True) |
| 222 | subcommand.logging_manager_object = self.logging |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 223 | |
mbligh | 0d0f67d | 2009-11-06 03:15:03 +0000 | [diff] [blame] | 224 | self.sysinfo = sysinfo.sysinfo(self.resultdir) |
jadmanski | 043e113 | 2008-11-19 17:10:32 +0000 | [diff] [blame] | 225 | self.profilers = profilers.profilers(self) |
jadmanski | c09fc15 | 2008-10-15 17:56:59 +0000 | [diff] [blame] | 226 | |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 227 | job_data = {'label' : label, 'user' : user, |
| 228 | 'hostname' : ','.join(machines), |
Eric Li | 861b2d5 | 2011-02-04 14:50:35 -0800 | [diff] [blame] | 229 | 'drone' : platform.node(), |
mbligh | 0d0f67d | 2009-11-06 03:15:03 +0000 | [diff] [blame] | 230 | 'status_version' : str(self._STATUS_VERSION), |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 231 | 'job_started' : str(int(time.time()))} |
mbligh | 374f341 | 2009-05-13 21:29:45 +0000 | [diff] [blame] | 232 | if group_name: |
| 233 | job_data['host_group_name'] = group_name |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 234 | |
mbligh | 0d0f67d | 2009-11-06 03:15:03 +0000 | [diff] [blame] | 235 | # only write these keyvals out on the first job in a resultdir |
| 236 | if 'job_started' not in utils.read_keyval(self.resultdir): |
| 237 | job_data.update(get_site_job_data(self)) |
| 238 | utils.write_keyval(self.resultdir, job_data) |
| 239 | |
| 240 | self._parse_job = parse_job |
Alex Miller | 44ae923 | 2014-06-20 17:24:25 -0700 | [diff] [blame] | 241 | self._using_parser = (INCREMENTAL_TKO_PARSING and self._parse_job |
| 242 | and len(machines) <= 1) |
mbligh | 0d0f67d | 2009-11-06 03:15:03 +0000 | [diff] [blame] | 243 | self.pkgmgr = packages.PackageManager( |
| 244 | self.autodir, run_function_dargs={'timeout':600}) |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 245 | self.num_tests_run = 0 |
| 246 | self.num_tests_failed = 0 |
| 247 | |
jadmanski | 550fdc2 | 2008-11-20 16:32:08 +0000 | [diff] [blame] | 248 | self._register_subcommand_hooks() |
| 249 | |
mbligh | 0d0f67d | 2009-11-06 03:15:03 +0000 | [diff] [blame] | 250 | # these components aren't usable on the server |
| 251 | self.bootloader = None |
| 252 | self.harness = None |
| 253 | |
jadmanski | 2a89dac | 2010-06-11 14:32:58 +0000 | [diff] [blame] | 254 | # set up the status logger |
jadmanski | 5205363 | 2010-06-11 21:08:10 +0000 | [diff] [blame] | 255 | self._indenter = status_indenter() |
jadmanski | 2a89dac | 2010-06-11 14:32:58 +0000 | [diff] [blame] | 256 | self._logger = base_job.status_logger( |
jadmanski | 5205363 | 2010-06-11 21:08:10 +0000 | [diff] [blame] | 257 | self, self._indenter, 'status.log', 'status.log', |
jadmanski | 2a89dac | 2010-06-11 14:32:58 +0000 | [diff] [blame] | 258 | record_hook=server_job_record_hook(self)) |
| 259 | |
Dan Shi | b03ea9d | 2013-08-15 17:13:27 -0700 | [diff] [blame] | 260 | # Initialize a flag to indicate DUT failure during the test, e.g., |
| 261 | # unexpected reboot. |
| 262 | self.failed_with_device_error = False |
| 263 | |
mbligh | 0d0f67d | 2009-11-06 03:15:03 +0000 | [diff] [blame] | 264 | |
| 265 | @classmethod |
| 266 | def _find_base_directories(cls): |
| 267 | """ |
| 268 | Determine locations of autodir, clientdir and serverdir. Assumes |
| 269 | that this file is located within serverdir and uses __file__ along |
| 270 | with relative paths to resolve the location. |
| 271 | """ |
| 272 | serverdir = os.path.abspath(os.path.dirname(__file__)) |
| 273 | autodir = os.path.normpath(os.path.join(serverdir, '..')) |
| 274 | clientdir = os.path.join(autodir, 'client') |
| 275 | return autodir, clientdir, serverdir |
| 276 | |
| 277 | |
Scott Zawalski | 91493c8 | 2013-01-25 16:15:20 -0500 | [diff] [blame] | 278 | def _find_resultdir(self, resultdir, *args, **dargs): |
mbligh | 0d0f67d | 2009-11-06 03:15:03 +0000 | [diff] [blame] | 279 | """ |
| 280 | Determine the location of resultdir. For server jobs we expect one to |
| 281 | always be explicitly passed in to __init__, so just return that. |
| 282 | """ |
| 283 | if resultdir: |
| 284 | return os.path.normpath(resultdir) |
| 285 | else: |
| 286 | return None |
| 287 | |
jadmanski | 550fdc2 | 2008-11-20 16:32:08 +0000 | [diff] [blame] | 288 | |
jadmanski | 2a89dac | 2010-06-11 14:32:58 +0000 | [diff] [blame] | 289 | def _get_status_logger(self): |
| 290 | """Return a reference to the status logger.""" |
| 291 | return self._logger |
| 292 | |
| 293 | |
jadmanski | e432dd2 | 2009-01-30 15:04:51 +0000 | [diff] [blame] | 294 | @staticmethod |
| 295 | def _load_control_file(path): |
| 296 | f = open(path) |
| 297 | try: |
| 298 | control_file = f.read() |
| 299 | finally: |
| 300 | f.close() |
| 301 | return re.sub('\r', '', control_file) |
| 302 | |
| 303 | |
jadmanski | 550fdc2 | 2008-11-20 16:32:08 +0000 | [diff] [blame] | 304 | def _register_subcommand_hooks(self): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 305 | """ |
| 306 | Register some hooks into the subcommand modules that allow us |
| 307 | to properly clean up self.hosts created in forked subprocesses. |
| 308 | """ |
jadmanski | 550fdc2 | 2008-11-20 16:32:08 +0000 | [diff] [blame] | 309 | def on_fork(cmd): |
| 310 | self._existing_hosts_on_fork = set(self.hosts) |
| 311 | def on_join(cmd): |
| 312 | new_hosts = self.hosts - self._existing_hosts_on_fork |
| 313 | for host in new_hosts: |
| 314 | host.close() |
| 315 | subcommand.subcommand.register_fork_hook(on_fork) |
| 316 | subcommand.subcommand.register_join_hook(on_join) |
| 317 | |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 318 | |
mbligh | 4608b00 | 2010-01-05 18:22:35 +0000 | [diff] [blame] | 319 | def init_parser(self): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 320 | """ |
mbligh | 4608b00 | 2010-01-05 18:22:35 +0000 | [diff] [blame] | 321 | Start the continuous parsing of self.resultdir. This sets up |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 322 | the database connection and inserts the basic job object into |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 323 | the database if necessary. |
| 324 | """ |
mbligh | 4608b00 | 2010-01-05 18:22:35 +0000 | [diff] [blame] | 325 | if not self._using_parser: |
| 326 | return |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 327 | # redirect parser debugging to .parse.log |
mbligh | 4608b00 | 2010-01-05 18:22:35 +0000 | [diff] [blame] | 328 | parse_log = os.path.join(self.resultdir, '.parse.log') |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 329 | parse_log = open(parse_log, 'w', 0) |
| 330 | tko_utils.redirect_parser_debugging(parse_log) |
| 331 | # create a job model object and set up the db |
| 332 | self.results_db = tko_db.db(autocommit=True) |
mbligh | 0d0f67d | 2009-11-06 03:15:03 +0000 | [diff] [blame] | 333 | self.parser = status_lib.parser(self._STATUS_VERSION) |
mbligh | 4608b00 | 2010-01-05 18:22:35 +0000 | [diff] [blame] | 334 | self.job_model = self.parser.make_job(self.resultdir) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 335 | self.parser.start(self.job_model) |
| 336 | # check if a job already exists in the db and insert it if |
| 337 | # it does not |
mbligh | 0d0f67d | 2009-11-06 03:15:03 +0000 | [diff] [blame] | 338 | job_idx = self.results_db.find_job(self._parse_job) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 339 | if job_idx is None: |
mbligh | 0d0f67d | 2009-11-06 03:15:03 +0000 | [diff] [blame] | 340 | self.results_db.insert_job(self._parse_job, self.job_model) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 341 | else: |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 342 | machine_idx = self.results_db.lookup_machine(self.job_model.machine) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 343 | self.job_model.index = job_idx |
| 344 | self.job_model.machine_idx = machine_idx |
| 345 | |
| 346 | |
| 347 | def cleanup_parser(self): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 348 | """ |
| 349 | This should be called after the server job is finished |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 350 | to carry out any remaining cleanup (e.g. flushing any |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 351 | remaining test results to the results db) |
| 352 | """ |
mbligh | 0d0f67d | 2009-11-06 03:15:03 +0000 | [diff] [blame] | 353 | if not self._using_parser: |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 354 | return |
| 355 | final_tests = self.parser.end() |
| 356 | for test in final_tests: |
| 357 | self.__insert_test(test) |
mbligh | 0d0f67d | 2009-11-06 03:15:03 +0000 | [diff] [blame] | 358 | self._using_parser = False |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 359 | |
Aviv Keshet | c5947fa | 2013-09-04 14:06:29 -0700 | [diff] [blame] | 360 | # TODO crbug.com/285395 add a kwargs parameter. |
| 361 | def _make_namespace(self): |
| 362 | """Create a namespace dictionary to be passed along to control file. |
| 363 | |
| 364 | Creates a namespace argument populated with standard values: |
| 365 | machines, job, ssh_user, ssh_port, ssh_pass, ssh_verbosity_flag, |
| 366 | and ssh_options. |
| 367 | """ |
| 368 | namespace = {'machines' : self.machines, |
| 369 | 'job' : self, |
| 370 | 'ssh_user' : self._ssh_user, |
| 371 | 'ssh_port' : self._ssh_port, |
| 372 | 'ssh_pass' : self._ssh_pass, |
| 373 | 'ssh_verbosity_flag' : self._ssh_verbosity_flag, |
| 374 | 'ssh_options' : self._ssh_options} |
| 375 | return namespace |
| 376 | |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 377 | |
Alex Miller | 667b5f2 | 2014-02-28 15:33:39 -0800 | [diff] [blame] | 378 | def verify(self, labels=''): |
Dan Shi | 07e09af | 2013-04-12 09:31:29 -0700 | [diff] [blame] | 379 | """Verify machines are all ssh-able.""" |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 380 | if not self.machines: |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 381 | raise error.AutoservError('No machines specified to verify') |
mbligh | 0fce411 | 2008-11-27 00:37:17 +0000 | [diff] [blame] | 382 | if self.resultdir: |
| 383 | os.chdir(self.resultdir) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 384 | try: |
Aviv Keshet | c5947fa | 2013-09-04 14:06:29 -0700 | [diff] [blame] | 385 | namespace = self._make_namespace() |
Alex Miller | 667b5f2 | 2014-02-28 15:33:39 -0800 | [diff] [blame] | 386 | namespace.update({'job_labels': labels, 'args': ''}) |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 387 | self._execute_code(VERIFY_CONTROL_FILE, namespace, protect=False) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 388 | except Exception, e: |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 389 | msg = ('Verify failed\n' + str(e) + '\n' + traceback.format_exc()) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 390 | raise |
| 391 | |
| 392 | |
Alex Miller | 667b5f2 | 2014-02-28 15:33:39 -0800 | [diff] [blame] | 393 | def reset(self, labels=''): |
Dan Shi | 07e09af | 2013-04-12 09:31:29 -0700 | [diff] [blame] | 394 | """Reset machines by first cleanup then verify each machine.""" |
| 395 | if not self.machines: |
| 396 | raise error.AutoservError('No machines specified to reset.') |
| 397 | if self.resultdir: |
| 398 | os.chdir(self.resultdir) |
| 399 | |
| 400 | try: |
Aviv Keshet | c5947fa | 2013-09-04 14:06:29 -0700 | [diff] [blame] | 401 | namespace = self._make_namespace() |
Alex Miller | 667b5f2 | 2014-02-28 15:33:39 -0800 | [diff] [blame] | 402 | namespace.update({'job_labels': labels, 'args': ''}) |
Dan Shi | 07e09af | 2013-04-12 09:31:29 -0700 | [diff] [blame] | 403 | self._execute_code(RESET_CONTROL_FILE, namespace, protect=False) |
| 404 | except Exception as e: |
| 405 | msg = ('Reset failed\n' + str(e) + '\n' + |
| 406 | traceback.format_exc()) |
Dan Shi | 07e09af | 2013-04-12 09:31:29 -0700 | [diff] [blame] | 407 | raise |
| 408 | |
| 409 | |
Alex Miller | 667b5f2 | 2014-02-28 15:33:39 -0800 | [diff] [blame] | 410 | def repair(self, host_protection, labels=''): |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 411 | if not self.machines: |
| 412 | raise error.AutoservError('No machines specified to repair') |
mbligh | 0fce411 | 2008-11-27 00:37:17 +0000 | [diff] [blame] | 413 | if self.resultdir: |
| 414 | os.chdir(self.resultdir) |
Aviv Keshet | c5947fa | 2013-09-04 14:06:29 -0700 | [diff] [blame] | 415 | |
| 416 | namespace = self._make_namespace() |
Alex Miller | 667b5f2 | 2014-02-28 15:33:39 -0800 | [diff] [blame] | 417 | namespace.update({'protection_level' : host_protection, |
| 418 | 'job_labels': labels, 'args': ''}) |
mbligh | 25c0b8c | 2009-01-24 01:44:17 +0000 | [diff] [blame] | 419 | |
mbligh | 0931b0a | 2009-04-08 17:44:48 +0000 | [diff] [blame] | 420 | self._execute_code(REPAIR_CONTROL_FILE, namespace, protect=False) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 421 | |
| 422 | |
Alex Miller | cb79ba7 | 2013-05-29 14:43:00 -0700 | [diff] [blame] | 423 | def provision(self, labels): |
| 424 | """ |
| 425 | Provision all hosts to match |labels|. |
| 426 | |
| 427 | @param labels: A comma seperated string of labels to provision the |
| 428 | host to. |
| 429 | |
| 430 | """ |
Alex Miller | cb79ba7 | 2013-05-29 14:43:00 -0700 | [diff] [blame] | 431 | control = self._load_control_file(PROVISION_CONTROL_FILE) |
Alex Miller | 686fca8 | 2014-04-23 17:21:13 -0700 | [diff] [blame] | 432 | self.run(control=control, job_labels=labels) |
Alex Miller | cb79ba7 | 2013-05-29 14:43:00 -0700 | [diff] [blame] | 433 | |
| 434 | |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 435 | def precheck(self): |
| 436 | """ |
| 437 | perform any additional checks in derived classes. |
| 438 | """ |
| 439 | pass |
| 440 | |
| 441 | |
| 442 | def enable_external_logging(self): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 443 | """ |
| 444 | Start or restart external logging mechanism. |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 445 | """ |
| 446 | pass |
| 447 | |
| 448 | |
| 449 | def disable_external_logging(self): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 450 | """ |
| 451 | Pause or stop external logging mechanism. |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 452 | """ |
| 453 | pass |
| 454 | |
| 455 | |
| 456 | def use_external_logging(self): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 457 | """ |
| 458 | Return True if external logging should be used. |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 459 | """ |
| 460 | return False |
| 461 | |
| 462 | |
mbligh | 415dc21 | 2009-06-15 21:53:34 +0000 | [diff] [blame] | 463 | def _make_parallel_wrapper(self, function, machines, log): |
| 464 | """Wrap function as appropriate for calling by parallel_simple.""" |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 465 | is_forking = not (len(machines) == 1 and self.machines == machines) |
mbligh | 0d0f67d | 2009-11-06 03:15:03 +0000 | [diff] [blame] | 466 | if self._parse_job and is_forking and log: |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 467 | def wrapper(machine): |
mbligh | 0d0f67d | 2009-11-06 03:15:03 +0000 | [diff] [blame] | 468 | self._parse_job += "/" + machine |
Alex Miller | 44ae923 | 2014-06-20 17:24:25 -0700 | [diff] [blame] | 469 | self._using_parser = INCREMENTAL_TKO_PARSING |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 470 | self.machines = [machine] |
mbligh | 0d0f67d | 2009-11-06 03:15:03 +0000 | [diff] [blame] | 471 | self.push_execution_context(machine) |
jadmanski | 609a5f4 | 2008-08-26 20:52:42 +0000 | [diff] [blame] | 472 | os.chdir(self.resultdir) |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 473 | utils.write_keyval(self.resultdir, {"hostname": machine}) |
mbligh | 4608b00 | 2010-01-05 18:22:35 +0000 | [diff] [blame] | 474 | self.init_parser() |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 475 | result = function(machine) |
| 476 | self.cleanup_parser() |
| 477 | return result |
jadmanski | 4dd1a00 | 2008-09-05 20:27:30 +0000 | [diff] [blame] | 478 | elif len(machines) > 1 and log: |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 479 | def wrapper(machine): |
mbligh | 0d0f67d | 2009-11-06 03:15:03 +0000 | [diff] [blame] | 480 | self.push_execution_context(machine) |
jadmanski | 609a5f4 | 2008-08-26 20:52:42 +0000 | [diff] [blame] | 481 | os.chdir(self.resultdir) |
mbligh | 838d82d | 2009-03-11 17:14:31 +0000 | [diff] [blame] | 482 | machine_data = {'hostname' : machine, |
mbligh | 0d0f67d | 2009-11-06 03:15:03 +0000 | [diff] [blame] | 483 | 'status_version' : str(self._STATUS_VERSION)} |
mbligh | 838d82d | 2009-03-11 17:14:31 +0000 | [diff] [blame] | 484 | utils.write_keyval(self.resultdir, machine_data) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 485 | result = function(machine) |
| 486 | return result |
| 487 | else: |
| 488 | wrapper = function |
mbligh | 415dc21 | 2009-06-15 21:53:34 +0000 | [diff] [blame] | 489 | return wrapper |
| 490 | |
| 491 | |
| 492 | def parallel_simple(self, function, machines, log=True, timeout=None, |
| 493 | return_results=False): |
| 494 | """ |
| 495 | Run 'function' using parallel_simple, with an extra wrapper to handle |
| 496 | the necessary setup for continuous parsing, if possible. If continuous |
| 497 | parsing is already properly initialized then this should just work. |
| 498 | |
| 499 | @param function: A callable to run in parallel given each machine. |
| 500 | @param machines: A list of machine names to be passed one per subcommand |
| 501 | invocation of function. |
| 502 | @param log: If True, output will be written to output in a subdirectory |
| 503 | named after each machine. |
| 504 | @param timeout: Seconds after which the function call should timeout. |
| 505 | @param return_results: If True instead of an AutoServError being raised |
| 506 | on any error a list of the results|exceptions from the function |
| 507 | called on each arg is returned. [default: False] |
| 508 | |
| 509 | @raises error.AutotestError: If any of the functions failed. |
| 510 | """ |
| 511 | wrapper = self._make_parallel_wrapper(function, machines, log) |
| 512 | return subcommand.parallel_simple(wrapper, machines, |
| 513 | log=log, timeout=timeout, |
| 514 | return_results=return_results) |
| 515 | |
| 516 | |
| 517 | def parallel_on_machines(self, function, machines, timeout=None): |
| 518 | """ |
showard | cd5fac4 | 2009-07-06 20:19:43 +0000 | [diff] [blame] | 519 | @param function: Called in parallel with one machine as its argument. |
mbligh | 415dc21 | 2009-06-15 21:53:34 +0000 | [diff] [blame] | 520 | @param machines: A list of machines to call function(machine) on. |
| 521 | @param timeout: Seconds after which the function call should timeout. |
| 522 | |
| 523 | @returns A list of machines on which function(machine) returned |
| 524 | without raising an exception. |
| 525 | """ |
showard | cd5fac4 | 2009-07-06 20:19:43 +0000 | [diff] [blame] | 526 | results = self.parallel_simple(function, machines, timeout=timeout, |
mbligh | 415dc21 | 2009-06-15 21:53:34 +0000 | [diff] [blame] | 527 | return_results=True) |
| 528 | success_machines = [] |
| 529 | for result, machine in itertools.izip(results, machines): |
| 530 | if not isinstance(result, Exception): |
| 531 | success_machines.append(machine) |
| 532 | return success_machines |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 533 | |
| 534 | |
mbligh | 0d0f67d | 2009-11-06 03:15:03 +0000 | [diff] [blame] | 535 | _USE_TEMP_DIR = object() |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 536 | def run(self, cleanup=False, install_before=False, install_after=False, |
jadmanski | e432dd2 | 2009-01-30 15:04:51 +0000 | [diff] [blame] | 537 | collect_crashdumps=True, namespace={}, control=None, |
beeps | cb6f1e2 | 2013-06-28 19:14:10 -0700 | [diff] [blame] | 538 | control_file_dir=None, verify_job_repo_url=False, |
Alex Miller | ca76bcc | 2014-04-18 18:47:28 -0700 | [diff] [blame] | 539 | only_collect_crashinfo=False, skip_crash_collection=False, |
| 540 | job_labels=''): |
jadmanski | fb9c0fa | 2009-04-29 17:39:16 +0000 | [diff] [blame] | 541 | # for a normal job, make sure the uncollected logs file exists |
| 542 | # for a crashinfo-only run it should already exist, bail out otherwise |
jadmanski | 648c39f | 2010-03-19 17:38:01 +0000 | [diff] [blame] | 543 | created_uncollected_logs = False |
Alex Miller | 45554f3 | 2013-08-13 16:48:29 -0700 | [diff] [blame] | 544 | logging.info("I am PID %s", os.getpid()) |
mbligh | 0d0f67d | 2009-11-06 03:15:03 +0000 | [diff] [blame] | 545 | if self.resultdir and not os.path.exists(self._uncollected_log_file): |
jadmanski | fb9c0fa | 2009-04-29 17:39:16 +0000 | [diff] [blame] | 546 | if only_collect_crashinfo: |
| 547 | # if this is a crashinfo-only run, and there were no existing |
| 548 | # uncollected logs, just bail out early |
| 549 | logging.info("No existing uncollected logs, " |
| 550 | "skipping crashinfo collection") |
| 551 | return |
| 552 | else: |
mbligh | 0d0f67d | 2009-11-06 03:15:03 +0000 | [diff] [blame] | 553 | log_file = open(self._uncollected_log_file, "w") |
jadmanski | fb9c0fa | 2009-04-29 17:39:16 +0000 | [diff] [blame] | 554 | pickle.dump([], log_file) |
| 555 | log_file.close() |
jadmanski | 648c39f | 2010-03-19 17:38:01 +0000 | [diff] [blame] | 556 | created_uncollected_logs = True |
jadmanski | fb9c0fa | 2009-04-29 17:39:16 +0000 | [diff] [blame] | 557 | |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 558 | # use a copy so changes don't affect the original dictionary |
| 559 | namespace = namespace.copy() |
| 560 | machines = self.machines |
jadmanski | e432dd2 | 2009-01-30 15:04:51 +0000 | [diff] [blame] | 561 | if control is None: |
jadmanski | 02a3ba2 | 2009-11-13 20:47:27 +0000 | [diff] [blame] | 562 | if self.control is None: |
| 563 | control = '' |
| 564 | else: |
| 565 | control = self._load_control_file(self.control) |
jadmanski | e432dd2 | 2009-01-30 15:04:51 +0000 | [diff] [blame] | 566 | if control_file_dir is None: |
| 567 | control_file_dir = self.resultdir |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 568 | |
| 569 | self.aborted = False |
Aviv Keshet | c5947fa | 2013-09-04 14:06:29 -0700 | [diff] [blame] | 570 | namespace.update(self._make_namespace()) |
Alex Miller | ca76bcc | 2014-04-18 18:47:28 -0700 | [diff] [blame] | 571 | namespace.update({'args' : self.args, |
| 572 | 'job_labels' : job_labels}) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 573 | test_start_time = int(time.time()) |
| 574 | |
mbligh | 80e1eba | 2008-11-19 00:26:18 +0000 | [diff] [blame] | 575 | if self.resultdir: |
| 576 | os.chdir(self.resultdir) |
jadmanski | 779bd29 | 2009-03-19 17:33:33 +0000 | [diff] [blame] | 577 | # touch status.log so that the parser knows a job is running here |
jadmanski | 382303a | 2009-04-21 19:53:39 +0000 | [diff] [blame] | 578 | open(self.get_status_log_path(), 'a').close() |
mbligh | 80e1eba | 2008-11-19 00:26:18 +0000 | [diff] [blame] | 579 | self.enable_external_logging() |
jadmanski | e432dd2 | 2009-01-30 15:04:51 +0000 | [diff] [blame] | 580 | |
jadmanski | cdd0c40 | 2008-09-19 21:21:31 +0000 | [diff] [blame] | 581 | collect_crashinfo = True |
mbligh | aebe3b6 | 2008-12-22 14:45:40 +0000 | [diff] [blame] | 582 | temp_control_file_dir = None |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 583 | try: |
showard | cf8d492 | 2009-10-14 16:08:39 +0000 | [diff] [blame] | 584 | try: |
| 585 | if install_before and machines: |
| 586 | self._execute_code(INSTALL_CONTROL_FILE, namespace) |
jadmanski | e432dd2 | 2009-01-30 15:04:51 +0000 | [diff] [blame] | 587 | |
showard | cf8d492 | 2009-10-14 16:08:39 +0000 | [diff] [blame] | 588 | if only_collect_crashinfo: |
| 589 | return |
| 590 | |
beeps | cb6f1e2 | 2013-06-28 19:14:10 -0700 | [diff] [blame] | 591 | # If the verify_job_repo_url option is set but we're unable |
| 592 | # to actually verify that the job_repo_url contains the autotest |
| 593 | # package, this job will fail. |
| 594 | if verify_job_repo_url: |
| 595 | self._execute_code(VERIFY_JOB_REPO_URL_CONTROL_FILE, |
| 596 | namespace) |
| 597 | else: |
| 598 | logging.warning('Not checking if job_repo_url contains ' |
| 599 | 'autotest packages on %s', machines) |
| 600 | |
jadmanski | def0c3c | 2009-03-25 20:07:10 +0000 | [diff] [blame] | 601 | # determine the dir to write the control files to |
| 602 | cfd_specified = (control_file_dir |
mbligh | 0d0f67d | 2009-11-06 03:15:03 +0000 | [diff] [blame] | 603 | and control_file_dir is not self._USE_TEMP_DIR) |
jadmanski | def0c3c | 2009-03-25 20:07:10 +0000 | [diff] [blame] | 604 | if cfd_specified: |
| 605 | temp_control_file_dir = None |
| 606 | else: |
| 607 | temp_control_file_dir = tempfile.mkdtemp( |
| 608 | suffix='temp_control_file_dir') |
| 609 | control_file_dir = temp_control_file_dir |
| 610 | server_control_file = os.path.join(control_file_dir, |
mbligh | e0cbc91 | 2010-03-11 18:03:07 +0000 | [diff] [blame] | 611 | self._control_filename) |
jadmanski | def0c3c | 2009-03-25 20:07:10 +0000 | [diff] [blame] | 612 | client_control_file = os.path.join(control_file_dir, |
| 613 | CLIENT_CONTROL_FILENAME) |
mbligh | 0d0f67d | 2009-11-06 03:15:03 +0000 | [diff] [blame] | 614 | if self._client: |
jadmanski | def0c3c | 2009-03-25 20:07:10 +0000 | [diff] [blame] | 615 | namespace['control'] = control |
| 616 | utils.open_write_close(client_control_file, control) |
mbligh | feac010 | 2009-04-28 18:31:12 +0000 | [diff] [blame] | 617 | shutil.copyfile(CLIENT_WRAPPER_CONTROL_FILE, |
| 618 | server_control_file) |
jadmanski | def0c3c | 2009-03-25 20:07:10 +0000 | [diff] [blame] | 619 | else: |
| 620 | utils.open_write_close(server_control_file, control) |
mbligh | 26f0d88 | 2009-06-22 18:30:01 +0000 | [diff] [blame] | 621 | logging.info("Processing control file") |
jadmanski | def0c3c | 2009-03-25 20:07:10 +0000 | [diff] [blame] | 622 | self._execute_code(server_control_file, namespace) |
mbligh | 26f0d88 | 2009-06-22 18:30:01 +0000 | [diff] [blame] | 623 | logging.info("Finished processing control file") |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 624 | |
Dan Shi | b03ea9d | 2013-08-15 17:13:27 -0700 | [diff] [blame] | 625 | # If no device error occured, no need to collect crashinfo. |
| 626 | collect_crashinfo = self.failed_with_device_error |
Eric Li | 6f27d4f | 2010-09-29 10:55:17 -0700 | [diff] [blame] | 627 | except Exception, e: |
showard | cf8d492 | 2009-10-14 16:08:39 +0000 | [diff] [blame] | 628 | try: |
| 629 | logging.exception( |
| 630 | 'Exception escaped control file, job aborting:') |
Eric Li | 6f27d4f | 2010-09-29 10:55:17 -0700 | [diff] [blame] | 631 | self.record('INFO', None, None, str(e), |
| 632 | {'job_abort_reason': str(e)}) |
showard | cf8d492 | 2009-10-14 16:08:39 +0000 | [diff] [blame] | 633 | except: |
| 634 | pass # don't let logging exceptions here interfere |
| 635 | raise |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 636 | finally: |
mbligh | aebe3b6 | 2008-12-22 14:45:40 +0000 | [diff] [blame] | 637 | if temp_control_file_dir: |
jadmanski | e432dd2 | 2009-01-30 15:04:51 +0000 | [diff] [blame] | 638 | # Clean up temp directory used for copies of the control files |
mbligh | aebe3b6 | 2008-12-22 14:45:40 +0000 | [diff] [blame] | 639 | try: |
| 640 | shutil.rmtree(temp_control_file_dir) |
| 641 | except Exception, e: |
Ilja H. Friedel | 04be2bd | 2014-05-07 21:29:59 -0700 | [diff] [blame] | 642 | logging.warning('Could not remove temp directory %s: %s', |
mbligh | e7d9c60 | 2009-07-02 19:02:33 +0000 | [diff] [blame] | 643 | temp_control_file_dir, e) |
jadmanski | e432dd2 | 2009-01-30 15:04:51 +0000 | [diff] [blame] | 644 | |
jadmanski | cdd0c40 | 2008-09-19 21:21:31 +0000 | [diff] [blame] | 645 | if machines and (collect_crashdumps or collect_crashinfo): |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 646 | namespace['test_start_time'] = test_start_time |
Christopher Wiley | f594c5e | 2013-07-03 18:25:30 -0700 | [diff] [blame] | 647 | if skip_crash_collection: |
| 648 | logging.info('Skipping crash dump/info collection ' |
| 649 | 'as requested.') |
| 650 | elif collect_crashinfo: |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 651 | # includes crashdumps |
| 652 | self._execute_code(CRASHINFO_CONTROL_FILE, namespace) |
jadmanski | cdd0c40 | 2008-09-19 21:21:31 +0000 | [diff] [blame] | 653 | else: |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 654 | self._execute_code(CRASHDUMPS_CONTROL_FILE, namespace) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 655 | self.disable_external_logging() |
showard | 45ae819 | 2008-11-05 19:32:53 +0000 | [diff] [blame] | 656 | if cleanup and machines: |
| 657 | self._execute_code(CLEANUP_CONTROL_FILE, namespace) |
Chris Sosa | f4d43ff | 2012-10-30 11:21:05 -0700 | [diff] [blame] | 658 | if self._uncollected_log_file and created_uncollected_logs: |
| 659 | os.remove(self._uncollected_log_file) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 660 | if install_after and machines: |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 661 | self._execute_code(INSTALL_CONTROL_FILE, namespace) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 662 | |
| 663 | |
| 664 | def run_test(self, url, *args, **dargs): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 665 | """ |
| 666 | Summon a test object and run it. |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 667 | |
| 668 | tag |
| 669 | tag to add to testname |
| 670 | url |
| 671 | url of the test to run |
| 672 | """ |
Christopher Wiley | 8a91f23 | 2013-07-09 11:02:27 -0700 | [diff] [blame] | 673 | if self._disable_sysinfo: |
| 674 | dargs['disable_sysinfo'] = True |
| 675 | |
mbligh | fc3da5b | 2010-01-06 18:37:22 +0000 | [diff] [blame] | 676 | group, testname = self.pkgmgr.get_package_name(url, 'test') |
| 677 | testname, subdir, tag = self._build_tagged_test_name(testname, dargs) |
| 678 | outputdir = self._make_test_outputdir(subdir) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 679 | |
| 680 | def group_func(): |
| 681 | try: |
| 682 | test.runtest(self, url, tag, args, dargs) |
| 683 | except error.TestBaseException, e: |
| 684 | self.record(e.exit_status, subdir, testname, str(e)) |
| 685 | raise |
| 686 | except Exception, e: |
| 687 | info = str(e) + "\n" + traceback.format_exc() |
| 688 | self.record('FAIL', subdir, testname, info) |
| 689 | raise |
| 690 | else: |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 691 | self.record('GOOD', subdir, testname, 'completed successfully') |
jadmanski | de292df | 2008-08-26 20:51:14 +0000 | [diff] [blame] | 692 | |
| 693 | result, exc_info = self._run_group(testname, subdir, group_func) |
| 694 | if exc_info and isinstance(exc_info[1], error.TestBaseException): |
| 695 | return False |
| 696 | elif exc_info: |
| 697 | raise exc_info[0], exc_info[1], exc_info[2] |
| 698 | else: |
| 699 | return True |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 700 | |
| 701 | |
| 702 | def _run_group(self, name, subdir, function, *args, **dargs): |
| 703 | """\ |
| 704 | Underlying method for running something inside of a group. |
| 705 | """ |
jadmanski | de292df | 2008-08-26 20:51:14 +0000 | [diff] [blame] | 706 | result, exc_info = None, None |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 707 | try: |
| 708 | self.record('START', subdir, name) |
jadmanski | 5205363 | 2010-06-11 21:08:10 +0000 | [diff] [blame] | 709 | result = function(*args, **dargs) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 710 | except error.TestBaseException, e: |
jadmanski | b88d6dc | 2009-01-10 00:33:18 +0000 | [diff] [blame] | 711 | self.record("END %s" % e.exit_status, subdir, name) |
jadmanski | de292df | 2008-08-26 20:51:14 +0000 | [diff] [blame] | 712 | exc_info = sys.exc_info() |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 713 | except Exception, e: |
| 714 | err_msg = str(e) + '\n' |
| 715 | err_msg += traceback.format_exc() |
| 716 | self.record('END ABORT', subdir, name, err_msg) |
| 717 | raise error.JobError(name + ' failed\n' + traceback.format_exc()) |
| 718 | else: |
| 719 | self.record('END GOOD', subdir, name) |
| 720 | |
jadmanski | de292df | 2008-08-26 20:51:14 +0000 | [diff] [blame] | 721 | return result, exc_info |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 722 | |
| 723 | |
| 724 | def run_group(self, function, *args, **dargs): |
| 725 | """\ |
| 726 | function: |
| 727 | subroutine to run |
| 728 | *args: |
| 729 | arguments for the function |
| 730 | """ |
| 731 | |
| 732 | name = function.__name__ |
| 733 | |
| 734 | # Allow the tag for the group to be specified. |
| 735 | tag = dargs.pop('tag', None) |
| 736 | if tag: |
| 737 | name = tag |
| 738 | |
jadmanski | de292df | 2008-08-26 20:51:14 +0000 | [diff] [blame] | 739 | return self._run_group(name, None, function, *args, **dargs)[0] |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 740 | |
| 741 | |
Gwendal Grignou | 7a61d2f | 2014-05-23 11:05:51 -0700 | [diff] [blame] | 742 | def run_op(self, op, op_func, get_kernel_func): |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 743 | """\ |
| 744 | A specialization of run_group meant specifically for handling |
Gwendal Grignou | 7a61d2f | 2014-05-23 11:05:51 -0700 | [diff] [blame] | 745 | management operation. Includes support for capturing the kernel version |
| 746 | after the operation. |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 747 | |
Gwendal Grignou | 7a61d2f | 2014-05-23 11:05:51 -0700 | [diff] [blame] | 748 | Args: |
| 749 | op: name of the operation. |
| 750 | op_func: a function that carries out the operation (reboot, suspend) |
| 751 | get_kernel_func: a function that returns a string |
| 752 | representing the kernel version. |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 753 | """ |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 754 | try: |
Gwendal Grignou | 7a61d2f | 2014-05-23 11:05:51 -0700 | [diff] [blame] | 755 | self.record('START', None, op) |
| 756 | op_func() |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 757 | except Exception, e: |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 758 | err_msg = str(e) + '\n' + traceback.format_exc() |
Gwendal Grignou | 7a61d2f | 2014-05-23 11:05:51 -0700 | [diff] [blame] | 759 | self.record('END FAIL', None, op, err_msg) |
jadmanski | 4b51d54 | 2009-04-08 14:17:16 +0000 | [diff] [blame] | 760 | raise |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 761 | else: |
| 762 | kernel = get_kernel_func() |
Gwendal Grignou | 7a61d2f | 2014-05-23 11:05:51 -0700 | [diff] [blame] | 763 | self.record('END GOOD', None, op, |
Dale Curtis | 74a314b | 2011-06-23 14:55:46 -0700 | [diff] [blame] | 764 | optional_fields={"kernel": kernel}) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 765 | |
| 766 | |
jadmanski | e432dd2 | 2009-01-30 15:04:51 +0000 | [diff] [blame] | 767 | def run_control(self, path): |
| 768 | """Execute a control file found at path (relative to the autotest |
| 769 | path). Intended for executing a control file within a control file, |
| 770 | not for running the top-level job control file.""" |
| 771 | path = os.path.join(self.autodir, path) |
| 772 | control_file = self._load_control_file(path) |
mbligh | 0d0f67d | 2009-11-06 03:15:03 +0000 | [diff] [blame] | 773 | self.run(control=control_file, control_file_dir=self._USE_TEMP_DIR) |
jadmanski | e432dd2 | 2009-01-30 15:04:51 +0000 | [diff] [blame] | 774 | |
| 775 | |
jadmanski | c09fc15 | 2008-10-15 17:56:59 +0000 | [diff] [blame] | 776 | def add_sysinfo_command(self, command, logfile=None, on_every_test=False): |
mbligh | 4395bbd | 2009-03-25 19:34:17 +0000 | [diff] [blame] | 777 | self._add_sysinfo_loggable(sysinfo.command(command, logf=logfile), |
jadmanski | c09fc15 | 2008-10-15 17:56:59 +0000 | [diff] [blame] | 778 | on_every_test) |
| 779 | |
| 780 | |
| 781 | def add_sysinfo_logfile(self, file, on_every_test=False): |
| 782 | self._add_sysinfo_loggable(sysinfo.logfile(file), on_every_test) |
| 783 | |
| 784 | |
| 785 | def _add_sysinfo_loggable(self, loggable, on_every_test): |
| 786 | if on_every_test: |
| 787 | self.sysinfo.test_loggables.add(loggable) |
| 788 | else: |
| 789 | self.sysinfo.boot_loggables.add(loggable) |
| 790 | |
| 791 | |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 792 | def _read_warnings(self): |
jadmanski | f37df84 | 2009-02-11 00:03:26 +0000 | [diff] [blame] | 793 | """Poll all the warning loggers and extract any new warnings that have |
| 794 | been logged. If the warnings belong to a category that is currently |
| 795 | disabled, this method will discard them and they will no longer be |
| 796 | retrievable. |
| 797 | |
| 798 | Returns a list of (timestamp, message) tuples, where timestamp is an |
| 799 | integer epoch timestamp.""" |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 800 | warnings = [] |
| 801 | while True: |
| 802 | # pull in a line of output from every logger that has |
| 803 | # output ready to be read |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 804 | loggers, _, _ = select.select(self.warning_loggers, [], [], 0) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 805 | closed_loggers = set() |
| 806 | for logger in loggers: |
| 807 | line = logger.readline() |
| 808 | # record any broken pipes (aka line == empty) |
| 809 | if len(line) == 0: |
| 810 | closed_loggers.add(logger) |
| 811 | continue |
jadmanski | f37df84 | 2009-02-11 00:03:26 +0000 | [diff] [blame] | 812 | # parse out the warning |
| 813 | timestamp, msgtype, msg = line.split('\t', 2) |
| 814 | timestamp = int(timestamp) |
| 815 | # if the warning is valid, add it to the results |
| 816 | if self.warning_manager.is_valid(timestamp, msgtype): |
| 817 | warnings.append((timestamp, msg.strip())) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 818 | |
| 819 | # stop listening to loggers that are closed |
| 820 | self.warning_loggers -= closed_loggers |
| 821 | |
| 822 | # stop if none of the loggers have any output left |
| 823 | if not loggers: |
| 824 | break |
| 825 | |
| 826 | # sort into timestamp order |
| 827 | warnings.sort() |
| 828 | return warnings |
| 829 | |
| 830 | |
showard | cc92936 | 2010-01-25 21:20:41 +0000 | [diff] [blame] | 831 | def _unique_subdirectory(self, base_subdirectory_name): |
| 832 | """Compute a unique results subdirectory based on the given name. |
| 833 | |
| 834 | Appends base_subdirectory_name with a number as necessary to find a |
| 835 | directory name that doesn't already exist. |
| 836 | """ |
| 837 | subdirectory = base_subdirectory_name |
| 838 | counter = 1 |
| 839 | while os.path.exists(os.path.join(self.resultdir, subdirectory)): |
| 840 | subdirectory = base_subdirectory_name + '.' + str(counter) |
| 841 | counter += 1 |
| 842 | return subdirectory |
| 843 | |
| 844 | |
jadmanski | 5205363 | 2010-06-11 21:08:10 +0000 | [diff] [blame] | 845 | def get_record_context(self): |
| 846 | """Returns an object representing the current job.record context. |
| 847 | |
| 848 | The object returned is an opaque object with a 0-arg restore method |
| 849 | which can be called to restore the job.record context (i.e. indentation) |
| 850 | to the current level. The intention is that it should be used when |
| 851 | something external which generate job.record calls (e.g. an autotest |
| 852 | client) can fail catastrophically and the server job record state |
| 853 | needs to be reset to its original "known good" state. |
| 854 | |
| 855 | @return: A context object with a 0-arg restore() method.""" |
| 856 | return self._indenter.get_context() |
| 857 | |
| 858 | |
showard | cc92936 | 2010-01-25 21:20:41 +0000 | [diff] [blame] | 859 | def record_summary(self, status_code, test_name, reason='', attributes=None, |
| 860 | distinguishing_attributes=(), child_test_ids=None): |
| 861 | """Record a summary test result. |
| 862 | |
| 863 | @param status_code: status code string, see |
| 864 | common_lib.log.is_valid_status() |
| 865 | @param test_name: name of the test |
| 866 | @param reason: (optional) string providing detailed reason for test |
| 867 | outcome |
| 868 | @param attributes: (optional) dict of string keyvals to associate with |
| 869 | this result |
| 870 | @param distinguishing_attributes: (optional) list of attribute names |
| 871 | that should be used to distinguish identically-named test |
| 872 | results. These attributes should be present in the attributes |
| 873 | parameter. This is used to generate user-friendly subdirectory |
| 874 | names. |
| 875 | @param child_test_ids: (optional) list of test indices for test results |
| 876 | used in generating this result. |
| 877 | """ |
| 878 | subdirectory_name_parts = [test_name] |
| 879 | for attribute in distinguishing_attributes: |
| 880 | assert attributes |
| 881 | assert attribute in attributes, '%s not in %s' % (attribute, |
| 882 | attributes) |
| 883 | subdirectory_name_parts.append(attributes[attribute]) |
| 884 | base_subdirectory_name = '.'.join(subdirectory_name_parts) |
| 885 | |
| 886 | subdirectory = self._unique_subdirectory(base_subdirectory_name) |
| 887 | subdirectory_path = os.path.join(self.resultdir, subdirectory) |
| 888 | os.mkdir(subdirectory_path) |
| 889 | |
| 890 | self.record(status_code, subdirectory, test_name, |
| 891 | status=reason, optional_fields={'is_summary': True}) |
| 892 | |
| 893 | if attributes: |
| 894 | utils.write_keyval(subdirectory_path, attributes) |
| 895 | |
| 896 | if child_test_ids: |
| 897 | ids_string = ','.join(str(test_id) for test_id in child_test_ids) |
| 898 | summary_data = {'child_test_ids': ids_string} |
| 899 | utils.write_keyval(os.path.join(subdirectory_path, 'summary_data'), |
| 900 | summary_data) |
| 901 | |
| 902 | |
jadmanski | 16a7ff7 | 2009-04-01 18:19:53 +0000 | [diff] [blame] | 903 | def disable_warnings(self, warning_type): |
jadmanski | f37df84 | 2009-02-11 00:03:26 +0000 | [diff] [blame] | 904 | self.warning_manager.disable_warnings(warning_type) |
jadmanski | 16a7ff7 | 2009-04-01 18:19:53 +0000 | [diff] [blame] | 905 | self.record("INFO", None, None, |
| 906 | "disabling %s warnings" % warning_type, |
| 907 | {"warnings.disable": warning_type}) |
jadmanski | f37df84 | 2009-02-11 00:03:26 +0000 | [diff] [blame] | 908 | |
| 909 | |
jadmanski | 16a7ff7 | 2009-04-01 18:19:53 +0000 | [diff] [blame] | 910 | def enable_warnings(self, warning_type): |
jadmanski | f37df84 | 2009-02-11 00:03:26 +0000 | [diff] [blame] | 911 | self.warning_manager.enable_warnings(warning_type) |
jadmanski | 16a7ff7 | 2009-04-01 18:19:53 +0000 | [diff] [blame] | 912 | self.record("INFO", None, None, |
| 913 | "enabling %s warnings" % warning_type, |
| 914 | {"warnings.enable": warning_type}) |
jadmanski | f37df84 | 2009-02-11 00:03:26 +0000 | [diff] [blame] | 915 | |
| 916 | |
jadmanski | 779bd29 | 2009-03-19 17:33:33 +0000 | [diff] [blame] | 917 | def get_status_log_path(self, subdir=None): |
| 918 | """Return the path to the job status log. |
| 919 | |
| 920 | @param subdir - Optional paramter indicating that you want the path |
| 921 | to a subdirectory status log. |
| 922 | |
| 923 | @returns The path where the status log should be. |
| 924 | """ |
mbligh | 210bae6 | 2009-04-01 18:33:13 +0000 | [diff] [blame] | 925 | if self.resultdir: |
| 926 | if subdir: |
| 927 | return os.path.join(self.resultdir, subdir, "status.log") |
| 928 | else: |
| 929 | return os.path.join(self.resultdir, "status.log") |
jadmanski | 779bd29 | 2009-03-19 17:33:33 +0000 | [diff] [blame] | 930 | else: |
mbligh | 210bae6 | 2009-04-01 18:33:13 +0000 | [diff] [blame] | 931 | return None |
jadmanski | 779bd29 | 2009-03-19 17:33:33 +0000 | [diff] [blame] | 932 | |
| 933 | |
jadmanski | 6bb32d7 | 2009-03-19 20:25:24 +0000 | [diff] [blame] | 934 | def _update_uncollected_logs_list(self, update_func): |
| 935 | """Updates the uncollected logs list in a multi-process safe manner. |
| 936 | |
| 937 | @param update_func - a function that updates the list of uncollected |
| 938 | logs. Should take one parameter, the list to be updated. |
| 939 | """ |
Dan Shi | 07e09af | 2013-04-12 09:31:29 -0700 | [diff] [blame] | 940 | # Skip log collection if file _uncollected_log_file does not exist. |
| 941 | if not (self._uncollected_log_file and |
| 942 | os.path.exists(self._uncollected_log_file)): |
| 943 | return |
mbligh | 0d0f67d | 2009-11-06 03:15:03 +0000 | [diff] [blame] | 944 | if self._uncollected_log_file: |
| 945 | log_file = open(self._uncollected_log_file, "r+") |
mbligh | a788dc4 | 2009-03-26 21:10:16 +0000 | [diff] [blame] | 946 | fcntl.flock(log_file, fcntl.LOCK_EX) |
jadmanski | 6bb32d7 | 2009-03-19 20:25:24 +0000 | [diff] [blame] | 947 | try: |
| 948 | uncollected_logs = pickle.load(log_file) |
| 949 | update_func(uncollected_logs) |
| 950 | log_file.seek(0) |
| 951 | log_file.truncate() |
| 952 | pickle.dump(uncollected_logs, log_file) |
jadmanski | 3bff909 | 2009-04-22 18:09:47 +0000 | [diff] [blame] | 953 | log_file.flush() |
jadmanski | 6bb32d7 | 2009-03-19 20:25:24 +0000 | [diff] [blame] | 954 | finally: |
| 955 | fcntl.flock(log_file, fcntl.LOCK_UN) |
| 956 | log_file.close() |
| 957 | |
| 958 | |
| 959 | def add_client_log(self, hostname, remote_path, local_path): |
| 960 | """Adds a new set of client logs to the list of uncollected logs, |
| 961 | to allow for future log recovery. |
| 962 | |
| 963 | @param host - the hostname of the machine holding the logs |
| 964 | @param remote_path - the directory on the remote machine holding logs |
| 965 | @param local_path - the local directory to copy the logs into |
| 966 | """ |
| 967 | def update_func(logs_list): |
| 968 | logs_list.append((hostname, remote_path, local_path)) |
| 969 | self._update_uncollected_logs_list(update_func) |
| 970 | |
| 971 | |
| 972 | def remove_client_log(self, hostname, remote_path, local_path): |
| 973 | """Removes a set of client logs from the list of uncollected logs, |
| 974 | to allow for future log recovery. |
| 975 | |
| 976 | @param host - the hostname of the machine holding the logs |
| 977 | @param remote_path - the directory on the remote machine holding logs |
| 978 | @param local_path - the local directory to copy the logs into |
| 979 | """ |
| 980 | def update_func(logs_list): |
| 981 | logs_list.remove((hostname, remote_path, local_path)) |
| 982 | self._update_uncollected_logs_list(update_func) |
| 983 | |
| 984 | |
mbligh | 0d0f67d | 2009-11-06 03:15:03 +0000 | [diff] [blame] | 985 | def get_client_logs(self): |
| 986 | """Retrieves the list of uncollected logs, if it exists. |
| 987 | |
| 988 | @returns A list of (host, remote_path, local_path) tuples. Returns |
| 989 | an empty list if no uncollected logs file exists. |
| 990 | """ |
| 991 | log_exists = (self._uncollected_log_file and |
| 992 | os.path.exists(self._uncollected_log_file)) |
| 993 | if log_exists: |
| 994 | return pickle.load(open(self._uncollected_log_file)) |
| 995 | else: |
| 996 | return [] |
| 997 | |
| 998 | |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 999 | def _fill_server_control_namespace(self, namespace, protect=True): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 1000 | """ |
| 1001 | Prepare a namespace to be used when executing server control files. |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 1002 | |
| 1003 | This sets up the control file API by importing modules and making them |
| 1004 | available under the appropriate names within namespace. |
| 1005 | |
| 1006 | For use by _execute_code(). |
| 1007 | |
| 1008 | Args: |
| 1009 | namespace: The namespace dictionary to fill in. |
| 1010 | protect: Boolean. If True (the default) any operation that would |
| 1011 | clobber an existing entry in namespace will cause an error. |
| 1012 | Raises: |
| 1013 | error.AutoservError: When a name would be clobbered by import. |
| 1014 | """ |
| 1015 | def _import_names(module_name, names=()): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 1016 | """ |
| 1017 | Import a module and assign named attributes into namespace. |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 1018 | |
| 1019 | Args: |
| 1020 | module_name: The string module name. |
| 1021 | names: A limiting list of names to import from module_name. If |
| 1022 | empty (the default), all names are imported from the module |
| 1023 | similar to a "from foo.bar import *" statement. |
| 1024 | Raises: |
| 1025 | error.AutoservError: When a name being imported would clobber |
| 1026 | a name already in namespace. |
| 1027 | """ |
| 1028 | module = __import__(module_name, {}, {}, names) |
| 1029 | |
| 1030 | # No names supplied? Import * from the lowest level module. |
| 1031 | # (Ugh, why do I have to implement this part myself?) |
| 1032 | if not names: |
| 1033 | for submodule_name in module_name.split('.')[1:]: |
| 1034 | module = getattr(module, submodule_name) |
| 1035 | if hasattr(module, '__all__'): |
| 1036 | names = getattr(module, '__all__') |
| 1037 | else: |
| 1038 | names = dir(module) |
| 1039 | |
| 1040 | # Install each name into namespace, checking to make sure it |
| 1041 | # doesn't override anything that already exists. |
| 1042 | for name in names: |
| 1043 | # Check for conflicts to help prevent future problems. |
| 1044 | if name in namespace and protect: |
| 1045 | if namespace[name] is not getattr(module, name): |
| 1046 | raise error.AutoservError('importing name ' |
| 1047 | '%s from %s %r would override %r' % |
| 1048 | (name, module_name, getattr(module, name), |
| 1049 | namespace[name])) |
| 1050 | else: |
| 1051 | # Encourage cleanliness and the use of __all__ for a |
| 1052 | # more concrete API with less surprises on '*' imports. |
| 1053 | warnings.warn('%s (%r) being imported from %s for use ' |
| 1054 | 'in server control files is not the ' |
| 1055 | 'first occurrance of that import.' % |
| 1056 | (name, namespace[name], module_name)) |
| 1057 | |
| 1058 | namespace[name] = getattr(module, name) |
| 1059 | |
| 1060 | |
| 1061 | # This is the equivalent of prepending a bunch of import statements to |
| 1062 | # the front of the control script. |
mbligh | a2b07dd | 2009-06-22 18:26:13 +0000 | [diff] [blame] | 1063 | namespace.update(os=os, sys=sys, logging=logging) |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 1064 | _import_names('autotest_lib.server', |
| 1065 | ('hosts', 'autotest', 'kvm', 'git', 'standalone_profiler', |
| 1066 | 'source_kernel', 'rpm_kernel', 'deb_kernel', 'git_kernel')) |
| 1067 | _import_names('autotest_lib.server.subcommand', |
| 1068 | ('parallel', 'parallel_simple', 'subcommand')) |
| 1069 | _import_names('autotest_lib.server.utils', |
| 1070 | ('run', 'get_tmp_dir', 'sh_escape', 'parse_machine')) |
| 1071 | _import_names('autotest_lib.client.common_lib.error') |
| 1072 | _import_names('autotest_lib.client.common_lib.barrier', ('barrier',)) |
| 1073 | |
| 1074 | # Inject ourself as the job object into other classes within the API. |
| 1075 | # (Yuck, this injection is a gross thing be part of a public API. -gps) |
| 1076 | # |
| 1077 | # XXX Base & SiteAutotest do not appear to use .job. Who does? |
| 1078 | namespace['autotest'].Autotest.job = self |
| 1079 | # server.hosts.base_classes.Host uses .job. |
| 1080 | namespace['hosts'].Host.job = self |
Eric Li | 10222b8 | 2010-11-24 09:33:15 -0800 | [diff] [blame] | 1081 | namespace['hosts'].factory.ssh_user = self._ssh_user |
| 1082 | namespace['hosts'].factory.ssh_port = self._ssh_port |
| 1083 | namespace['hosts'].factory.ssh_pass = self._ssh_pass |
Fang Deng | d1c2b73 | 2013-08-20 12:59:46 -0700 | [diff] [blame] | 1084 | namespace['hosts'].factory.ssh_verbosity_flag = ( |
| 1085 | self._ssh_verbosity_flag) |
Aviv Keshet | c5947fa | 2013-09-04 14:06:29 -0700 | [diff] [blame] | 1086 | namespace['hosts'].factory.ssh_options = self._ssh_options |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 1087 | |
| 1088 | |
| 1089 | def _execute_code(self, code_file, namespace, protect=True): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 1090 | """ |
| 1091 | Execute code using a copy of namespace as a server control script. |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 1092 | |
| 1093 | Unless protect_namespace is explicitly set to False, the dict will not |
| 1094 | be modified. |
| 1095 | |
| 1096 | Args: |
| 1097 | code_file: The filename of the control file to execute. |
| 1098 | namespace: A dict containing names to make available during execution. |
| 1099 | protect: Boolean. If True (the default) a copy of the namespace dict |
| 1100 | is used during execution to prevent the code from modifying its |
| 1101 | contents outside of this function. If False the raw dict is |
| 1102 | passed in and modifications will be allowed. |
| 1103 | """ |
| 1104 | if protect: |
| 1105 | namespace = namespace.copy() |
| 1106 | self._fill_server_control_namespace(namespace, protect=protect) |
| 1107 | # TODO: Simplify and get rid of the special cases for only 1 machine. |
showard | 3e66e8c | 2008-10-27 19:20:51 +0000 | [diff] [blame] | 1108 | if len(self.machines) > 1: |
mbligh | 084bc17 | 2008-10-18 14:02:45 +0000 | [diff] [blame] | 1109 | machines_text = '\n'.join(self.machines) + '\n' |
| 1110 | # Only rewrite the file if it does not match our machine list. |
| 1111 | try: |
| 1112 | machines_f = open(MACHINES_FILENAME, 'r') |
| 1113 | existing_machines_text = machines_f.read() |
| 1114 | machines_f.close() |
| 1115 | except EnvironmentError: |
| 1116 | existing_machines_text = None |
| 1117 | if machines_text != existing_machines_text: |
| 1118 | utils.open_write_close(MACHINES_FILENAME, machines_text) |
| 1119 | execfile(code_file, namespace, namespace) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 1120 | |
| 1121 | |
jadmanski | e29d0e4 | 2010-06-17 16:06:52 +0000 | [diff] [blame] | 1122 | def _parse_status(self, new_line): |
mbligh | 0d0f67d | 2009-11-06 03:15:03 +0000 | [diff] [blame] | 1123 | if not self._using_parser: |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 1124 | return |
jadmanski | e29d0e4 | 2010-06-17 16:06:52 +0000 | [diff] [blame] | 1125 | new_tests = self.parser.process_lines([new_line]) |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 1126 | for test in new_tests: |
| 1127 | self.__insert_test(test) |
| 1128 | |
| 1129 | |
| 1130 | def __insert_test(self, test): |
mbligh | 2b92b86 | 2008-11-22 13:25:32 +0000 | [diff] [blame] | 1131 | """ |
| 1132 | An internal method to insert a new test result into the |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 1133 | database. This method will not raise an exception, even if an |
| 1134 | error occurs during the insert, to avoid failing a test |
| 1135 | simply because of unexpected database issues.""" |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 1136 | self.num_tests_run += 1 |
| 1137 | if status_lib.is_worse_than_or_equal_to(test.status, 'FAIL'): |
| 1138 | self.num_tests_failed += 1 |
jadmanski | 1064644 | 2008-08-13 14:05:21 +0000 | [diff] [blame] | 1139 | try: |
| 1140 | self.results_db.insert_test(self.job_model, test) |
| 1141 | except Exception: |
| 1142 | msg = ("WARNING: An unexpected error occured while " |
| 1143 | "inserting test results into the database. " |
| 1144 | "Ignoring error.\n" + traceback.format_exc()) |
| 1145 | print >> sys.stderr, msg |
| 1146 | |
mbligh | caa62c2 | 2008-04-07 21:51:17 +0000 | [diff] [blame] | 1147 | |
mbligh | fc3da5b | 2010-01-06 18:37:22 +0000 | [diff] [blame] | 1148 | def preprocess_client_state(self): |
| 1149 | """ |
| 1150 | Produce a state file for initializing the state of a client job. |
| 1151 | |
| 1152 | Creates a new client state file with all the current server state, as |
| 1153 | well as some pre-set client state. |
| 1154 | |
| 1155 | @returns The path of the file the state was written into. |
| 1156 | """ |
| 1157 | # initialize the sysinfo state |
| 1158 | self._state.set('client', 'sysinfo', self.sysinfo.serialize()) |
| 1159 | |
| 1160 | # dump the state out to a tempfile |
| 1161 | fd, file_path = tempfile.mkstemp(dir=self.tmpdir) |
| 1162 | os.close(fd) |
mbligh | a2c9949 | 2010-01-27 22:59:50 +0000 | [diff] [blame] | 1163 | |
| 1164 | # write_to_file doesn't need locking, we exclusively own file_path |
mbligh | fc3da5b | 2010-01-06 18:37:22 +0000 | [diff] [blame] | 1165 | self._state.write_to_file(file_path) |
| 1166 | return file_path |
| 1167 | |
| 1168 | |
| 1169 | def postprocess_client_state(self, state_path): |
| 1170 | """ |
| 1171 | Update the state of this job with the state from a client job. |
| 1172 | |
| 1173 | Updates the state of the server side of a job with the final state |
| 1174 | of a client job that was run. Updates the non-client-specific state, |
| 1175 | pulls in some specific bits from the client-specific state, and then |
| 1176 | discards the rest. Removes the state file afterwards |
| 1177 | |
| 1178 | @param state_file A path to the state file from the client. |
| 1179 | """ |
| 1180 | # update the on-disk state |
mbligh | fc3da5b | 2010-01-06 18:37:22 +0000 | [diff] [blame] | 1181 | try: |
jadmanski | b6e7bdb | 2010-04-13 16:00:39 +0000 | [diff] [blame] | 1182 | self._state.read_from_file(state_path) |
mbligh | fc3da5b | 2010-01-06 18:37:22 +0000 | [diff] [blame] | 1183 | os.remove(state_path) |
mbligh | a2c9949 | 2010-01-27 22:59:50 +0000 | [diff] [blame] | 1184 | except OSError, e: |
mbligh | fc3da5b | 2010-01-06 18:37:22 +0000 | [diff] [blame] | 1185 | # ignore file-not-found errors |
| 1186 | if e.errno != errno.ENOENT: |
| 1187 | raise |
jadmanski | b6e7bdb | 2010-04-13 16:00:39 +0000 | [diff] [blame] | 1188 | else: |
| 1189 | logging.debug('Client state file %s not found', state_path) |
mbligh | fc3da5b | 2010-01-06 18:37:22 +0000 | [diff] [blame] | 1190 | |
| 1191 | # update the sysinfo state |
| 1192 | if self._state.has('client', 'sysinfo'): |
| 1193 | self.sysinfo.deserialize(self._state.get('client', 'sysinfo')) |
| 1194 | |
| 1195 | # drop all the client-specific state |
| 1196 | self._state.discard_namespace('client') |
| 1197 | |
| 1198 | |
mbligh | 0a88370 | 2010-04-21 01:58:34 +0000 | [diff] [blame] | 1199 | def clear_all_known_hosts(self): |
| 1200 | """Clears known hosts files for all AbstractSSHHosts.""" |
| 1201 | for host in self.hosts: |
| 1202 | if isinstance(host, abstract_ssh.AbstractSSHHost): |
| 1203 | host.clear_known_hosts() |
| 1204 | |
| 1205 | |
jadmanski | f37df84 | 2009-02-11 00:03:26 +0000 | [diff] [blame] | 1206 | class warning_manager(object): |
| 1207 | """Class for controlling warning logs. Manages the enabling and disabling |
| 1208 | of warnings.""" |
| 1209 | def __init__(self): |
| 1210 | # a map of warning types to a list of disabled time intervals |
| 1211 | self.disabled_warnings = {} |
| 1212 | |
| 1213 | |
| 1214 | def is_valid(self, timestamp, warning_type): |
| 1215 | """Indicates if a warning (based on the time it occured and its type) |
| 1216 | is a valid warning. A warning is considered "invalid" if this type of |
| 1217 | warning was marked as "disabled" at the time the warning occured.""" |
| 1218 | disabled_intervals = self.disabled_warnings.get(warning_type, []) |
| 1219 | for start, end in disabled_intervals: |
| 1220 | if timestamp >= start and (end is None or timestamp < end): |
| 1221 | return False |
| 1222 | return True |
| 1223 | |
| 1224 | |
| 1225 | def disable_warnings(self, warning_type, current_time_func=time.time): |
| 1226 | """As of now, disables all further warnings of this type.""" |
| 1227 | intervals = self.disabled_warnings.setdefault(warning_type, []) |
| 1228 | if not intervals or intervals[-1][1] is not None: |
jadmanski | 16a7ff7 | 2009-04-01 18:19:53 +0000 | [diff] [blame] | 1229 | intervals.append((int(current_time_func()), None)) |
jadmanski | f37df84 | 2009-02-11 00:03:26 +0000 | [diff] [blame] | 1230 | |
| 1231 | |
| 1232 | def enable_warnings(self, warning_type, current_time_func=time.time): |
| 1233 | """As of now, enables all further warnings of this type.""" |
| 1234 | intervals = self.disabled_warnings.get(warning_type, []) |
| 1235 | if intervals and intervals[-1][1] is None: |
jadmanski | 16a7ff7 | 2009-04-01 18:19:53 +0000 | [diff] [blame] | 1236 | intervals[-1] = (intervals[-1][0], int(current_time_func())) |
Paul Pendlebury | 5759356 | 2011-06-15 10:45:49 -0700 | [diff] [blame] | 1237 | |
| 1238 | |
| 1239 | # load up site-specific code for generating site-specific job data |
| 1240 | get_site_job_data = utils.import_site_function(__file__, |
| 1241 | "autotest_lib.server.site_server_job", "get_site_job_data", |
| 1242 | _get_site_job_data_dummy) |
| 1243 | |
| 1244 | |
| 1245 | site_server_job = utils.import_site_class( |
| 1246 | __file__, "autotest_lib.server.site_server_job", "site_server_job", |
| 1247 | base_server_job) |
| 1248 | |
| 1249 | |
| 1250 | class server_job(site_server_job): |
Dale Curtis | 456d3c1 | 2011-07-19 11:42:51 -0700 | [diff] [blame] | 1251 | pass |