Aviv Keshet | 021c19f | 2013-02-22 13:19:43 -0800 | [diff] [blame] | 1 | #!/usr/bin/python |
| 2 | # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. |
| 3 | # Use of this source code is governed by a BSD-style license that can be |
| 4 | # found in the LICENSE file. |
| 5 | |
| 6 | import argparse |
Aviv Keshet | 15782de | 2013-07-31 11:40:41 -0700 | [diff] [blame] | 7 | import errno |
Aviv Keshet | 0e5d525 | 2013-04-26 16:01:36 -0700 | [diff] [blame] | 8 | import os |
Aviv Keshet | 7cd1231 | 2013-07-25 10:25:55 -0700 | [diff] [blame] | 9 | import pipes |
Aviv Keshet | d4a0430 | 2013-04-30 15:48:30 -0700 | [diff] [blame] | 10 | import re |
Aviv Keshet | 1de5bc6 | 2013-08-19 14:19:26 -0700 | [diff] [blame] | 11 | import shutil |
Aviv Keshet | 1d991ea | 2013-06-12 17:24:23 -0700 | [diff] [blame] | 12 | import signal |
Aviv Keshet | 1de5bc6 | 2013-08-19 14:19:26 -0700 | [diff] [blame] | 13 | import stat |
Aviv Keshet | 0e5d525 | 2013-04-26 16:01:36 -0700 | [diff] [blame] | 14 | import subprocess |
Aviv Keshet | 021c19f | 2013-02-22 13:19:43 -0800 | [diff] [blame] | 15 | import sys |
Aviv Keshet | d4a0430 | 2013-04-30 15:48:30 -0700 | [diff] [blame] | 16 | import tempfile |
Aviv Keshet | 1d991ea | 2013-06-12 17:24:23 -0700 | [diff] [blame] | 17 | import threading |
Aviv Keshet | 021c19f | 2013-02-22 13:19:43 -0800 | [diff] [blame] | 18 | |
Aviv Keshet | e43bccf | 2013-08-14 14:11:59 -0700 | [diff] [blame] | 19 | import logging |
| 20 | # Turn the logging level to INFO before importing other autotest |
| 21 | # code, to avoid having failed import logging messages confuse the |
| 22 | # test_that user. |
| 23 | logging.basicConfig(level=logging.INFO) |
| 24 | |
Aviv Keshet | 021c19f | 2013-02-22 13:19:43 -0800 | [diff] [blame] | 25 | import common |
Aviv Keshet | 1d991ea | 2013-06-12 17:24:23 -0700 | [diff] [blame] | 26 | from autotest_lib.client.common_lib.cros import dev_server, retry |
beeps | 4efdf03 | 2013-09-17 11:27:14 -0700 | [diff] [blame] | 27 | from autotest_lib.client.common_lib import error, logging_manager |
| 28 | from autotest_lib.server.cros.dynamic_suite import suite, constants |
Aviv Keshet | f2b6910 | 2013-08-28 10:34:49 -0700 | [diff] [blame] | 29 | from autotest_lib.server.cros import provision |
beeps | 4efdf03 | 2013-09-17 11:27:14 -0700 | [diff] [blame] | 30 | from autotest_lib.server.hosts import factory |
Aviv Keshet | d4a0430 | 2013-04-30 15:48:30 -0700 | [diff] [blame] | 31 | from autotest_lib.server import autoserv_utils |
Aviv Keshet | e43bccf | 2013-08-14 14:11:59 -0700 | [diff] [blame] | 32 | from autotest_lib.server import server_logging_config |
Fang Deng | 2db9676 | 2013-10-03 16:45:31 -0700 | [diff] [blame] | 33 | from autotest_lib.server import utils |
Aviv Keshet | e43bccf | 2013-08-14 14:11:59 -0700 | [diff] [blame] | 34 | |
Aviv Keshet | d4a0430 | 2013-04-30 15:48:30 -0700 | [diff] [blame] | 35 | |
Aviv Keshet | 0e5d525 | 2013-04-26 16:01:36 -0700 | [diff] [blame] | 36 | try: |
| 37 | from chromite.lib import cros_build_lib |
| 38 | except ImportError: |
| 39 | print 'Unable to import chromite.' |
| 40 | print 'This script must be either:' |
| 41 | print ' - Be run in the chroot.' |
| 42 | print ' - (not yet supported) be run after running ' |
| 43 | print ' ../utils/build_externals.py' |
Aviv Keshet | 021c19f | 2013-02-22 13:19:43 -0800 | [diff] [blame] | 44 | |
Aviv Keshet | 1d991ea | 2013-06-12 17:24:23 -0700 | [diff] [blame] | 45 | _autoserv_proc = None |
| 46 | _sigint_handler_lock = threading.Lock() |
| 47 | |
| 48 | _AUTOSERV_SIGINT_TIMEOUT_SECONDS = 5 |
Alex Miller | a091307 | 2013-06-12 10:01:51 -0700 | [diff] [blame] | 49 | _NO_BOARD = 'ad_hoc_board' |
Aviv Keshet | 1071196 | 2013-06-24 12:20:33 -0700 | [diff] [blame] | 50 | _NO_BUILD = 'ad_hoc_build' |
Fang Deng | b1da830 | 2013-09-24 13:57:54 -0700 | [diff] [blame] | 51 | _SUITE_REGEX = r'suite:(.*)' |
Aviv Keshet | 1d991ea | 2013-06-12 17:24:23 -0700 | [diff] [blame] | 52 | |
Aviv Keshet | 2dc9893 | 2013-06-17 15:22:10 -0700 | [diff] [blame] | 53 | _QUICKMERGE_SCRIPTNAME = '/mnt/host/source/chromite/bin/autotest_quickmerge' |
Aviv Keshet | 1de5bc6 | 2013-08-19 14:19:26 -0700 | [diff] [blame] | 54 | _TEST_KEY_FILENAME = 'testing_rsa' |
| 55 | _TEST_KEY_PATH = ('/mnt/host/source/src/scripts/mod_for_test_scripts/' |
| 56 | 'ssh_keys/%s' % _TEST_KEY_FILENAME) |
Aviv Keshet | 2dc9893 | 2013-06-17 15:22:10 -0700 | [diff] [blame] | 57 | |
Aviv Keshet | ba4992a | 2013-07-02 14:09:23 -0700 | [diff] [blame] | 58 | _TEST_REPORT_SCRIPTNAME = '/usr/bin/generate_test_report' |
| 59 | |
Aviv Keshet | c9e7462 | 2013-07-18 10:11:11 -0700 | [diff] [blame] | 60 | _LATEST_RESULTS_DIRECTORY = '/tmp/test_that_latest' |
| 61 | |
Aviv Keshet | d4a0430 | 2013-04-30 15:48:30 -0700 | [diff] [blame] | 62 | |
beeps | 4efdf03 | 2013-09-17 11:27:14 -0700 | [diff] [blame] | 63 | class TestThatRunError(Exception): |
| 64 | """Raised if test_that encounters something unexpected while running.""" |
| 65 | |
| 66 | |
Fang Deng | 9a84123 | 2013-09-17 16:29:14 -0700 | [diff] [blame] | 67 | class TestThatProvisioningError(Exception): |
| 68 | """Raised when it fails to provision the DUT to the requested build.""" |
| 69 | |
| 70 | |
Aviv Keshet | 20b20c7 | 2014-04-02 11:54:28 -0700 | [diff] [blame] | 71 | def fetch_local_suite(autotest_path, suite_predicate, afe, remote, |
Fang Deng | b1da830 | 2013-09-24 13:57:54 -0700 | [diff] [blame] | 72 | build=_NO_BUILD, board=_NO_BOARD, |
| 73 | results_directory=None, no_experimental=False, |
| 74 | ignore_deps=True): |
Aviv Keshet | 20b20c7 | 2014-04-02 11:54:28 -0700 | [diff] [blame] | 75 | """Create a suite from the given suite predicate. |
Fang Deng | b1da830 | 2013-09-24 13:57:54 -0700 | [diff] [blame] | 76 | |
| 77 | Satisfaction of dependencies is enforced by Suite.schedule() if |
| 78 | ignore_deps is False. Note that this method assumes only one host, |
| 79 | i.e. |remote|, was added to afe. Suite.schedule() will not |
| 80 | schedule a job if none of the hosts in the afe (in our case, |
| 81 | just one host |remote|) has a label that matches a requested |
| 82 | test dependency. |
| 83 | |
Fang Deng | b3ee20b | 2013-09-17 10:34:13 -0700 | [diff] [blame] | 84 | @param autotest_path: Absolute path to autotest (in sysroot or |
| 85 | custom autotest directory set by --autotest_dir). |
Aviv Keshet | a6adc7a | 2013-08-30 11:13:38 -0700 | [diff] [blame] | 86 | @param suite_predicate: callable that takes ControlData objects, and |
| 87 | returns True on those that should be in suite |
Aviv Keshet | d4a0430 | 2013-04-30 15:48:30 -0700 | [diff] [blame] | 88 | @param afe: afe object to schedule against (typically a directAFE) |
Fang Deng | b1da830 | 2013-09-24 13:57:54 -0700 | [diff] [blame] | 89 | @param remote: String representing the IP of the remote host. |
Aviv Keshet | d4a0430 | 2013-04-30 15:48:30 -0700 | [diff] [blame] | 90 | @param build: Build to schedule suite for. |
Aviv Keshet | 1071196 | 2013-06-24 12:20:33 -0700 | [diff] [blame] | 91 | @param board: Board to schedule suite for. |
Aviv Keshet | ba4992a | 2013-07-02 14:09:23 -0700 | [diff] [blame] | 92 | @param results_directory: Absolute path of directory to store results in. |
| 93 | (results will be stored in subdirectory of this). |
Aviv Keshet | e9170d9 | 2013-07-19 11:20:45 -0700 | [diff] [blame] | 94 | @param no_experimental: Skip experimental tests when scheduling a suite. |
Fang Deng | b1da830 | 2013-09-24 13:57:54 -0700 | [diff] [blame] | 95 | @param ignore_deps: If True, test dependencies will be ignored. |
| 96 | |
Aviv Keshet | 20b20c7 | 2014-04-02 11:54:28 -0700 | [diff] [blame] | 97 | @returns: A suite.Suite object. |
Fang Deng | b1da830 | 2013-09-24 13:57:54 -0700 | [diff] [blame] | 98 | |
Aviv Keshet | d4a0430 | 2013-04-30 15:48:30 -0700 | [diff] [blame] | 99 | """ |
| 100 | fs_getter = suite.Suite.create_fs_getter(autotest_path) |
| 101 | devserver = dev_server.ImageServer('') |
Aviv Keshet | a6adc7a | 2013-08-30 11:13:38 -0700 | [diff] [blame] | 102 | my_suite = suite.Suite.create_from_predicates([suite_predicate], |
Fang Deng | b1da830 | 2013-09-24 13:57:54 -0700 | [diff] [blame] | 103 | build, constants.BOARD_PREFIX + board, |
| 104 | devserver, fs_getter, afe=afe, |
| 105 | ignore_deps=ignore_deps, |
beeps | 89f1e06 | 2013-09-18 12:00:17 -0700 | [diff] [blame] | 106 | results_dir=results_directory, forgiving_parser=False) |
Aviv Keshet | d4a0430 | 2013-04-30 15:48:30 -0700 | [diff] [blame] | 107 | if len(my_suite.tests) == 0: |
Aviv Keshet | a6adc7a | 2013-08-30 11:13:38 -0700 | [diff] [blame] | 108 | raise ValueError('Suite contained no tests.') |
Fang Deng | b1da830 | 2013-09-24 13:57:54 -0700 | [diff] [blame] | 109 | |
| 110 | if not ignore_deps: |
| 111 | # Log tests whose dependencies can't be satisfied. |
| 112 | labels = [label.name for label in |
| 113 | afe.get_labels(host__hostname=remote)] |
| 114 | for test in my_suite.tests: |
| 115 | if test.experimental and no_experimental: |
| 116 | continue |
| 117 | unsatisfiable_deps = set(test.dependencies).difference(labels) |
| 118 | if unsatisfiable_deps: |
Ilja H. Friedel | 04be2bd | 2014-05-07 21:29:59 -0700 | [diff] [blame] | 119 | logging.warning('%s will be skipped, unsatisfiable ' |
Fang Deng | b1da830 | 2013-09-24 13:57:54 -0700 | [diff] [blame] | 120 | 'test dependencies: %s', test.name, |
| 121 | unsatisfiable_deps) |
Aviv Keshet | 20b20c7 | 2014-04-02 11:54:28 -0700 | [diff] [blame] | 122 | return my_suite |
Aviv Keshet | 021c19f | 2013-02-22 13:19:43 -0800 | [diff] [blame] | 123 | |
Fang Deng | 9a84123 | 2013-09-17 16:29:14 -0700 | [diff] [blame] | 124 | def _run_autoserv(command, pretend=False): |
| 125 | """Run autoserv command. |
| 126 | |
| 127 | Run the autoserv command and wait on it. Log the stdout. |
| 128 | Ensure that SIGINT signals are passed along to autoserv. |
| 129 | |
| 130 | @param command: the autoserv command to run. |
| 131 | @returns: exit code of the command. |
| 132 | |
| 133 | """ |
| 134 | if not pretend: |
| 135 | logging.debug('Running autoserv command: %s', command) |
| 136 | global _autoserv_proc |
| 137 | _autoserv_proc = subprocess.Popen(command, |
| 138 | stdout=subprocess.PIPE, |
| 139 | stderr=subprocess.STDOUT) |
| 140 | # This incantation forces unbuffered reading from stdout, |
| 141 | # so that autoserv output can be displayed to the user |
| 142 | # immediately. |
| 143 | for message in iter(_autoserv_proc.stdout.readline, b''): |
| 144 | logging.info('autoserv| %s', message.strip()) |
| 145 | |
| 146 | _autoserv_proc.wait() |
| 147 | returncode = _autoserv_proc.returncode |
| 148 | _autoserv_proc = None |
| 149 | else: |
| 150 | logging.info('Pretend mode. Would run autoserv command: %s', |
| 151 | command) |
| 152 | returncode = 0 |
| 153 | return returncode |
| 154 | |
| 155 | |
| 156 | def run_provisioning_job(provision_label, host, autotest_path, |
| 157 | results_directory, fast_mode, |
| 158 | ssh_verbosity=0, ssh_options=None, |
| 159 | pretend=False, autoserv_verbose=False): |
| 160 | """Shell out to autoserv to run provisioning job. |
| 161 | |
| 162 | @param provision_label: Label to provision the machine to. |
| 163 | @param host: Hostname of DUT. |
| 164 | @param autotest_path: Absolute path of autotest directory. |
| 165 | @param results_directory: Absolute path of directory to store results in. |
| 166 | (results will be stored in subdirectory of this). |
| 167 | @param fast_mode: bool to use fast mode (disables slow autotest features). |
| 168 | @param ssh_verbosity: SSH verbosity level, passed along to autoserv_utils |
| 169 | @param ssh_options: Additional ssh options to be passed to autoserv_utils |
| 170 | @param pretend: If True, will print out autoserv commands rather than |
| 171 | running them. |
| 172 | @param autoserv_verbose: If true, pass the --verbose flag to autoserv. |
| 173 | |
| 174 | @returns: Absolute path of directory where results were stored. |
| 175 | |
| 176 | """ |
| 177 | # TODO(fdeng): When running against a local DUT, autoserv |
| 178 | # is still hitting the AFE in the lab. |
| 179 | # provision_AutoUpdate checks the current build of DUT by |
| 180 | # retrieving build info from AFE. crosbug.com/295178 |
| 181 | results_directory = os.path.join(results_directory, 'results-provision') |
| 182 | provision_arg = '='.join(['--provision', provision_label]) |
| 183 | command = autoserv_utils.autoserv_run_job_command( |
| 184 | os.path.join(autotest_path, 'server'), |
| 185 | machines=host, job=None, verbose=autoserv_verbose, |
| 186 | results_directory=results_directory, |
| 187 | fast_mode=fast_mode, ssh_verbosity=ssh_verbosity, |
| 188 | ssh_options=ssh_options, extra_args=[provision_arg], |
| 189 | no_console_prefix=True) |
| 190 | if _run_autoserv(command, pretend) != 0: |
| 191 | raise TestThatProvisioningError('Command returns non-zero code: %s ' % |
| 192 | command) |
| 193 | return results_directory |
| 194 | |
| 195 | |
Fang Deng | b3ee20b | 2013-09-17 10:34:13 -0700 | [diff] [blame] | 196 | def run_job(job, host, autotest_path, results_directory, fast_mode, |
Aviv Keshet | 6a704f7 | 2013-09-04 14:57:43 -0700 | [diff] [blame] | 197 | id_digits=1, ssh_verbosity=0, ssh_options=None, |
| 198 | args=None, pretend=False, |
Aviv Keshet | 2ebbd35 | 2013-08-27 17:04:43 -0700 | [diff] [blame] | 199 | autoserv_verbose=False): |
Aviv Keshet | d4a0430 | 2013-04-30 15:48:30 -0700 | [diff] [blame] | 200 | """ |
| 201 | Shell out to autoserv to run an individual test job. |
| 202 | |
| 203 | @param job: A Job object containing the control file contents and other |
| 204 | relevent metadata for this test. |
| 205 | @param host: Hostname of DUT to run test against. |
Fang Deng | b3ee20b | 2013-09-17 10:34:13 -0700 | [diff] [blame] | 206 | @param autotest_path: Absolute path of autotest directory. |
Aviv Keshet | c882440 | 2013-06-29 20:37:30 -0700 | [diff] [blame] | 207 | @param results_directory: Absolute path of directory to store results in. |
| 208 | (results will be stored in subdirectory of this). |
Christopher Wiley | f6b5aae | 2013-07-09 10:14:02 -0700 | [diff] [blame] | 209 | @param fast_mode: bool to use fast mode (disables slow autotest features). |
Aviv Keshet | 5e33c17 | 2013-07-16 05:00:49 -0700 | [diff] [blame] | 210 | @param id_digits: The minimum number of digits that job ids should be |
| 211 | 0-padded to when formatting as a string for results |
| 212 | directory. |
Aviv Keshet | c14951a | 2013-08-12 18:17:35 -0700 | [diff] [blame] | 213 | @param ssh_verbosity: SSH verbosity level, passed along to autoserv_utils |
Aviv Keshet | 6a704f7 | 2013-09-04 14:57:43 -0700 | [diff] [blame] | 214 | @param ssh_options: Additional ssh options to be passed to autoserv_utils |
Aviv Keshet | 30322f9 | 2013-07-18 13:21:52 -0700 | [diff] [blame] | 215 | @param args: String that should be passed as args parameter to autoserv, |
| 216 | and then ultimitely to test itself. |
Aviv Keshet | c5e4609 | 2013-07-19 10:15:40 -0700 | [diff] [blame] | 217 | @param pretend: If True, will print out autoserv commands rather than |
| 218 | running them. |
Aviv Keshet | 2ebbd35 | 2013-08-27 17:04:43 -0700 | [diff] [blame] | 219 | @param autoserv_verbose: If true, pass the --verbose flag to autoserv. |
Aviv Keshet | c882440 | 2013-06-29 20:37:30 -0700 | [diff] [blame] | 220 | @returns: Absolute path of directory where results were stored. |
Aviv Keshet | d4a0430 | 2013-04-30 15:48:30 -0700 | [diff] [blame] | 221 | """ |
| 222 | with tempfile.NamedTemporaryFile() as temp_file: |
| 223 | temp_file.write(job.control_file) |
| 224 | temp_file.flush() |
Aviv Keshet | ad7e34e | 2013-08-19 18:09:27 -0700 | [diff] [blame] | 225 | name_tail = job.name.split('/')[-1] |
Aviv Keshet | c882440 | 2013-06-29 20:37:30 -0700 | [diff] [blame] | 226 | results_directory = os.path.join(results_directory, |
Aviv Keshet | ad7e34e | 2013-08-19 18:09:27 -0700 | [diff] [blame] | 227 | 'results-%0*d-%s' % (id_digits, job.id, |
| 228 | name_tail)) |
Fang Deng | 2db9676 | 2013-10-03 16:45:31 -0700 | [diff] [blame] | 229 | # Drop experimental keyval in the keval file in the job result folder. |
| 230 | os.makedirs(results_directory) |
| 231 | utils.write_keyval(results_directory, |
| 232 | {constants.JOB_EXPERIMENTAL_KEY: job.keyvals[ |
| 233 | constants.JOB_EXPERIMENTAL_KEY]}) |
Aviv Keshet | 30322f9 | 2013-07-18 13:21:52 -0700 | [diff] [blame] | 234 | extra_args = [temp_file.name] |
| 235 | if args: |
| 236 | extra_args.extend(['--args', args]) |
Aviv Keshet | c882440 | 2013-06-29 20:37:30 -0700 | [diff] [blame] | 237 | |
Aviv Keshet | d4a0430 | 2013-04-30 15:48:30 -0700 | [diff] [blame] | 238 | command = autoserv_utils.autoserv_run_job_command( |
Fang Deng | b3ee20b | 2013-09-17 10:34:13 -0700 | [diff] [blame] | 239 | os.path.join(autotest_path, 'server'), |
Aviv Keshet | 2ebbd35 | 2013-08-27 17:04:43 -0700 | [diff] [blame] | 240 | machines=host, job=job, verbose=autoserv_verbose, |
Aviv Keshet | c882440 | 2013-06-29 20:37:30 -0700 | [diff] [blame] | 241 | results_directory=results_directory, |
Aviv Keshet | c14951a | 2013-08-12 18:17:35 -0700 | [diff] [blame] | 242 | fast_mode=fast_mode, ssh_verbosity=ssh_verbosity, |
Aviv Keshet | 6a704f7 | 2013-09-04 14:57:43 -0700 | [diff] [blame] | 243 | ssh_options=ssh_options, |
Aviv Keshet | e43bccf | 2013-08-14 14:11:59 -0700 | [diff] [blame] | 244 | extra_args=extra_args, |
| 245 | no_console_prefix=True) |
Aviv Keshet | c5e4609 | 2013-07-19 10:15:40 -0700 | [diff] [blame] | 246 | |
Fang Deng | 9a84123 | 2013-09-17 16:29:14 -0700 | [diff] [blame] | 247 | _run_autoserv(command, pretend) |
| 248 | return results_directory |
Aviv Keshet | d4a0430 | 2013-04-30 15:48:30 -0700 | [diff] [blame] | 249 | |
| 250 | |
| 251 | def setup_local_afe(): |
| 252 | """ |
| 253 | Setup a local afe database and return a direct_afe object to access it. |
| 254 | |
| 255 | @returns: A autotest_lib.frontend.afe.direct_afe instance. |
| 256 | """ |
| 257 | # This import statement is delayed until now rather than running at |
| 258 | # module load time, because it kicks off a local sqlite :memory: backed |
| 259 | # database, and we don't need that unless we are doing a local run. |
| 260 | from autotest_lib.frontend import setup_django_lite_environment |
| 261 | from autotest_lib.frontend.afe import direct_afe |
| 262 | return direct_afe.directAFE() |
| 263 | |
| 264 | |
Aviv Keshet | a6adc7a | 2013-08-30 11:13:38 -0700 | [diff] [blame] | 265 | def get_predicate_for_test_arg(test): |
| 266 | """ |
| 267 | Gets a suite predicte function for a given command-line argument. |
| 268 | |
| 269 | @param test: String. An individual TEST command line argument, e.g. |
| 270 | 'login_CryptohomeMounted' or 'suite:smoke' |
| 271 | @returns: A (predicate, string) tuple with the necessary suite |
| 272 | predicate, and a description string of the suite that |
| 273 | this predicate will produce. |
| 274 | """ |
Fang Deng | b1da830 | 2013-09-24 13:57:54 -0700 | [diff] [blame] | 275 | suitematch = re.match(_SUITE_REGEX, test) |
Aviv Keshet | a6adc7a | 2013-08-30 11:13:38 -0700 | [diff] [blame] | 276 | name_pattern_match = re.match(r'e:(.*)', test) |
| 277 | file_pattern_match = re.match(r'f:(.*)', test) |
| 278 | if suitematch: |
| 279 | suitename = suitematch.group(1) |
| 280 | return (suite.Suite.name_in_tag_predicate(suitename), |
| 281 | 'suite named %s' % suitename) |
| 282 | if name_pattern_match: |
| 283 | pattern = '^%s$' % name_pattern_match.group(1) |
| 284 | return (suite.Suite.test_name_matches_pattern_predicate(pattern), |
| 285 | 'suite to match name pattern %s' % pattern) |
| 286 | if file_pattern_match: |
| 287 | pattern = '^%s$' % file_pattern_match.group(1) |
| 288 | return (suite.Suite.test_file_matches_pattern_predicate(pattern), |
| 289 | 'suite to match file name pattern %s' % pattern) |
| 290 | return (suite.Suite.test_name_equals_predicate(test), |
| 291 | 'job named %s' % test) |
| 292 | |
| 293 | |
beeps | 4efdf03 | 2013-09-17 11:27:14 -0700 | [diff] [blame] | 294 | def _add_ssh_identity(temp_directory): |
| 295 | """Add an ssh identity to the agent. |
| 296 | |
| 297 | @param temp_directory: A directory to copy the testing_rsa into. |
| 298 | """ |
| 299 | # Add the testing key to the current ssh agent. |
| 300 | if os.environ.has_key('SSH_AGENT_PID'): |
| 301 | # Copy the testing key to the temp directory and make it NOT |
| 302 | # world-readable. Otherwise, ssh-add complains. |
| 303 | shutil.copy(_TEST_KEY_PATH, temp_directory) |
| 304 | key_copy_path = os.path.join(temp_directory, _TEST_KEY_FILENAME) |
| 305 | os.chmod(key_copy_path, stat.S_IRUSR | stat.S_IWUSR) |
| 306 | p = subprocess.Popen(['ssh-add', key_copy_path], |
| 307 | stderr=subprocess.STDOUT, stdout=subprocess.PIPE) |
| 308 | p_out, _ = p.communicate() |
| 309 | for line in p_out.splitlines(): |
Fang Deng | b1da830 | 2013-09-24 13:57:54 -0700 | [diff] [blame] | 310 | logging.info(line) |
beeps | 4efdf03 | 2013-09-17 11:27:14 -0700 | [diff] [blame] | 311 | else: |
Fang Deng | b1da830 | 2013-09-24 13:57:54 -0700 | [diff] [blame] | 312 | logging.warning('There appears to be no running ssh-agent. Attempting ' |
| 313 | 'to continue without running ssh-add, but ssh commands ' |
| 314 | 'may fail.') |
beeps | 4efdf03 | 2013-09-17 11:27:14 -0700 | [diff] [blame] | 315 | |
| 316 | |
| 317 | def _get_board_from_host(remote): |
| 318 | """Get the board of the remote host. |
| 319 | |
| 320 | @param remote: string representing the IP of the remote host. |
| 321 | |
| 322 | @return: A string representing the board of the remote host. |
| 323 | """ |
| 324 | logging.info('Board unspecified, attempting to determine board from host.') |
| 325 | host = factory.create_host(remote) |
| 326 | try: |
| 327 | board = host.get_board().replace(constants.BOARD_PREFIX, '') |
| 328 | except error.AutoservRunError: |
| 329 | raise TestThatRunError('Cannot determine board, please specify ' |
| 330 | 'a --board option.') |
| 331 | logging.info('Detected host board: %s', board) |
| 332 | return board |
| 333 | |
| 334 | |
Fang Deng | b1da830 | 2013-09-24 13:57:54 -0700 | [diff] [blame] | 335 | def _auto_detect_labels(afe, remote): |
| 336 | """Automatically detect host labels and add them to the host in afe. |
| 337 | |
| 338 | Note that the label of board will not be auto-detected. |
| 339 | This method assumes the host |remote| has already been added to afe. |
| 340 | |
| 341 | @param afe: A direct_afe object used to interact with local afe database. |
| 342 | @param remote: The hostname of the remote device. |
| 343 | |
| 344 | """ |
| 345 | cros_host = factory.create_host(remote) |
| 346 | labels_to_create = [label for label in cros_host.get_labels() |
| 347 | if not label.startswith(constants.BOARD_PREFIX)] |
| 348 | labels_to_add_to_afe_host = [] |
| 349 | for label in labels_to_create: |
| 350 | new_label = afe.create_label(label) |
| 351 | labels_to_add_to_afe_host.append(new_label.name) |
| 352 | hosts = afe.get_hosts(hostname=remote) |
| 353 | if not hosts: |
| 354 | raise TestThatRunError('Unexpected error: %s has not ' |
| 355 | 'been added to afe.' % remote) |
| 356 | afe_host = hosts[0] |
| 357 | afe_host.add_labels(labels_to_add_to_afe_host) |
| 358 | |
| 359 | |
Christopher Wiley | f6b5aae | 2013-07-09 10:14:02 -0700 | [diff] [blame] | 360 | def perform_local_run(afe, autotest_path, tests, remote, fast_mode, |
Aviv Keshet | c5e4609 | 2013-07-19 10:15:40 -0700 | [diff] [blame] | 361 | build=_NO_BUILD, board=_NO_BOARD, args=None, |
Aviv Keshet | 15782de | 2013-07-31 11:40:41 -0700 | [diff] [blame] | 362 | pretend=False, no_experimental=False, |
Fang Deng | a6d597a | 2013-10-10 13:58:42 -0700 | [diff] [blame] | 363 | ignore_deps=True, |
Aviv Keshet | 2ebbd35 | 2013-08-27 17:04:43 -0700 | [diff] [blame] | 364 | results_directory=None, ssh_verbosity=0, |
Aviv Keshet | 6a704f7 | 2013-09-04 14:57:43 -0700 | [diff] [blame] | 365 | ssh_options=None, |
Christopher Wiley | 2ce92d5 | 2014-03-14 11:07:49 -0700 | [diff] [blame] | 366 | autoserv_verbose=False, |
| 367 | iterations=1): |
Fang Deng | b1da830 | 2013-09-24 13:57:54 -0700 | [diff] [blame] | 368 | """Perform local run of tests. |
| 369 | |
| 370 | This method enforces satisfaction of test dependencies for tests that are |
| 371 | run as a part of a suite. |
| 372 | |
Aviv Keshet | d4a0430 | 2013-04-30 15:48:30 -0700 | [diff] [blame] | 373 | @param afe: A direct_afe object used to interact with local afe database. |
Fang Deng | b3ee20b | 2013-09-17 10:34:13 -0700 | [diff] [blame] | 374 | @param autotest_path: Absolute path of autotest installed in sysroot or |
| 375 | custom autotest path set by --autotest_dir. |
Aviv Keshet | d4a0430 | 2013-04-30 15:48:30 -0700 | [diff] [blame] | 376 | @param tests: List of strings naming tests and suites to run. Suite strings |
| 377 | should be formed like "suite:smoke". |
| 378 | @param remote: Remote hostname. |
Christopher Wiley | f6b5aae | 2013-07-09 10:14:02 -0700 | [diff] [blame] | 379 | @param fast_mode: bool to use fast mode (disables slow autotest features). |
Aviv Keshet | d4a0430 | 2013-04-30 15:48:30 -0700 | [diff] [blame] | 380 | @param build: String specifying build for local run. |
Aviv Keshet | 1071196 | 2013-06-24 12:20:33 -0700 | [diff] [blame] | 381 | @param board: String specifyinb board for local run. |
Aviv Keshet | 30322f9 | 2013-07-18 13:21:52 -0700 | [diff] [blame] | 382 | @param args: String that should be passed as args parameter to autoserv, |
| 383 | and then ultimitely to test itself. |
Aviv Keshet | c5e4609 | 2013-07-19 10:15:40 -0700 | [diff] [blame] | 384 | @param pretend: If True, will print out autoserv commands rather than |
| 385 | running them. |
Aviv Keshet | e9170d9 | 2013-07-19 11:20:45 -0700 | [diff] [blame] | 386 | @param no_experimental: Skip experimental tests when scheduling a suite. |
Fang Deng | a6d597a | 2013-10-10 13:58:42 -0700 | [diff] [blame] | 387 | @param ignore_deps: If True, test dependencies will be ignored. |
Aviv Keshet | 15782de | 2013-07-31 11:40:41 -0700 | [diff] [blame] | 388 | @param results_directory: Directory to store results in. Defaults to None, |
| 389 | in which case results will be stored in a new |
| 390 | subdirectory of /tmp |
Aviv Keshet | c14951a | 2013-08-12 18:17:35 -0700 | [diff] [blame] | 391 | @param ssh_verbosity: SSH verbosity level, passed through to |
| 392 | autoserv_utils. |
Aviv Keshet | 6a704f7 | 2013-09-04 14:57:43 -0700 | [diff] [blame] | 393 | @param ssh_options: Additional ssh options to be passed to autoserv_utils |
Aviv Keshet | 2ebbd35 | 2013-08-27 17:04:43 -0700 | [diff] [blame] | 394 | @param autoserv_verbose: If true, pass the --verbose flag to autoserv. |
Christopher Wiley | 2ce92d5 | 2014-03-14 11:07:49 -0700 | [diff] [blame] | 395 | @param iterations: int number of times to schedule tests. |
Aviv Keshet | d4a0430 | 2013-04-30 15:48:30 -0700 | [diff] [blame] | 396 | """ |
Fang Deng | b1da830 | 2013-09-24 13:57:54 -0700 | [diff] [blame] | 397 | |
| 398 | # Create host in afe, add board and build labels. |
Fang Deng | 9a84123 | 2013-09-17 16:29:14 -0700 | [diff] [blame] | 399 | cros_version_label = provision.cros_version_to_label(build) |
| 400 | build_label = afe.create_label(cros_version_label) |
Fang Deng | b1da830 | 2013-09-24 13:57:54 -0700 | [diff] [blame] | 401 | board_label = afe.create_label(constants.BOARD_PREFIX + board) |
Aviv Keshet | f2b6910 | 2013-08-28 10:34:49 -0700 | [diff] [blame] | 402 | new_host = afe.create_host(remote) |
| 403 | new_host.add_labels([build_label.name, board_label.name]) |
Fang Deng | a6d597a | 2013-10-10 13:58:42 -0700 | [diff] [blame] | 404 | if not ignore_deps: |
| 405 | logging.info('Auto-detecting labels for %s', remote) |
| 406 | _auto_detect_labels(afe, remote) |
Fang Deng | 9a84123 | 2013-09-17 16:29:14 -0700 | [diff] [blame] | 407 | # Provision the host to |build|. |
| 408 | if build != _NO_BUILD: |
| 409 | logging.info('Provisioning %s...', cros_version_label) |
| 410 | try: |
| 411 | run_provisioning_job(cros_version_label, remote, autotest_path, |
| 412 | results_directory, fast_mode, |
| 413 | ssh_verbosity, ssh_options, |
| 414 | pretend, autoserv_verbose) |
| 415 | except TestThatProvisioningError as e: |
| 416 | logging.error('Provisioning %s to %s failed, tests are aborted, ' |
| 417 | 'failure reason: %s', |
| 418 | remote, cros_version_label, e) |
| 419 | return |
Aviv Keshet | d4a0430 | 2013-04-30 15:48:30 -0700 | [diff] [blame] | 420 | |
Aviv Keshet | 20b20c7 | 2014-04-02 11:54:28 -0700 | [diff] [blame] | 421 | # Create suites that will be scheduled. |
| 422 | suites_and_descriptions = [] |
| 423 | for test in tests: |
| 424 | (predicate, description) = get_predicate_for_test_arg(test) |
| 425 | logging.info('Fetching suite for %s...', description) |
| 426 | suite = fetch_local_suite(autotest_path, predicate, afe, |
| 427 | remote=remote, |
| 428 | build=build, board=board, |
| 429 | results_directory=results_directory, |
| 430 | no_experimental=no_experimental, |
| 431 | ignore_deps=ignore_deps) |
| 432 | suites_and_descriptions.append((suite, description)) |
| 433 | |
| 434 | # Schedule the suites, looping over iterations if necessary. |
Christopher Wiley | 2ce92d5 | 2014-03-14 11:07:49 -0700 | [diff] [blame] | 435 | for iteration in range(iterations): |
| 436 | if iteration > 0: |
| 437 | logging.info('Repeating scheduling for iteration %d:', iteration) |
Aviv Keshet | 20b20c7 | 2014-04-02 11:54:28 -0700 | [diff] [blame] | 438 | |
| 439 | for suite, description in suites_and_descriptions: |
| 440 | logging.info('Scheduling suite for %s...', description) |
| 441 | ntests = suite.schedule( |
| 442 | lambda log_entry, log_in_subdir=False: None, |
| 443 | add_experimental=not no_experimental) |
Christopher Wiley | 2ce92d5 | 2014-03-14 11:07:49 -0700 | [diff] [blame] | 444 | logging.info('... scheduled %s job(s).', ntests) |
Aviv Keshet | d4a0430 | 2013-04-30 15:48:30 -0700 | [diff] [blame] | 445 | |
Aviv Keshet | 5e33c17 | 2013-07-16 05:00:49 -0700 | [diff] [blame] | 446 | if not afe.get_jobs(): |
| 447 | logging.info('No jobs scheduled. End of local run.') |
Fang Deng | b1da830 | 2013-09-24 13:57:54 -0700 | [diff] [blame] | 448 | return |
Aviv Keshet | 5e33c17 | 2013-07-16 05:00:49 -0700 | [diff] [blame] | 449 | |
| 450 | last_job_id = afe.get_jobs()[-1].id |
Fang Deng | b3ee20b | 2013-09-17 10:34:13 -0700 | [diff] [blame] | 451 | job_id_digits = len(str(last_job_id)) |
Aviv Keshet | d4a0430 | 2013-04-30 15:48:30 -0700 | [diff] [blame] | 452 | for job in afe.get_jobs(): |
Aviv Keshet | 5e33c17 | 2013-07-16 05:00:49 -0700 | [diff] [blame] | 453 | run_job(job, remote, autotest_path, results_directory, fast_mode, |
Aviv Keshet | 6a704f7 | 2013-09-04 14:57:43 -0700 | [diff] [blame] | 454 | job_id_digits, ssh_verbosity, ssh_options, args, pretend, |
| 455 | autoserv_verbose) |
Aviv Keshet | d4a0430 | 2013-04-30 15:48:30 -0700 | [diff] [blame] | 456 | |
Aviv Keshet | d4a0430 | 2013-04-30 15:48:30 -0700 | [diff] [blame] | 457 | |
| 458 | def validate_arguments(arguments): |
Aviv Keshet | 0e5d525 | 2013-04-26 16:01:36 -0700 | [diff] [blame] | 459 | """ |
| 460 | Validates parsed arguments. |
| 461 | |
| 462 | @param arguments: arguments object, as parsed by ParseArguments |
| 463 | @raises: ValueError if arguments were invalid. |
| 464 | """ |
Aviv Keshet | d4a0430 | 2013-04-30 15:48:30 -0700 | [diff] [blame] | 465 | if arguments.remote == ':lab:': |
Aviv Keshet | 30322f9 | 2013-07-18 13:21:52 -0700 | [diff] [blame] | 466 | if arguments.args: |
| 467 | raise ValueError('--args flag not supported when running against ' |
| 468 | ':lab:') |
Aviv Keshet | c5e4609 | 2013-07-19 10:15:40 -0700 | [diff] [blame] | 469 | if arguments.pretend: |
| 470 | raise ValueError('--pretend flag not supported when running ' |
| 471 | 'against :lab:') |
Aviv Keshet | c14951a | 2013-08-12 18:17:35 -0700 | [diff] [blame] | 472 | if arguments.ssh_verbosity: |
| 473 | raise ValueError('--ssh_verbosity flag not supported when running ' |
| 474 | 'against :lab:') |
| 475 | |
Aviv Keshet | d4a0430 | 2013-04-30 15:48:30 -0700 | [diff] [blame] | 476 | |
| 477 | def parse_arguments(argv): |
Aviv Keshet | 021c19f | 2013-02-22 13:19:43 -0800 | [diff] [blame] | 478 | """ |
| 479 | Parse command line arguments |
| 480 | |
| 481 | @param argv: argument list to parse |
| 482 | @returns: parsed arguments. |
Aviv Keshet | 7cd1231 | 2013-07-25 10:25:55 -0700 | [diff] [blame] | 483 | @raises SystemExit if arguments are malformed, or required arguments |
| 484 | are not present. |
Aviv Keshet | 021c19f | 2013-02-22 13:19:43 -0800 | [diff] [blame] | 485 | """ |
| 486 | parser = argparse.ArgumentParser(description='Run remote tests.') |
| 487 | |
Aviv Keshet | 0e5d525 | 2013-04-26 16:01:36 -0700 | [diff] [blame] | 488 | parser.add_argument('remote', metavar='REMOTE', |
Aviv Keshet | 021c19f | 2013-02-22 13:19:43 -0800 | [diff] [blame] | 489 | help='hostname[:port] for remote device. Specify ' |
Aviv Keshet | e43bccf | 2013-08-14 14:11:59 -0700 | [diff] [blame] | 490 | ':lab: to run in test lab, or :vm:PORT_NUMBER to ' |
| 491 | 'run in vm.') |
Aviv Keshet | 021c19f | 2013-02-22 13:19:43 -0800 | [diff] [blame] | 492 | parser.add_argument('tests', nargs='+', metavar='TEST', |
| 493 | help='Run given test(s). Use suite:SUITE to specify ' |
Aviv Keshet | a6adc7a | 2013-08-30 11:13:38 -0700 | [diff] [blame] | 494 | 'test suite. Use e:[NAME_PATTERN] to specify a ' |
| 495 | 'NAME-matching regular expression. Use ' |
| 496 | 'f:[FILE_PATTERN] to specify a filename matching ' |
| 497 | 'regular expression. Specified regular ' |
Jorge Lucangeli Obes | f81917c | 2014-04-17 16:28:39 -0700 | [diff] [blame] | 498 | 'expressions will be implicitly wrapped in ' |
Aviv Keshet | a6adc7a | 2013-08-30 11:13:38 -0700 | [diff] [blame] | 499 | '^ and $.') |
Prathmesh Prabhu | 63f00aa | 2013-07-26 13:33:06 -0700 | [diff] [blame] | 500 | default_board = cros_build_lib.GetDefaultBoard() |
| 501 | parser.add_argument('-b', '--board', metavar='BOARD', default=default_board, |
Aviv Keshet | 0e5d525 | 2013-04-26 16:01:36 -0700 | [diff] [blame] | 502 | action='store', |
Prathmesh Prabhu | 63f00aa | 2013-07-26 13:33:06 -0700 | [diff] [blame] | 503 | help='Board for which the test will run. Default: %s' % |
| 504 | (default_board or 'Not configured')) |
Fang Deng | 9a84123 | 2013-09-17 16:29:14 -0700 | [diff] [blame] | 505 | parser.add_argument('-i', '--build', metavar='BUILD', default=_NO_BUILD, |
Aviv Keshet | 021c19f | 2013-02-22 13:19:43 -0800 | [diff] [blame] | 506 | help='Build to test. Device will be reimaged if ' |
Aviv Keshet | e43bccf | 2013-08-14 14:11:59 -0700 | [diff] [blame] | 507 | 'necessary. Omit flag to skip reimage and test ' |
Aviv Keshet | 2113774 | 2014-01-10 14:52:38 -0800 | [diff] [blame] | 508 | 'against already installed DUT image. Examples: ' |
| 509 | 'link-paladin/R34-5222.0.0-rc2, ' |
| 510 | 'lumpy-release/R34-5205.0.0') |
Christopher Wiley | f6b5aae | 2013-07-09 10:14:02 -0700 | [diff] [blame] | 511 | parser.add_argument('--fast', action='store_true', dest='fast_mode', |
| 512 | default=False, |
| 513 | help='Enable fast mode. This will cause test_that to ' |
| 514 | 'skip time consuming steps like sysinfo and ' |
| 515 | 'collecting crash information.') |
Aviv Keshet | d4a0430 | 2013-04-30 15:48:30 -0700 | [diff] [blame] | 516 | parser.add_argument('--args', metavar='ARGS', |
beeps | f860bd1 | 2014-01-23 18:06:06 -0800 | [diff] [blame] | 517 | help='Whitespace separated argument string to pass ' |
| 518 | 'through to test. Only supported for runs ' |
| 519 | 'against a local DUT.') |
Fang Deng | b3ee20b | 2013-09-17 10:34:13 -0700 | [diff] [blame] | 520 | parser.add_argument('--autotest_dir', metavar='AUTOTEST_DIR', |
| 521 | help='Use AUTOTEST_DIR instead of normal board sysroot ' |
| 522 | 'copy of autotest, and skip the quickmerge step.') |
beeps | 4efdf03 | 2013-09-17 11:27:14 -0700 | [diff] [blame] | 523 | parser.add_argument('--results_dir', metavar='RESULTS_DIR', default=None, |
Aviv Keshet | 15782de | 2013-07-31 11:40:41 -0700 | [diff] [blame] | 524 | help='Instead of storing results in a new subdirectory' |
Aviv Keshet | e43bccf | 2013-08-14 14:11:59 -0700 | [diff] [blame] | 525 | ' of /tmp , store results in RESULTS_DIR. If ' |
Dan Shi | 49db6aa | 2013-09-13 12:09:39 -0700 | [diff] [blame] | 526 | 'RESULTS_DIR already exists, it will be deleted.') |
Aviv Keshet | c5e4609 | 2013-07-19 10:15:40 -0700 | [diff] [blame] | 527 | parser.add_argument('--pretend', action='store_true', default=False, |
| 528 | help='Print autoserv commands that would be run, ' |
| 529 | 'rather than running them.') |
Aviv Keshet | 8ea71df | 2013-07-19 10:49:36 -0700 | [diff] [blame] | 530 | parser.add_argument('--no-quickmerge', action='store_true', default=False, |
| 531 | dest='no_quickmerge', |
| 532 | help='Skip the quickmerge step and use the sysroot ' |
| 533 | 'as it currently is. May result in un-merged ' |
Jorge Lucangeli Obes | ad7309b | 2014-02-12 09:57:37 -0800 | [diff] [blame] | 534 | 'source tree changes not being reflected in the ' |
| 535 | 'run. If using --autotest_dir, this flag is ' |
Fang Deng | b3ee20b | 2013-09-17 10:34:13 -0700 | [diff] [blame] | 536 | 'automatically applied.') |
Aviv Keshet | e9170d9 | 2013-07-19 11:20:45 -0700 | [diff] [blame] | 537 | parser.add_argument('--no-experimental', action='store_true', |
| 538 | default=False, dest='no_experimental', |
| 539 | help='When scheduling a suite, skip any tests marked ' |
| 540 | 'as experimental. Applies only to tests scheduled' |
| 541 | ' via suite:[SUITE].') |
Aviv Keshet | fd77591 | 2013-08-06 11:37:16 -0700 | [diff] [blame] | 542 | parser.add_argument('--whitelist-chrome-crashes', action='store_true', |
| 543 | default=False, dest='whitelist_chrome_crashes', |
| 544 | help='Ignore chrome crashes when producing test ' |
Aviv Keshet | e43bccf | 2013-08-14 14:11:59 -0700 | [diff] [blame] | 545 | 'report. This flag gets passed along to the ' |
| 546 | 'report generation tool.') |
Fang Deng | a6d597a | 2013-10-10 13:58:42 -0700 | [diff] [blame] | 547 | parser.add_argument('--enforce-deps', action='store_true', |
| 548 | default=False, dest='enforce_deps', |
| 549 | help='Skip tests whose DEPENDENCIES can not ' |
| 550 | 'be satisfied.') |
Aviv Keshet | c14951a | 2013-08-12 18:17:35 -0700 | [diff] [blame] | 551 | parser.add_argument('--ssh_verbosity', action='store', type=int, |
| 552 | choices=[0, 1, 2, 3], default=0, |
| 553 | help='Verbosity level for ssh, between 0 and 3 ' |
| 554 | 'inclusive.') |
Aviv Keshet | 6a704f7 | 2013-09-04 14:57:43 -0700 | [diff] [blame] | 555 | parser.add_argument('--ssh_options', action='store', default=None, |
| 556 | help='A string giving additional options to be ' |
| 557 | 'added to ssh commands.') |
Aviv Keshet | e43bccf | 2013-08-14 14:11:59 -0700 | [diff] [blame] | 558 | parser.add_argument('--debug', action='store_true', |
| 559 | help='Include DEBUG level messages in stdout. Note: ' |
| 560 | 'these messages will be included in output log ' |
Aviv Keshet | 2ebbd35 | 2013-08-27 17:04:43 -0700 | [diff] [blame] | 561 | 'file regardless. In addition, turn on autoserv ' |
| 562 | 'verbosity.') |
Christopher Wiley | 2ce92d5 | 2014-03-14 11:07:49 -0700 | [diff] [blame] | 563 | parser.add_argument('--iterations', action='store', type=int, default=1, |
| 564 | help='Number of times to run the tests specified.') |
Aviv Keshet | 021c19f | 2013-02-22 13:19:43 -0800 | [diff] [blame] | 565 | return parser.parse_args(argv) |
| 566 | |
| 567 | |
Aviv Keshet | 1d991ea | 2013-06-12 17:24:23 -0700 | [diff] [blame] | 568 | def sigint_handler(signum, stack_frame): |
| 569 | #pylint: disable-msg=C0111 |
| 570 | """Handle SIGINT or SIGTERM to a local test_that run. |
| 571 | |
| 572 | This handler sends a SIGINT to the running autoserv process, |
| 573 | if one is running, giving it up to 5 seconds to clean up and exit. After |
| 574 | the timeout elapses, autoserv is killed. In either case, after autoserv |
| 575 | exits then this process exits with status 1. |
| 576 | """ |
| 577 | # If multiple signals arrive before handler is unset, ignore duplicates |
| 578 | if not _sigint_handler_lock.acquire(False): |
| 579 | return |
| 580 | try: |
| 581 | # Ignore future signals by unsetting handler. |
| 582 | signal.signal(signal.SIGINT, signal.SIG_IGN) |
| 583 | signal.signal(signal.SIGTERM, signal.SIG_IGN) |
| 584 | |
| 585 | logging.warning('Received SIGINT or SIGTERM. Cleaning up and exiting.') |
| 586 | if _autoserv_proc: |
| 587 | logging.warning('Sending SIGINT to autoserv process. Waiting up ' |
| 588 | 'to %s seconds for cleanup.', |
| 589 | _AUTOSERV_SIGINT_TIMEOUT_SECONDS) |
| 590 | _autoserv_proc.send_signal(signal.SIGINT) |
| 591 | timed_out, _ = retry.timeout(_autoserv_proc.wait, |
| 592 | timeout_sec=_AUTOSERV_SIGINT_TIMEOUT_SECONDS) |
| 593 | if timed_out: |
| 594 | _autoserv_proc.kill() |
| 595 | logging.warning('Timed out waiting for autoserv to handle ' |
| 596 | 'SIGINT. Killed autoserv.') |
| 597 | finally: |
| 598 | _sigint_handler_lock.release() # this is not really necessary? |
| 599 | sys.exit(1) |
| 600 | |
| 601 | |
beeps | 4efdf03 | 2013-09-17 11:27:14 -0700 | [diff] [blame] | 602 | def _create_results_directory(results_directory=None): |
| 603 | """Create a results directory. |
| 604 | |
| 605 | If no directory is specified this method will create and return a |
| 606 | temp directory to hold results. If a directory name is specified this |
| 607 | method will create a directory at the given path, provided it doesn't |
| 608 | already exist. |
| 609 | |
| 610 | @param results_directory: The path to the results_directory to create. |
| 611 | |
| 612 | @return results_directory: A path to the results_directory, ready for use. |
| 613 | """ |
| 614 | if results_directory is None: |
| 615 | # Create a results_directory as subdir of /tmp |
| 616 | results_directory = tempfile.mkdtemp(prefix='test_that_results_') |
| 617 | else: |
| 618 | # Delete results_directory if it already exists. |
| 619 | try: |
| 620 | shutil.rmtree(results_directory) |
| 621 | except OSError as e: |
| 622 | if e.errno != errno.ENOENT: |
| 623 | raise |
| 624 | |
| 625 | # Create results_directory if it does not exist |
| 626 | try: |
| 627 | os.makedirs(results_directory) |
| 628 | except OSError as e: |
| 629 | if e.errno != errno.EEXIST: |
| 630 | raise |
| 631 | return results_directory |
| 632 | |
| 633 | |
Aviv Keshet | 6c02839 | 2013-07-22 12:57:32 -0700 | [diff] [blame] | 634 | def _perform_bootstrap_into_autotest_root(arguments, autotest_path, argv, |
| 635 | legacy_path=False): |
Aviv Keshet | 021c19f | 2013-02-22 13:19:43 -0800 | [diff] [blame] | 636 | """ |
Fang Deng | b3ee20b | 2013-09-17 10:34:13 -0700 | [diff] [blame] | 637 | Perfoms a bootstrap to run test_that from the |autotest_path|. |
Aviv Keshet | e10e8d0 | 2013-09-09 17:04:58 -0700 | [diff] [blame] | 638 | |
| 639 | This function is to be called from test_that's main() script, when |
| 640 | test_that is executed from the source tree location. It runs |
Fang Deng | b3ee20b | 2013-09-17 10:34:13 -0700 | [diff] [blame] | 641 | autotest_quickmerge to update the sysroot unless arguments.no_quickmerge |
| 642 | is set. It then executes and waits on the version of test_that.py |
| 643 | in |autotest_path|. |
Aviv Keshet | e10e8d0 | 2013-09-09 17:04:58 -0700 | [diff] [blame] | 644 | |
| 645 | @param arguments: A parsed arguments object, as returned from |
| 646 | parse_arguments(...). |
Fang Deng | b3ee20b | 2013-09-17 10:34:13 -0700 | [diff] [blame] | 647 | @param autotest_path: Full absolute path to the autotest root directory. |
Aviv Keshet | e10e8d0 | 2013-09-09 17:04:58 -0700 | [diff] [blame] | 648 | @param argv: The arguments list, as passed to main(...) |
Aviv Keshet | 6c02839 | 2013-07-22 12:57:32 -0700 | [diff] [blame] | 649 | @param legacy_path: Flag for backwards compatibility with builds |
| 650 | that have autotest in old usr/local/autotest location |
Aviv Keshet | e10e8d0 | 2013-09-09 17:04:58 -0700 | [diff] [blame] | 651 | |
Fang Deng | b3ee20b | 2013-09-17 10:34:13 -0700 | [diff] [blame] | 652 | @returns: The return code of the test_that script that was executed in |
| 653 | |autotest_path|. |
Aviv Keshet | 021c19f | 2013-02-22 13:19:43 -0800 | [diff] [blame] | 654 | """ |
Aviv Keshet | e10e8d0 | 2013-09-09 17:04:58 -0700 | [diff] [blame] | 655 | logging_manager.configure_logging( |
| 656 | server_logging_config.ServerLoggingConfig(), |
| 657 | use_console=True, |
| 658 | verbose=arguments.debug) |
| 659 | if arguments.no_quickmerge: |
Fang Deng | b3ee20b | 2013-09-17 10:34:13 -0700 | [diff] [blame] | 660 | logging.info('Skipping quickmerge step.') |
Aviv Keshet | e10e8d0 | 2013-09-09 17:04:58 -0700 | [diff] [blame] | 661 | else: |
| 662 | logging.info('Running autotest_quickmerge step.') |
Aviv Keshet | 6c02839 | 2013-07-22 12:57:32 -0700 | [diff] [blame] | 663 | command = [_QUICKMERGE_SCRIPTNAME, '--board='+arguments.board] |
| 664 | if legacy_path: |
| 665 | command.append('--legacy_path') |
| 666 | s = subprocess.Popen(command, |
| 667 | stdout=subprocess.PIPE, |
| 668 | stderr=subprocess.STDOUT) |
Aviv Keshet | e10e8d0 | 2013-09-09 17:04:58 -0700 | [diff] [blame] | 669 | for message in iter(s.stdout.readline, b''): |
Aviv Keshet | 9525124 | 2013-10-10 07:43:41 -0700 | [diff] [blame] | 670 | logging.info('quickmerge| %s', message.strip()) |
Aviv Keshet | f022942 | 2013-10-24 10:12:14 -0400 | [diff] [blame] | 671 | return_code = s.wait() |
| 672 | if return_code: |
| 673 | raise TestThatRunError('autotest_quickmerge failed with error code' |
| 674 | ' %s.' % return_code) |
Aviv Keshet | 0e5d525 | 2013-04-26 16:01:36 -0700 | [diff] [blame] | 675 | |
Fang Deng | b3ee20b | 2013-09-17 10:34:13 -0700 | [diff] [blame] | 676 | logging.info('Re-running test_that script in %s copy of autotest.', |
| 677 | autotest_path) |
| 678 | script_command = os.path.join(autotest_path, 'site_utils', |
Aviv Keshet | e10e8d0 | 2013-09-09 17:04:58 -0700 | [diff] [blame] | 679 | os.path.basename(__file__)) |
Aviv Keshet | 1a6ce6e | 2013-10-22 12:32:38 -0400 | [diff] [blame] | 680 | if not os.path.exists(script_command): |
| 681 | raise TestThatRunError('Unable to bootstrap to autotest root, ' |
| 682 | '%s not found.' % script_command) |
Aviv Keshet | e10e8d0 | 2013-09-09 17:04:58 -0700 | [diff] [blame] | 683 | proc = None |
| 684 | def resend_sig(signum, stack_frame): |
| 685 | #pylint: disable-msg=C0111 |
| 686 | if proc: |
| 687 | proc.send_signal(signum) |
| 688 | signal.signal(signal.SIGINT, resend_sig) |
| 689 | signal.signal(signal.SIGTERM, resend_sig) |
Aviv Keshet | 712128f | 2013-06-11 14:41:08 -0700 | [diff] [blame] | 690 | |
Aviv Keshet | e10e8d0 | 2013-09-09 17:04:58 -0700 | [diff] [blame] | 691 | proc = subprocess.Popen([script_command] + argv) |
Aviv Keshet | 0e5d525 | 2013-04-26 16:01:36 -0700 | [diff] [blame] | 692 | |
Aviv Keshet | e10e8d0 | 2013-09-09 17:04:58 -0700 | [diff] [blame] | 693 | return proc.wait() |
Aviv Keshet | 0e5d525 | 2013-04-26 16:01:36 -0700 | [diff] [blame] | 694 | |
Aviv Keshet | 0e5d525 | 2013-04-26 16:01:36 -0700 | [diff] [blame] | 695 | |
Fang Deng | b3ee20b | 2013-09-17 10:34:13 -0700 | [diff] [blame] | 696 | def _perform_run_from_autotest_root(arguments, autotest_path, argv): |
Aviv Keshet | e10e8d0 | 2013-09-09 17:04:58 -0700 | [diff] [blame] | 697 | """ |
Fang Deng | b3ee20b | 2013-09-17 10:34:13 -0700 | [diff] [blame] | 698 | Perform a test_that run, from the |autotest_path|. |
Aviv Keshet | 2dc9893 | 2013-06-17 15:22:10 -0700 | [diff] [blame] | 699 | |
Aviv Keshet | e10e8d0 | 2013-09-09 17:04:58 -0700 | [diff] [blame] | 700 | This function is to be called from test_that's main() script, when |
Fang Deng | b3ee20b | 2013-09-17 10:34:13 -0700 | [diff] [blame] | 701 | test_that is executed from the |autotest_path|. It handles all stages |
| 702 | of a test_that run that come after the bootstrap into |autotest_path|. |
Aviv Keshet | 1d991ea | 2013-06-12 17:24:23 -0700 | [diff] [blame] | 703 | |
Aviv Keshet | e10e8d0 | 2013-09-09 17:04:58 -0700 | [diff] [blame] | 704 | @param arguments: A parsed arguments object, as returned from |
| 705 | parse_arguments(...). |
Fang Deng | b3ee20b | 2013-09-17 10:34:13 -0700 | [diff] [blame] | 706 | @param autotest_path: Full absolute path to the autotest root directory. |
Aviv Keshet | e10e8d0 | 2013-09-09 17:04:58 -0700 | [diff] [blame] | 707 | @param argv: The arguments list, as passed to main(...) |
Aviv Keshet | 1d991ea | 2013-06-12 17:24:23 -0700 | [diff] [blame] | 708 | |
Aviv Keshet | e10e8d0 | 2013-09-09 17:04:58 -0700 | [diff] [blame] | 709 | @returns: A return code that test_that should exit with. |
| 710 | """ |
Aviv Keshet | e43bccf | 2013-08-14 14:11:59 -0700 | [diff] [blame] | 711 | results_directory = arguments.results_dir |
beeps | 4efdf03 | 2013-09-17 11:27:14 -0700 | [diff] [blame] | 712 | if results_directory is None or not os.path.exists(results_directory): |
| 713 | raise ValueError('Expected valid results directory, got %s' % |
| 714 | results_directory) |
Aviv Keshet | e43bccf | 2013-08-14 14:11:59 -0700 | [diff] [blame] | 715 | |
| 716 | logging_manager.configure_logging( |
| 717 | server_logging_config.ServerLoggingConfig(), |
| 718 | results_dir=results_directory, |
| 719 | use_console=True, |
| 720 | verbose=arguments.debug, |
| 721 | debug_log_name='test_that') |
| 722 | logging.info('Began logging to %s', results_directory) |
| 723 | |
Aviv Keshet | bebd621 | 2013-09-04 17:36:23 -0700 | [diff] [blame] | 724 | logging.debug('test_that command line was: %s', argv) |
| 725 | |
Aviv Keshet | 1d991ea | 2013-06-12 17:24:23 -0700 | [diff] [blame] | 726 | signal.signal(signal.SIGINT, sigint_handler) |
| 727 | signal.signal(signal.SIGTERM, sigint_handler) |
| 728 | |
Aviv Keshet | 6f74e19 | 2013-09-23 11:09:07 -0700 | [diff] [blame] | 729 | afe = setup_local_afe() |
| 730 | perform_local_run(afe, autotest_path, arguments.tests, |
| 731 | arguments.remote, arguments.fast_mode, |
Fang Deng | b1da830 | 2013-09-24 13:57:54 -0700 | [diff] [blame] | 732 | arguments.build, arguments.board, |
Aviv Keshet | 6f74e19 | 2013-09-23 11:09:07 -0700 | [diff] [blame] | 733 | args=arguments.args, |
| 734 | pretend=arguments.pretend, |
| 735 | no_experimental=arguments.no_experimental, |
Fang Deng | a6d597a | 2013-10-10 13:58:42 -0700 | [diff] [blame] | 736 | ignore_deps=not arguments.enforce_deps, |
Aviv Keshet | 6f74e19 | 2013-09-23 11:09:07 -0700 | [diff] [blame] | 737 | results_directory=results_directory, |
| 738 | ssh_verbosity=arguments.ssh_verbosity, |
| 739 | ssh_options=arguments.ssh_options, |
Christopher Wiley | 2ce92d5 | 2014-03-14 11:07:49 -0700 | [diff] [blame] | 740 | autoserv_verbose=arguments.debug, |
| 741 | iterations=arguments.iterations) |
Aviv Keshet | 6f74e19 | 2013-09-23 11:09:07 -0700 | [diff] [blame] | 742 | if arguments.pretend: |
| 743 | logging.info('Finished pretend run. Exiting.') |
| 744 | return 0 |
Aviv Keshet | c5e4609 | 2013-07-19 10:15:40 -0700 | [diff] [blame] | 745 | |
Aviv Keshet | 6f74e19 | 2013-09-23 11:09:07 -0700 | [diff] [blame] | 746 | test_report_command = [_TEST_REPORT_SCRIPTNAME] |
Fang Deng | a222727 | 2013-10-03 11:26:43 -0700 | [diff] [blame] | 747 | # Experimental test results do not influence the exit code. |
| 748 | test_report_command.append('--ignore_experimental_tests') |
Aviv Keshet | 6f74e19 | 2013-09-23 11:09:07 -0700 | [diff] [blame] | 749 | if arguments.whitelist_chrome_crashes: |
| 750 | test_report_command.append('--whitelist_chrome_crashes') |
| 751 | test_report_command.append(results_directory) |
| 752 | final_result = subprocess.call(test_report_command) |
| 753 | with open(os.path.join(results_directory, 'test_report.log'), |
| 754 | 'w') as report_log: |
| 755 | subprocess.call(test_report_command, stdout=report_log) |
Aviv Keshet | e10e8d0 | 2013-09-09 17:04:58 -0700 | [diff] [blame] | 756 | try: |
Aviv Keshet | 6f74e19 | 2013-09-23 11:09:07 -0700 | [diff] [blame] | 757 | os.unlink(_LATEST_RESULTS_DIRECTORY) |
| 758 | except OSError: |
| 759 | pass |
Josh Triplett | 6303341 | 2014-06-27 10:02:08 -0700 | [diff] [blame] | 760 | link_target = os.path.relpath(results_directory, |
| 761 | os.path.dirname(_LATEST_RESULTS_DIRECTORY)) |
| 762 | os.symlink(link_target, _LATEST_RESULTS_DIRECTORY) |
Josh Triplett | 8e55533 | 2014-06-27 10:06:01 -0700 | [diff] [blame^] | 763 | logging.info('Finished running tests. Results can be found in %s or %s', |
| 764 | results_directory, _LATEST_RESULTS_DIRECTORY) |
Aviv Keshet | 6f74e19 | 2013-09-23 11:09:07 -0700 | [diff] [blame] | 765 | return final_result |
| 766 | |
| 767 | |
| 768 | def _main_for_local_run(argv, arguments): |
| 769 | """ |
| 770 | Effective entry point for local test_that runs. |
| 771 | |
| 772 | @param argv: Script command line arguments. |
| 773 | @param arguments: Parsed command line arguments. |
| 774 | """ |
| 775 | if not cros_build_lib.IsInsideChroot(): |
| 776 | print >> sys.stderr, 'For local runs, script must be run inside chroot.' |
Aviv Keshet | e10e8d0 | 2013-09-09 17:04:58 -0700 | [diff] [blame] | 777 | return 1 |
| 778 | |
beeps | 4efdf03 | 2013-09-17 11:27:14 -0700 | [diff] [blame] | 779 | results_directory = _create_results_directory(arguments.results_dir) |
| 780 | _add_ssh_identity(results_directory) |
| 781 | arguments.results_dir = results_directory |
Aviv Keshet | 6c02839 | 2013-07-22 12:57:32 -0700 | [diff] [blame] | 782 | legacy_path = False |
beeps | 4efdf03 | 2013-09-17 11:27:14 -0700 | [diff] [blame] | 783 | |
| 784 | # If the board has not been specified through --board, and is not set in the |
| 785 | # default_board file, determine the board by ssh-ing into the host. Also |
| 786 | # prepend it to argv so we can re-use it when we run test_that from the |
| 787 | # sysroot. |
| 788 | if arguments.board is None: |
| 789 | arguments.board = _get_board_from_host(arguments.remote) |
| 790 | argv = ['--board', arguments.board] + argv |
| 791 | |
Fang Deng | b3ee20b | 2013-09-17 10:34:13 -0700 | [diff] [blame] | 792 | if arguments.autotest_dir: |
| 793 | autotest_path = arguments.autotest_dir |
| 794 | arguments.no_quickmerge = True |
| 795 | else: |
| 796 | sysroot_path = os.path.join('/build', arguments.board, '') |
Aviv Keshet | 6c02839 | 2013-07-22 12:57:32 -0700 | [diff] [blame] | 797 | |
Fang Deng | b3ee20b | 2013-09-17 10:34:13 -0700 | [diff] [blame] | 798 | if not os.path.exists(sysroot_path): |
| 799 | print >> sys.stderr, ('%s does not exist. Have you run ' |
| 800 | 'setup_board?' % sysroot_path) |
| 801 | return 1 |
Aviv Keshet | e10e8d0 | 2013-09-09 17:04:58 -0700 | [diff] [blame] | 802 | |
Aviv Keshet | 6c02839 | 2013-07-22 12:57:32 -0700 | [diff] [blame] | 803 | # For backwards compatibility with builds that pre-date |
| 804 | # https://chromium-review.googlesource.com/#/c/62880/ |
| 805 | # This code can eventually be removed once those builds no longer need |
| 806 | # test_that support. |
Aviv Keshet | 3f6c99a | 2013-10-31 15:01:56 -0700 | [diff] [blame] | 807 | new_path = 'usr/local/build/autotest' |
| 808 | old_path = 'usr/local/autotest' |
| 809 | legacy_path = (os.path.exists(os.path.join(sysroot_path, old_path)) |
| 810 | and not |
| 811 | os.path.exists(os.path.join(sysroot_path, new_path))) |
Aviv Keshet | 6c02839 | 2013-07-22 12:57:32 -0700 | [diff] [blame] | 812 | if legacy_path: |
Aviv Keshet | 3f6c99a | 2013-10-31 15:01:56 -0700 | [diff] [blame] | 813 | path_ending = old_path |
Aviv Keshet | 6c02839 | 2013-07-22 12:57:32 -0700 | [diff] [blame] | 814 | else: |
Aviv Keshet | 3f6c99a | 2013-10-31 15:01:56 -0700 | [diff] [blame] | 815 | path_ending = new_path |
Aviv Keshet | 6c02839 | 2013-07-22 12:57:32 -0700 | [diff] [blame] | 816 | autotest_path = os.path.join(sysroot_path, path_ending) |
| 817 | |
Fang Deng | b3ee20b | 2013-09-17 10:34:13 -0700 | [diff] [blame] | 818 | site_utils_path = os.path.join(autotest_path, 'site_utils') |
| 819 | |
| 820 | if not os.path.exists(autotest_path): |
Aviv Keshet | e10e8d0 | 2013-09-09 17:04:58 -0700 | [diff] [blame] | 821 | print >> sys.stderr, ('%s does not exist. Have you run ' |
Fang Deng | b3ee20b | 2013-09-17 10:34:13 -0700 | [diff] [blame] | 822 | 'build_packages? Or if you are using ' |
Jorge Lucangeli Obes | 89dc43d | 2014-05-29 14:44:04 -0700 | [diff] [blame] | 823 | '--autotest-dir, make sure it points to ' |
Fang Deng | b3ee20b | 2013-09-17 10:34:13 -0700 | [diff] [blame] | 824 | 'a valid autotest directory.' % autotest_path) |
Aviv Keshet | e10e8d0 | 2013-09-09 17:04:58 -0700 | [diff] [blame] | 825 | return 1 |
| 826 | |
| 827 | realpath = os.path.realpath(__file__) |
| 828 | |
| 829 | # If we are not running the sysroot version of script, perform |
| 830 | # a quickmerge if necessary and then re-execute |
| 831 | # the sysroot version of script with the same arguments. |
Fang Deng | b3ee20b | 2013-09-17 10:34:13 -0700 | [diff] [blame] | 832 | if os.path.dirname(realpath) != site_utils_path: |
| 833 | return _perform_bootstrap_into_autotest_root( |
Aviv Keshet | 6c02839 | 2013-07-22 12:57:32 -0700 | [diff] [blame] | 834 | arguments, autotest_path, argv, legacy_path) |
Aviv Keshet | e10e8d0 | 2013-09-09 17:04:58 -0700 | [diff] [blame] | 835 | else: |
Fang Deng | b3ee20b | 2013-09-17 10:34:13 -0700 | [diff] [blame] | 836 | return _perform_run_from_autotest_root( |
| 837 | arguments, autotest_path, argv) |
Aviv Keshet | ba4992a | 2013-07-02 14:09:23 -0700 | [diff] [blame] | 838 | |
Aviv Keshet | 021c19f | 2013-02-22 13:19:43 -0800 | [diff] [blame] | 839 | |
Aviv Keshet | 6f74e19 | 2013-09-23 11:09:07 -0700 | [diff] [blame] | 840 | def _main_for_lab_run(argv, arguments): |
| 841 | """ |
| 842 | Effective entry point for lab test_that runs. |
| 843 | |
| 844 | @param argv: Script command line arguments. |
| 845 | @param arguments: Parsed command line arguments. |
| 846 | """ |
| 847 | autotest_path = os.path.realpath(os.path.join(os.path.dirname(__file__), |
| 848 | '..')) |
| 849 | flattened_argv = ' '.join([pipes.quote(item) for item in argv]) |
| 850 | command = [os.path.join(autotest_path, 'site_utils', |
| 851 | 'run_suite.py'), |
| 852 | '--board', arguments.board, |
| 853 | '--build', arguments.build, |
| 854 | '--suite_name', 'test_that_wrapper', |
| 855 | '--pool', 'try-bot', |
| 856 | '--suite_args', flattened_argv] |
| 857 | logging.info('About to start lab suite with command %s.', command) |
| 858 | return subprocess.call(command) |
| 859 | |
| 860 | |
| 861 | def main(argv): |
| 862 | """ |
| 863 | Entry point for test_that script. |
| 864 | |
| 865 | @param argv: arguments list |
| 866 | """ |
| 867 | arguments = parse_arguments(argv) |
| 868 | try: |
| 869 | validate_arguments(arguments) |
| 870 | except ValueError as err: |
| 871 | print >> sys.stderr, ('Invalid arguments. %s' % err.message) |
| 872 | return 1 |
| 873 | |
| 874 | if arguments.remote == ':lab:': |
| 875 | return _main_for_lab_run(argv, arguments) |
| 876 | else: |
| 877 | return _main_for_local_run(argv, arguments) |
| 878 | |
| 879 | |
Aviv Keshet | 021c19f | 2013-02-22 13:19:43 -0800 | [diff] [blame] | 880 | if __name__ == '__main__': |
Aviv Keshet | d4a0430 | 2013-04-30 15:48:30 -0700 | [diff] [blame] | 881 | sys.exit(main(sys.argv[1:])) |