mbligh | 6203ace | 2007-10-04 21:54:24 +0000 | [diff] [blame] | 1 | #!/usr/bin/python -u |
mbligh | 1ffd5dc | 2008-11-25 13:24:05 +0000 | [diff] [blame] | 2 | # Copyright 2007-2008 Martin J. Bligh <mbligh@google.com>, Google Inc. |
mbligh | 82648e5 | 2008-11-20 16:54:25 +0000 | [diff] [blame] | 3 | # Released under the GPL v2 |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 4 | |
mbligh | c8949b8 | 2007-07-23 16:33:58 +0000 | [diff] [blame] | 5 | """ |
Aviv Keshet | de6bb19 | 2013-01-30 16:17:22 -0800 | [diff] [blame] | 6 | Run a control file through the server side engine |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 7 | """ |
mbligh | 1ffd5dc | 2008-11-25 13:24:05 +0000 | [diff] [blame] | 8 | |
Fang Deng | 042c147 | 2014-10-23 13:56:41 -0700 | [diff] [blame] | 9 | import datetime |
Paul Hobbs | 20cc72a | 2016-08-30 16:57:05 -0700 | [diff] [blame] | 10 | import contextlib |
Fang Deng | 042c147 | 2014-10-23 13:56:41 -0700 | [diff] [blame] | 11 | import getpass |
| 12 | import logging |
| 13 | import os |
| 14 | import re |
Prathmesh Prabhu | 4604736 | 2018-03-16 10:33:19 -0700 | [diff] [blame] | 15 | import shutil |
Fang Deng | 042c147 | 2014-10-23 13:56:41 -0700 | [diff] [blame] | 16 | import signal |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 17 | import socket |
Fang Deng | 042c147 | 2014-10-23 13:56:41 -0700 | [diff] [blame] | 18 | import sys |
| 19 | import traceback |
| 20 | import time |
| 21 | import urllib2 |
mbligh | 1ffd5dc | 2008-11-25 13:24:05 +0000 | [diff] [blame] | 22 | |
mbligh | f5427bb | 2008-04-09 15:55:57 +0000 | [diff] [blame] | 23 | import common |
Dan Shi | 4f8c024 | 2017-07-07 15:34:49 -0700 | [diff] [blame] | 24 | from autotest_lib.client.bin.result_tools import utils as result_utils |
| 25 | from autotest_lib.client.bin.result_tools import view as result_view |
Dan Shi | a1ecd5c | 2013-06-06 11:21:31 -0700 | [diff] [blame] | 26 | from autotest_lib.client.common_lib import control_data |
Prathmesh Prabhu | 9a63108 | 2018-05-11 17:30:09 -0700 | [diff] [blame] | 27 | from autotest_lib.client.common_lib import enum |
Dan Shi | 32649b8 | 2015-08-29 20:53:36 -0700 | [diff] [blame] | 28 | from autotest_lib.client.common_lib import error |
Dan Shi | a1ecd5c | 2013-06-06 11:21:31 -0700 | [diff] [blame] | 29 | from autotest_lib.client.common_lib import global_config |
Prathmesh Prabhu | 9a63108 | 2018-05-11 17:30:09 -0700 | [diff] [blame] | 30 | from autotest_lib.client.common_lib import host_queue_entry_states |
| 31 | from autotest_lib.client.common_lib import host_states |
Allen Li | f146e87 | 2017-08-15 18:24:31 -0700 | [diff] [blame] | 32 | from autotest_lib.server import results_mocker |
Prathmesh Prabhu | 4604736 | 2018-03-16 10:33:19 -0700 | [diff] [blame] | 33 | from autotest_lib.server.cros.dynamic_suite import suite |
Prathmesh Prabhu | a5eecda | 2016-11-23 16:48:40 -0800 | [diff] [blame] | 34 | |
Dan Shi | 5e2efb7 | 2017-02-07 11:40:23 -0800 | [diff] [blame] | 35 | try: |
| 36 | from chromite.lib import metrics |
Paul Hobbs | e9fd557 | 2017-08-22 02:48:25 -0700 | [diff] [blame] | 37 | from chromite.lib import cloud_trace |
Dan Shi | 5e2efb7 | 2017-02-07 11:40:23 -0800 | [diff] [blame] | 38 | except ImportError: |
Prathmesh Prabhu | d16c801 | 2017-08-28 11:42:46 -0700 | [diff] [blame] | 39 | from autotest_lib.client.common_lib import utils as common_utils |
| 40 | metrics = common_utils.metrics_mock |
Paul Hobbs | e9fd557 | 2017-08-22 02:48:25 -0700 | [diff] [blame] | 41 | import mock |
| 42 | cloud_trace = mock.MagicMock() |
Prathmesh Prabhu | a5eecda | 2016-11-23 16:48:40 -0800 | [diff] [blame] | 43 | |
Dan Shi | a06f3e2 | 2015-09-03 16:15:15 -0700 | [diff] [blame] | 44 | _CONFIG = global_config.global_config |
| 45 | |
Jakob Juelich | e497b55 | 2014-09-23 19:11:59 -0700 | [diff] [blame] | 46 | # Number of seconds to wait before returning if testing mode is enabled |
Prashanth B | 6285f6a | 2014-05-08 18:01:27 -0700 | [diff] [blame] | 47 | TESTING_MODE_SLEEP_SECS = 1 |
Jakob Juelich | e497b55 | 2014-09-23 19:11:59 -0700 | [diff] [blame] | 48 | |
mbligh | 9ff89cd | 2009-09-03 20:28:17 +0000 | [diff] [blame] | 49 | |
Kevin Cheng | 9b6930f | 2016-07-20 14:57:15 -0700 | [diff] [blame] | 50 | from autotest_lib.server import frontend |
showard | 75cdfee | 2009-06-10 17:40:41 +0000 | [diff] [blame] | 51 | from autotest_lib.server import server_logging_config |
showard | 043c62a | 2009-06-10 19:48:57 +0000 | [diff] [blame] | 52 | from autotest_lib.server import server_job, utils, autoserv_parser, autotest |
Dan Shi | a1ecd5c | 2013-06-06 11:21:31 -0700 | [diff] [blame] | 53 | from autotest_lib.server import utils as server_utils |
Paul Hobbs | 20cc72a | 2016-08-30 16:57:05 -0700 | [diff] [blame] | 54 | from autotest_lib.server import site_utils |
Kevin Cheng | adc99f9 | 2016-07-20 08:21:58 -0700 | [diff] [blame] | 55 | from autotest_lib.server.cros.dynamic_suite import frontend_wrappers |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 56 | from autotest_lib.site_utils import job_directories |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 57 | from autotest_lib.site_utils import lxc |
Ben Kwa | 966db08 | 2017-06-05 14:17:23 -0700 | [diff] [blame] | 58 | from autotest_lib.site_utils.lxc import utils as lxc_utils |
showard | 75cdfee | 2009-06-10 17:40:41 +0000 | [diff] [blame] | 59 | from autotest_lib.client.common_lib import pidfile, logging_manager |
mbligh | 92c0fc2 | 2008-11-20 16:52:23 +0000 | [diff] [blame] | 60 | |
Paul Hobbs | 20cc72a | 2016-08-30 16:57:05 -0700 | [diff] [blame] | 61 | |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 62 | # Control segment to stage server-side package. |
| 63 | STAGE_SERVER_SIDE_PACKAGE_CONTROL_FILE = server_job._control_segment_path( |
| 64 | 'stage_server_side_package') |
| 65 | |
Dan Shi | a06f3e2 | 2015-09-03 16:15:15 -0700 | [diff] [blame] | 66 | # Command line to start servod in a moblab. |
| 67 | START_SERVOD_CMD = 'sudo start servod BOARD=%s PORT=%s' |
| 68 | STOP_SERVOD_CMD = 'sudo stop servod' |
| 69 | |
Prathmesh Prabhu | 4604736 | 2018-03-16 10:33:19 -0700 | [diff] [blame] | 70 | _AUTOTEST_ROOT = os.path.realpath(os.path.join(os.path.dirname(__file__), '..')) |
| 71 | _CONTROL_FILE_FROM_CONTROL_NAME = 'control.from_control_name' |
| 72 | |
Prathmesh Prabhu | 7b05ecc | 2018-08-13 14:44:14 -0700 | [diff] [blame] | 73 | _LXC_JOB_FOLDER = 'lxc_job_folder' |
| 74 | |
Alex Miller | f1af17e | 2013-01-09 22:50:32 -0800 | [diff] [blame] | 75 | def log_alarm(signum, frame): |
| 76 | logging.error("Received SIGALARM. Ignoring and continuing on.") |
Alex Miller | 0528d6f | 2013-01-11 10:49:48 -0800 | [diff] [blame] | 77 | sys.exit(1) |
Alex Miller | f1af17e | 2013-01-09 22:50:32 -0800 | [diff] [blame] | 78 | |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 79 | |
| 80 | def _get_machines(parser): |
| 81 | """Get a list of machine names from command line arg -m or a file. |
| 82 | |
| 83 | @param parser: Parser for the command line arguments. |
| 84 | |
| 85 | @return: A list of machine names from command line arg -m or the |
| 86 | machines file specified in the command line arg -M. |
| 87 | """ |
| 88 | if parser.options.machines: |
| 89 | machines = parser.options.machines.replace(',', ' ').strip().split() |
| 90 | else: |
| 91 | machines = [] |
| 92 | machines_file = parser.options.machines_file |
| 93 | if machines_file: |
| 94 | machines = [] |
| 95 | for m in open(machines_file, 'r').readlines(): |
| 96 | # remove comments, spaces |
| 97 | m = re.sub('#.*', '', m).strip() |
| 98 | if m: |
| 99 | machines.append(m) |
| 100 | logging.debug('Read list of machines from file: %s', machines_file) |
| 101 | logging.debug('Machines: %s', ','.join(machines)) |
| 102 | |
| 103 | if machines: |
| 104 | for machine in machines: |
| 105 | if not machine or re.search('\s', machine): |
| 106 | parser.parser.error("Invalid machine: %s" % str(machine)) |
| 107 | machines = list(set(machines)) |
| 108 | machines.sort() |
| 109 | return machines |
| 110 | |
| 111 | |
Prathmesh Prabhu | 588007d | 2017-06-15 00:31:31 -0700 | [diff] [blame] | 112 | def _stage_ssp(parser, resultsdir): |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 113 | """Stage server-side package. |
| 114 | |
| 115 | This function calls a control segment to stage server-side package based on |
| 116 | the job and autoserv command line option. The detail implementation could |
| 117 | be different for each host type. Currently, only CrosHost has |
| 118 | stage_server_side_package function defined. |
| 119 | The script returns None if no server-side package is available. However, |
| 120 | it may raise exception if it failed for reasons other than artifact (the |
| 121 | server-side package) not found. |
| 122 | |
| 123 | @param parser: Command line arguments parser passed in the autoserv process. |
Prathmesh Prabhu | 588007d | 2017-06-15 00:31:31 -0700 | [diff] [blame] | 124 | @param resultsdir: Folder to store results. This could be different from |
| 125 | parser.options.results: parser.options.results can be set to None |
| 126 | for results to be stored in a temp folder. resultsdir can be None |
| 127 | for autoserv run requires no logging. |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 128 | |
Prathmesh Prabhu | 80f9cb8 | 2018-08-11 18:32:08 -0700 | [diff] [blame] | 129 | @return: url to the autotest server-side package. None in case of errors. |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 130 | """ |
Kevin Cheng | adc99f9 | 2016-07-20 08:21:58 -0700 | [diff] [blame] | 131 | machines_list = _get_machines(parser) |
Prathmesh Prabhu | 7fc39c5 | 2018-03-21 14:08:30 -0700 | [diff] [blame] | 132 | machines_list = server_job.get_machine_dicts( |
| 133 | machine_names=machines_list, |
| 134 | store_dir=os.path.join(resultsdir, parser.options.host_info_subdir), |
| 135 | in_lab=parser.options.lab, |
| 136 | use_shadow_store=not parser.options.local_only_host_info, |
| 137 | host_attributes=parser.options.host_attributes, |
| 138 | ) |
Kevin Cheng | adc99f9 | 2016-07-20 08:21:58 -0700 | [diff] [blame] | 139 | |
Kevin Cheng | adc99f9 | 2016-07-20 08:21:58 -0700 | [diff] [blame] | 140 | namespace = {'machines': machines_list, |
Jacob Kopczynski | 9a3a0c3 | 2018-07-18 18:36:20 -0700 | [diff] [blame] | 141 | 'isolate_hash': parser.options.isolate, |
Richard Barnette | 71854c7 | 2018-03-30 14:22:09 -0700 | [diff] [blame] | 142 | 'image': parser.options.test_source_build} |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 143 | script_locals = {} |
| 144 | execfile(STAGE_SERVER_SIDE_PACKAGE_CONTROL_FILE, namespace, script_locals) |
Prathmesh Prabhu | 80f9cb8 | 2018-08-11 18:32:08 -0700 | [diff] [blame] | 145 | ssp_url = script_locals['ssp_url'] |
| 146 | if not ssp_url: |
| 147 | logging.error('Failed to stage SSP package: %s', |
| 148 | script_locals['error_msg']) |
| 149 | logging.error('This job will fail later, when attempting to run with' |
| 150 | ' SSP') |
| 151 | return ssp_url |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 152 | |
| 153 | |
Ben Kwa | bedacad | 2017-08-28 12:20:38 -0700 | [diff] [blame] | 154 | def _run_with_ssp(job, container_id, job_id, results, parser, ssp_url, |
Prathmesh Prabhu | 7b05ecc | 2018-08-13 14:44:14 -0700 | [diff] [blame] | 155 | machines): |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 156 | """Run the server job with server-side packaging. |
| 157 | |
Dan Shi | 37befda | 2015-12-07 13:16:56 -0800 | [diff] [blame] | 158 | @param job: The server job object. |
Ben Kwa | bedacad | 2017-08-28 12:20:38 -0700 | [diff] [blame] | 159 | @param container_id: ID of the container to run the test. |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 160 | @param job_id: ID of the test job. |
| 161 | @param results: Folder to store results. This could be different from |
| 162 | parser.options.results: |
| 163 | parser.options.results can be set to None for results to be |
| 164 | stored in a temp folder. |
| 165 | results can be None for autoserv run requires no logging. |
| 166 | @param parser: Command line parser that contains the options. |
| 167 | @param ssp_url: url of the staged server-side package. |
Dan Shi | 3be35af | 2016-08-25 23:22:40 -0700 | [diff] [blame] | 168 | @param machines: A list of machines to run the test. |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 169 | """ |
Prathmesh Prabhu | 80f9cb8 | 2018-08-11 18:32:08 -0700 | [diff] [blame] | 170 | if not ssp_url: |
| 171 | job.record('FAIL', None, None, |
| 172 | 'Failed to stage server-side package') |
| 173 | raise error.AutoservError('Failed to stage server-side package') |
| 174 | |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 175 | bucket = lxc.ContainerBucket() |
| 176 | control = (parser.args[0] if len(parser.args) > 0 and parser.args[0] != '' |
| 177 | else None) |
Dan Shi | 37befda | 2015-12-07 13:16:56 -0800 | [diff] [blame] | 178 | try: |
Dan Shi | 3be35af | 2016-08-25 23:22:40 -0700 | [diff] [blame] | 179 | dut_name = machines[0] if len(machines) >= 1 else None |
Ben Kwa | bedacad | 2017-08-28 12:20:38 -0700 | [diff] [blame] | 180 | test_container = bucket.setup_test(container_id, job_id, ssp_url, |
Dan Shi | afa6387 | 2016-02-23 15:32:31 -0800 | [diff] [blame] | 181 | results, control=control, |
Prathmesh Prabhu | 7b05ecc | 2018-08-13 14:44:14 -0700 | [diff] [blame] | 182 | job_folder=_LXC_JOB_FOLDER, |
Jacob Kopczynski | 22bb596 | 2018-07-24 11:56:02 -0700 | [diff] [blame] | 183 | dut_name=dut_name, |
| 184 | isolate_hash=parser.options.isolate) |
Dan Shi | 37befda | 2015-12-07 13:16:56 -0800 | [diff] [blame] | 185 | except Exception as e: |
| 186 | job.record('FAIL', None, None, |
| 187 | 'Failed to setup container for test: %s. Check logs in ' |
| 188 | 'ssp_logs folder for more details.' % e) |
| 189 | raise |
| 190 | |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 191 | args = sys.argv[:] |
| 192 | args.remove('--require-ssp') |
Dan Shi | 77b79a6 | 2015-07-29 16:22:05 -0700 | [diff] [blame] | 193 | # --parent_job_id is only useful in autoserv running in host, not in |
| 194 | # container. Include this argument will cause test to fail for builds before |
| 195 | # CL 286265 was merged. |
| 196 | if '--parent_job_id' in args: |
| 197 | index = args.index('--parent_job_id') |
| 198 | args.remove('--parent_job_id') |
| 199 | # Remove the actual parent job id in command line arg. |
| 200 | del args[index] |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 201 | |
| 202 | # A dictionary of paths to replace in the command line. Key is the path to |
| 203 | # be replaced with the one in value. |
| 204 | paths_to_replace = {} |
| 205 | # Replace the control file path with the one in container. |
| 206 | if control: |
| 207 | container_control_filename = os.path.join( |
| 208 | lxc.CONTROL_TEMP_PATH, os.path.basename(control)) |
| 209 | paths_to_replace[control] = container_control_filename |
| 210 | # Update result directory with the one in container. |
Prathmesh Prabhu | 7b05ecc | 2018-08-13 14:44:14 -0700 | [diff] [blame] | 211 | container_result_dir = os.path.join(lxc.RESULT_DIR_FMT % _LXC_JOB_FOLDER) |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 212 | if parser.options.results: |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 213 | paths_to_replace[parser.options.results] = container_result_dir |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 214 | args = [paths_to_replace.get(arg, arg) for arg in args] |
| 215 | |
| 216 | # Apply --use-existing-results, results directory is aready created and |
| 217 | # mounted in container. Apply this arg to avoid exception being raised. |
| 218 | if not '--use-existing-results' in args: |
| 219 | args.append('--use-existing-results') |
| 220 | |
| 221 | # Make sure autoserv running in container using a different pid file. |
| 222 | if not '--pidfile-label' in args: |
| 223 | args.extend(['--pidfile-label', 'container_autoserv']) |
| 224 | |
Dan Shi | d1f5123 | 2015-04-18 00:29:14 -0700 | [diff] [blame] | 225 | cmd_line = ' '.join(["'%s'" % arg if ' ' in arg else arg for arg in args]) |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 226 | logging.info('Run command in container: %s', cmd_line) |
Dan Shi | 37bee22 | 2015-04-13 15:46:47 -0700 | [diff] [blame] | 227 | success = False |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 228 | try: |
| 229 | test_container.attach_run(cmd_line) |
Dan Shi | 37bee22 | 2015-04-13 15:46:47 -0700 | [diff] [blame] | 230 | success = True |
Dan Shi | 9d3454e | 2015-12-08 09:16:08 -0800 | [diff] [blame] | 231 | except Exception as e: |
| 232 | # If the test run inside container fails without generating any log, |
| 233 | # write a message to status.log to help troubleshooting. |
| 234 | debug_files = os.listdir(os.path.join(results, 'debug')) |
| 235 | if not debug_files: |
| 236 | job.record('FAIL', None, None, |
| 237 | 'Failed to run test inside the container: %s. Check ' |
| 238 | 'logs in ssp_logs folder for more details.' % e) |
| 239 | raise |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 240 | finally: |
Prathmesh Prabhu | a5eecda | 2016-11-23 16:48:40 -0800 | [diff] [blame] | 241 | metrics.Counter( |
| 242 | 'chromeos/autotest/experimental/execute_job_in_ssp').increment( |
| 243 | fields={'success': success}) |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 244 | test_container.destroy() |
| 245 | |
| 246 | |
Dan Shi | 3f1b8a5 | 2015-04-21 11:11:06 -0700 | [diff] [blame] | 247 | def correct_results_folder_permission(results): |
| 248 | """Make sure the results folder has the right permission settings. |
| 249 | |
| 250 | For tests running with server-side packaging, the results folder has the |
| 251 | owner of root. This must be changed to the user running the autoserv |
| 252 | process, so parsing job can access the results folder. |
| 253 | TODO(dshi): crbug.com/459344 Remove this function when test container can be |
| 254 | unprivileged container. |
| 255 | |
| 256 | @param results: Path to the results folder. |
| 257 | |
| 258 | """ |
| 259 | if not results: |
| 260 | return |
| 261 | |
Aviv Keshet | c03de79 | 2017-07-18 14:24:31 -0700 | [diff] [blame] | 262 | utils.run('sudo -n chown -R %s "%s"' % (os.getuid(), results)) |
| 263 | utils.run('sudo -n chgrp -R %s "%s"' % (os.getgid(), results)) |
Dan Shi | 3f1b8a5 | 2015-04-21 11:11:06 -0700 | [diff] [blame] | 264 | |
| 265 | |
Dan Shi | a06f3e2 | 2015-09-03 16:15:15 -0700 | [diff] [blame] | 266 | def _start_servod(machine): |
| 267 | """Try to start servod in moblab if it's not already running or running with |
| 268 | different board or port. |
| 269 | |
| 270 | @param machine: Name of the dut used for test. |
| 271 | """ |
| 272 | if not utils.is_moblab(): |
| 273 | return |
| 274 | |
Dan Shi | 1cded88 | 2015-09-23 16:52:26 -0700 | [diff] [blame] | 275 | logging.debug('Trying to start servod.') |
Dan Shi | a06f3e2 | 2015-09-03 16:15:15 -0700 | [diff] [blame] | 276 | try: |
Kevin Cheng | 9b6930f | 2016-07-20 14:57:15 -0700 | [diff] [blame] | 277 | afe = frontend.AFE() |
Dan Shi | a06f3e2 | 2015-09-03 16:15:15 -0700 | [diff] [blame] | 278 | board = server_utils.get_board_from_afe(machine, afe) |
| 279 | hosts = afe.get_hosts(hostname=machine) |
| 280 | servo_host = hosts[0].attributes.get('servo_host', None) |
| 281 | servo_port = hosts[0].attributes.get('servo_port', 9999) |
| 282 | if not servo_host in ['localhost', '127.0.0.1']: |
Dan Shi | 1cded88 | 2015-09-23 16:52:26 -0700 | [diff] [blame] | 283 | logging.warn('Starting servod is aborted. The dut\'s servo_host ' |
| 284 | 'attribute is not set to localhost.') |
Dan Shi | a06f3e2 | 2015-09-03 16:15:15 -0700 | [diff] [blame] | 285 | return |
| 286 | except (urllib2.HTTPError, urllib2.URLError): |
| 287 | # Ignore error if RPC failed to get board |
| 288 | logging.error('Failed to get board name from AFE. Start servod is ' |
| 289 | 'aborted') |
| 290 | return |
| 291 | |
| 292 | try: |
| 293 | pid = utils.run('pgrep servod').stdout |
| 294 | cmd_line = utils.run('ps -fp %s' % pid).stdout |
| 295 | if ('--board %s' % board in cmd_line and |
| 296 | '--port %s' % servo_port in cmd_line): |
| 297 | logging.debug('Servod is already running with given board and port.' |
| 298 | ' There is no need to restart servod.') |
| 299 | return |
| 300 | logging.debug('Servod is running with different board or port. ' |
| 301 | 'Stopping existing servod.') |
| 302 | utils.run('sudo stop servod') |
| 303 | except error.CmdError: |
| 304 | # servod is not running. |
| 305 | pass |
| 306 | |
| 307 | try: |
| 308 | utils.run(START_SERVOD_CMD % (board, servo_port)) |
| 309 | logging.debug('Servod is started') |
| 310 | except error.CmdError as e: |
| 311 | logging.error('Servod failed to be started, error: %s', e) |
| 312 | |
| 313 | |
Prathmesh Prabhu | 4604736 | 2018-03-16 10:33:19 -0700 | [diff] [blame] | 314 | def _control_path_on_disk(control_name): |
| 315 | """Find the control file corresponding to the given control name, on disk. |
| 316 | |
| 317 | @param control_name: NAME attribute of the control file to fetch. |
| 318 | @return: Path to the control file. |
| 319 | """ |
| 320 | cf_getter = suite.create_fs_getter(_AUTOTEST_ROOT) |
| 321 | control_name_predicate = suite.test_name_matches_pattern_predicate( |
| 322 | '^%s$' % control_name) |
| 323 | tests = suite.find_and_parse_tests(cf_getter, control_name_predicate) |
| 324 | if not tests: |
| 325 | raise error.AutoservError( |
| 326 | 'Failed to find any control files with NAME %s' % control_name) |
| 327 | if len(tests) > 1: |
| 328 | logging.error('Found more than one control file with NAME %s: %s', |
| 329 | control_name, [t.path for t in tests]) |
| 330 | raise error.AutoservError( |
| 331 | 'Found more than one control file with NAME %s' % control_name) |
| 332 | return tests[0].path |
| 333 | |
| 334 | |
Prathmesh Prabhu | 492ef46 | 2018-08-11 17:10:33 -0700 | [diff] [blame] | 335 | def _stage_control_file(control_name, results_dir): |
| 336 | """Stage the control file to execute from local autotest checkout. |
Prathmesh Prabhu | 4604736 | 2018-03-16 10:33:19 -0700 | [diff] [blame] | 337 | |
Prathmesh Prabhu | 492ef46 | 2018-08-11 17:10:33 -0700 | [diff] [blame] | 338 | @param control_name: Name of the control file to stage. |
Prathmesh Prabhu | 4604736 | 2018-03-16 10:33:19 -0700 | [diff] [blame] | 339 | @param results_dir: Results directory to stage the control file into. |
| 340 | @return: Absolute path to the staged control file. |
| 341 | """ |
Prathmesh Prabhu | 492ef46 | 2018-08-11 17:10:33 -0700 | [diff] [blame] | 342 | control_path = _control_path_on_disk(control_name) |
Prathmesh Prabhu | 4604736 | 2018-03-16 10:33:19 -0700 | [diff] [blame] | 343 | new_control = os.path.join(results_dir, _CONTROL_FILE_FROM_CONTROL_NAME) |
| 344 | shutil.copy2(control_path, new_control) |
| 345 | return new_control |
| 346 | |
| 347 | |
Dan Shi | c68fefb | 2015-04-07 10:10:52 -0700 | [diff] [blame] | 348 | def run_autoserv(pid_file_manager, results, parser, ssp_url, use_ssp): |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 349 | """Run server job with given options. |
| 350 | |
| 351 | @param pid_file_manager: PidFileManager used to monitor the autoserv process |
| 352 | @param results: Folder to store results. |
| 353 | @param parser: Parser for the command line arguments. |
| 354 | @param ssp_url: Url to server-side package. |
Dan Shi | c68fefb | 2015-04-07 10:10:52 -0700 | [diff] [blame] | 355 | @param use_ssp: Set to True to run with server-side packaging. |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 356 | """ |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 357 | # send stdin to /dev/null |
| 358 | dev_null = os.open(os.devnull, os.O_RDONLY) |
| 359 | os.dup2(dev_null, sys.stdin.fileno()) |
| 360 | os.close(dev_null) |
mbligh | dbf3761 | 2007-11-24 19:38:11 +0000 | [diff] [blame] | 361 | |
Dan Shi | e8aeb66 | 2016-06-30 11:22:03 -0700 | [diff] [blame] | 362 | # Create separate process group if the process is not a process group |
| 363 | # leader. This allows autoserv process to keep running after the caller |
| 364 | # process (drone manager call) exits. |
| 365 | if os.getpid() != os.getpgid(0): |
| 366 | os.setsid() |
mbligh | 1d42d4e | 2007-11-05 22:42:00 +0000 | [diff] [blame] | 367 | |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 368 | # Container name is predefined so the container can be destroyed in |
| 369 | # handle_sigterm. |
| 370 | job_or_task_id = job_directories.get_job_id_or_task_id( |
| 371 | parser.options.results) |
Ben Kwa | bedacad | 2017-08-28 12:20:38 -0700 | [diff] [blame] | 372 | container_id = lxc.ContainerId(job_or_task_id, time.time(), os.getpid()) |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 373 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 374 | # Implement SIGTERM handler |
mbligh | c229956 | 2009-07-02 19:00:36 +0000 | [diff] [blame] | 375 | def handle_sigterm(signum, frame): |
Simran Basi | 9d9b729 | 2013-10-16 16:44:07 -0700 | [diff] [blame] | 376 | logging.debug('Received SIGTERM') |
mbligh | ff7d61f | 2008-12-22 14:53:35 +0000 | [diff] [blame] | 377 | if pid_file_manager: |
| 378 | pid_file_manager.close_file(1, signal.SIGTERM) |
Simran Basi | 49e21e6 | 2013-10-17 12:40:33 -0700 | [diff] [blame] | 379 | logging.debug('Finished writing to pid_file. Killing process.') |
Dan Shi | 3f1b8a5 | 2015-04-21 11:11:06 -0700 | [diff] [blame] | 380 | |
| 381 | # Update results folder's file permission. This needs to be done ASAP |
| 382 | # before the parsing process tries to access the log. |
| 383 | if use_ssp and results: |
| 384 | correct_results_folder_permission(results) |
| 385 | |
Simran Basi | d6b8377 | 2014-01-06 16:31:30 -0800 | [diff] [blame] | 386 | # This sleep allows the pending output to be logged before the kill |
| 387 | # signal is sent. |
| 388 | time.sleep(.1) |
Dan Shi | c68fefb | 2015-04-07 10:10:52 -0700 | [diff] [blame] | 389 | if use_ssp: |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 390 | logging.debug('Destroy container %s before aborting the autoserv ' |
Ben Kwa | bedacad | 2017-08-28 12:20:38 -0700 | [diff] [blame] | 391 | 'process.', container_id) |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 392 | try: |
| 393 | bucket = lxc.ContainerBucket() |
Ben Kwa | bedacad | 2017-08-28 12:20:38 -0700 | [diff] [blame] | 394 | container = bucket.get_container(container_id) |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 395 | if container: |
| 396 | container.destroy() |
Jacob Kopczynski | b83de7a | 2019-05-13 15:37:10 -0700 | [diff] [blame^] | 397 | logging.debug("Container %s destroyed.", container_id) |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 398 | else: |
Ben Kwa | bedacad | 2017-08-28 12:20:38 -0700 | [diff] [blame] | 399 | logging.debug('Container %s is not found.', container_id) |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 400 | except: |
| 401 | # Handle any exception so the autoserv process can be aborted. |
Dan Shi | 65374e2 | 2016-09-15 16:14:05 -0700 | [diff] [blame] | 402 | logging.exception('Failed to destroy container %s.', |
Ben Kwa | bedacad | 2017-08-28 12:20:38 -0700 | [diff] [blame] | 403 | container_id) |
Dan Shi | e4a4f9f | 2015-07-20 09:00:25 -0700 | [diff] [blame] | 404 | # Try to correct the result file permission again after the |
| 405 | # container is destroyed, as the container might have created some |
| 406 | # new files in the result folder. |
| 407 | if results: |
| 408 | correct_results_folder_permission(results) |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 409 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 410 | os.killpg(os.getpgrp(), signal.SIGKILL) |
mbligh | faf0cd4 | 2007-11-19 16:00:24 +0000 | [diff] [blame] | 411 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 412 | # Set signal handler |
mbligh | c229956 | 2009-07-02 19:00:36 +0000 | [diff] [blame] | 413 | signal.signal(signal.SIGTERM, handle_sigterm) |
mbligh | a46678d | 2008-05-01 20:00:01 +0000 | [diff] [blame] | 414 | |
Simran Basi | d6b8377 | 2014-01-06 16:31:30 -0800 | [diff] [blame] | 415 | # faulthandler is only needed to debug in the Lab and is not avaliable to |
| 416 | # be imported in the chroot as part of VMTest, so Try-Except it. |
| 417 | try: |
| 418 | import faulthandler |
| 419 | faulthandler.register(signal.SIGTERM, all_threads=True, chain=True) |
| 420 | logging.debug('faulthandler registered on SIGTERM.') |
| 421 | except ImportError: |
Christopher Grant | 4beca02 | 2015-06-16 15:14:47 -0400 | [diff] [blame] | 422 | sys.exc_clear() |
Simran Basi | d6b8377 | 2014-01-06 16:31:30 -0800 | [diff] [blame] | 423 | |
David Rochberg | 8a60d1e | 2011-02-01 14:22:07 -0500 | [diff] [blame] | 424 | # Ignore SIGTTOU's generated by output from forked children. |
| 425 | signal.signal(signal.SIGTTOU, signal.SIG_IGN) |
| 426 | |
Alex Miller | f1af17e | 2013-01-09 22:50:32 -0800 | [diff] [blame] | 427 | # If we received a SIGALARM, let's be loud about it. |
| 428 | signal.signal(signal.SIGALRM, log_alarm) |
| 429 | |
mbligh | a5f5e54 | 2009-12-30 16:57:49 +0000 | [diff] [blame] | 430 | # Server side tests that call shell scripts often depend on $USER being set |
| 431 | # but depending on how you launch your autotest scheduler it may not be set. |
| 432 | os.environ['USER'] = getpass.getuser() |
| 433 | |
mbligh | b2bea30 | 2008-07-24 20:25:57 +0000 | [diff] [blame] | 434 | label = parser.options.label |
mbligh | 374f341 | 2009-05-13 21:29:45 +0000 | [diff] [blame] | 435 | group_name = parser.options.group_name |
mbligh | b2bea30 | 2008-07-24 20:25:57 +0000 | [diff] [blame] | 436 | user = parser.options.user |
| 437 | client = parser.options.client |
| 438 | server = parser.options.server |
mbligh | b2bea30 | 2008-07-24 20:25:57 +0000 | [diff] [blame] | 439 | verify = parser.options.verify |
| 440 | repair = parser.options.repair |
showard | 45ae819 | 2008-11-05 19:32:53 +0000 | [diff] [blame] | 441 | cleanup = parser.options.cleanup |
Alex Miller | cb79ba7 | 2013-05-29 14:43:00 -0700 | [diff] [blame] | 442 | provision = parser.options.provision |
Dan Shi | 07e09af | 2013-04-12 09:31:29 -0700 | [diff] [blame] | 443 | reset = parser.options.reset |
Alex Miller | 667b5f2 | 2014-02-28 15:33:39 -0800 | [diff] [blame] | 444 | job_labels = parser.options.job_labels |
mbligh | b2bea30 | 2008-07-24 20:25:57 +0000 | [diff] [blame] | 445 | no_tee = parser.options.no_tee |
mbligh | e7d9c60 | 2009-07-02 19:02:33 +0000 | [diff] [blame] | 446 | execution_tag = parser.options.execution_tag |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 447 | ssh_user = parser.options.ssh_user |
| 448 | ssh_port = parser.options.ssh_port |
| 449 | ssh_pass = parser.options.ssh_pass |
jadmanski | def0c3c | 2009-03-25 20:07:10 +0000 | [diff] [blame] | 450 | collect_crashinfo = parser.options.collect_crashinfo |
mbligh | e0cbc91 | 2010-03-11 18:03:07 +0000 | [diff] [blame] | 451 | control_filename = parser.options.control_filename |
beeps | cb6f1e2 | 2013-06-28 19:14:10 -0700 | [diff] [blame] | 452 | verify_job_repo_url = parser.options.verify_job_repo_url |
Christopher Wiley | f594c5e | 2013-07-03 18:25:30 -0700 | [diff] [blame] | 453 | skip_crash_collection = parser.options.skip_crash_collection |
Aviv Keshet | 18ee314 | 2013-08-12 15:01:51 -0700 | [diff] [blame] | 454 | ssh_verbosity = int(parser.options.ssh_verbosity) |
Fang Deng | 6cc20de | 2013-09-06 15:47:32 -0700 | [diff] [blame] | 455 | ssh_options = parser.options.ssh_options |
Dan Shi | b669cbd | 2013-09-13 11:17:17 -0700 | [diff] [blame] | 456 | no_use_packaging = parser.options.no_use_packaging |
Simran Basi | 1bf60eb | 2015-12-01 16:39:29 -0800 | [diff] [blame] | 457 | in_lab = bool(parser.options.lab) |
mbligh | a46678d | 2008-05-01 20:00:01 +0000 | [diff] [blame] | 458 | |
mbligh | b2bea30 | 2008-07-24 20:25:57 +0000 | [diff] [blame] | 459 | # can't be both a client and a server side test |
| 460 | if client and server: |
Eric Li | 861b2d5 | 2011-02-04 14:50:35 -0800 | [diff] [blame] | 461 | parser.parser.error("Can not specify a test as both server and client!") |
mbligh | b2bea30 | 2008-07-24 20:25:57 +0000 | [diff] [blame] | 462 | |
Alex Miller | cb79ba7 | 2013-05-29 14:43:00 -0700 | [diff] [blame] | 463 | if provision and client: |
| 464 | parser.parser.error("Cannot specify provisioning and client!") |
| 465 | |
| 466 | is_special_task = (verify or repair or cleanup or collect_crashinfo or |
Dan Shi | 07e09af | 2013-04-12 09:31:29 -0700 | [diff] [blame] | 467 | provision or reset) |
Allen Li | 5913d8c | 2018-09-25 17:56:31 -0700 | [diff] [blame] | 468 | use_client_trampoline = False |
Prathmesh Prabhu | 4604736 | 2018-03-16 10:33:19 -0700 | [diff] [blame] | 469 | if parser.options.control_name: |
Prathmesh Prabhu | 412d151 | 2018-08-11 17:26:04 -0700 | [diff] [blame] | 470 | if use_ssp: |
| 471 | # When use_ssp is True, autoserv will be re-executed inside a |
| 472 | # container preserving the --control-name argument. Control file |
| 473 | # will be staged inside the rexecuted autoserv. |
| 474 | control = None |
| 475 | else: |
Allen Li | 5913d8c | 2018-09-25 17:56:31 -0700 | [diff] [blame] | 476 | try: |
| 477 | control = _stage_control_file(parser.options.control_name, |
| 478 | results) |
| 479 | except error.AutoservError as e: |
| 480 | logging.info("Using client trampoline because of: %s", e) |
| 481 | control = parser.options.control_name |
| 482 | use_client_trampoline = True |
| 483 | |
Prathmesh Prabhu | 4604736 | 2018-03-16 10:33:19 -0700 | [diff] [blame] | 484 | elif parser.args: |
| 485 | control = parser.args[0] |
| 486 | else: |
Prathmesh Prabhu | 412d151 | 2018-08-11 17:26:04 -0700 | [diff] [blame] | 487 | if not is_special_task: |
| 488 | parser.parser.error("Missing argument: control file") |
Prathmesh Prabhu | 4604736 | 2018-03-16 10:33:19 -0700 | [diff] [blame] | 489 | control = None |
| 490 | |
Aviv Keshet | 18ee314 | 2013-08-12 15:01:51 -0700 | [diff] [blame] | 491 | if ssh_verbosity > 0: |
| 492 | # ssh_verbosity is an integer between 0 and 3, inclusive |
| 493 | ssh_verbosity_flag = '-' + 'v' * ssh_verbosity |
Fang Deng | d1c2b73 | 2013-08-20 12:59:46 -0700 | [diff] [blame] | 494 | else: |
| 495 | ssh_verbosity_flag = '' |
Aviv Keshet | 18ee314 | 2013-08-12 15:01:51 -0700 | [diff] [blame] | 496 | |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 497 | machines = _get_machines(parser) |
mbligh | 374f341 | 2009-05-13 21:29:45 +0000 | [diff] [blame] | 498 | if group_name and len(machines) < 2: |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 499 | parser.parser.error('-G %r may only be supplied with more than one ' |
| 500 | 'machine.' % group_name) |
mbligh | 374f341 | 2009-05-13 21:29:45 +0000 | [diff] [blame] | 501 | |
Prathmesh Prabhu | 7fc39c5 | 2018-03-21 14:08:30 -0700 | [diff] [blame] | 502 | job_kwargs = { |
| 503 | 'control': control, |
| 504 | 'args': parser.args[1:], |
| 505 | 'resultdir': results, |
| 506 | 'label': label, |
| 507 | 'user': user, |
| 508 | 'machines': machines, |
| 509 | 'machine_dict_list': server_job.get_machine_dicts( |
| 510 | machine_names=machines, |
| 511 | store_dir=os.path.join(results, |
| 512 | parser.options.host_info_subdir), |
| 513 | in_lab=in_lab, |
| 514 | use_shadow_store=not parser.options.local_only_host_info, |
| 515 | host_attributes=parser.options.host_attributes, |
| 516 | ), |
| 517 | 'client': client, |
Prathmesh Prabhu | 7fc39c5 | 2018-03-21 14:08:30 -0700 | [diff] [blame] | 518 | 'ssh_user': ssh_user, |
| 519 | 'ssh_port': ssh_port, |
| 520 | 'ssh_pass': ssh_pass, |
| 521 | 'ssh_verbosity_flag': ssh_verbosity_flag, |
| 522 | 'ssh_options': ssh_options, |
Prathmesh Prabhu | 7fc39c5 | 2018-03-21 14:08:30 -0700 | [diff] [blame] | 523 | 'group_name': group_name, |
| 524 | 'tag': execution_tag, |
| 525 | 'disable_sysinfo': parser.options.disable_sysinfo, |
| 526 | 'in_lab': in_lab, |
Allen Li | 5913d8c | 2018-09-25 17:56:31 -0700 | [diff] [blame] | 527 | 'use_client_trampoline': use_client_trampoline, |
Prathmesh Prabhu | 7fc39c5 | 2018-03-21 14:08:30 -0700 | [diff] [blame] | 528 | } |
Dan Shi | 70647ca | 2015-07-16 22:52:35 -0700 | [diff] [blame] | 529 | if parser.options.parent_job_id: |
Prathmesh Prabhu | 7fc39c5 | 2018-03-21 14:08:30 -0700 | [diff] [blame] | 530 | job_kwargs['parent_job_id'] = int(parser.options.parent_job_id) |
mbligh | e0cbc91 | 2010-03-11 18:03:07 +0000 | [diff] [blame] | 531 | if control_filename: |
Prathmesh Prabhu | 7fc39c5 | 2018-03-21 14:08:30 -0700 | [diff] [blame] | 532 | job_kwargs['control_filename'] = control_filename |
| 533 | job = server_job.server_job(**job_kwargs) |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 534 | |
showard | 75cdfee | 2009-06-10 17:40:41 +0000 | [diff] [blame] | 535 | job.logging.start_logging() |
mbligh | a46678d | 2008-05-01 20:00:01 +0000 | [diff] [blame] | 536 | |
mbligh | 161fe6f | 2008-06-19 16:26:04 +0000 | [diff] [blame] | 537 | # perform checks |
| 538 | job.precheck() |
| 539 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 540 | # run the job |
| 541 | exit_code = 0 |
Dan Shi | c1b8bdd | 2015-09-14 23:11:24 -0700 | [diff] [blame] | 542 | auto_start_servod = _CONFIG.get_config_value( |
| 543 | 'AUTOSERV', 'auto_start_servod', type=bool, default=False) |
Paul Hobbs | 20cc72a | 2016-08-30 16:57:05 -0700 | [diff] [blame] | 544 | |
Prathmesh Prabhu | 27bba96 | 2017-01-24 15:13:07 -0800 | [diff] [blame] | 545 | site_utils.SetupTsMonGlobalState('autoserv', indirect=False, |
Aviv Keshet | aece204 | 2018-07-17 16:27:22 -0700 | [diff] [blame] | 546 | short_lived=True) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 547 | try: |
Prathmesh Prabhu | 27bba96 | 2017-01-24 15:13:07 -0800 | [diff] [blame] | 548 | try: |
| 549 | if repair: |
| 550 | if auto_start_servod and len(machines) == 1: |
| 551 | _start_servod(machines[0]) |
| 552 | job.repair(job_labels) |
| 553 | elif verify: |
| 554 | job.verify(job_labels) |
| 555 | elif provision: |
| 556 | job.provision(job_labels) |
| 557 | elif reset: |
| 558 | job.reset(job_labels) |
| 559 | elif cleanup: |
| 560 | job.cleanup(job_labels) |
| 561 | else: |
| 562 | if auto_start_servod and len(machines) == 1: |
| 563 | _start_servod(machines[0]) |
| 564 | if use_ssp: |
| 565 | try: |
Ben Kwa | bedacad | 2017-08-28 12:20:38 -0700 | [diff] [blame] | 566 | _run_with_ssp(job, container_id, job_or_task_id, |
Prathmesh Prabhu | 7b05ecc | 2018-08-13 14:44:14 -0700 | [diff] [blame] | 567 | results, parser, ssp_url, machines) |
Prathmesh Prabhu | 27bba96 | 2017-01-24 15:13:07 -0800 | [diff] [blame] | 568 | finally: |
| 569 | # Update the ownership of files in result folder. |
| 570 | correct_results_folder_permission(results) |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 571 | else: |
Prathmesh Prabhu | 27bba96 | 2017-01-24 15:13:07 -0800 | [diff] [blame] | 572 | if collect_crashinfo: |
| 573 | # Update the ownership of files in result folder. If the |
| 574 | # job to collect crashinfo was running inside container |
| 575 | # (SSP) and crashed before correcting folder permission, |
| 576 | # the result folder might have wrong permission setting. |
Dan Shi | afa6387 | 2016-02-23 15:32:31 -0800 | [diff] [blame] | 577 | try: |
| 578 | correct_results_folder_permission(results) |
Prathmesh Prabhu | 27bba96 | 2017-01-24 15:13:07 -0800 | [diff] [blame] | 579 | except: |
| 580 | # Ignore any error as the user may not have root |
| 581 | # permission to run sudo command. |
| 582 | pass |
Aviv Keshet | 92bf7b6 | 2017-02-13 15:34:03 -0800 | [diff] [blame] | 583 | metric_name = ('chromeos/autotest/experimental/' |
| 584 | 'autoserv_job_run_duration') |
| 585 | f = {'in_container': utils.is_in_container(), |
| 586 | 'success': False} |
| 587 | with metrics.SecondsTimer(metric_name, fields=f) as c: |
Richard Barnette | 71854c7 | 2018-03-30 14:22:09 -0700 | [diff] [blame] | 588 | job.run(verify_job_repo_url=verify_job_repo_url, |
Aviv Keshet | 92bf7b6 | 2017-02-13 15:34:03 -0800 | [diff] [blame] | 589 | only_collect_crashinfo=collect_crashinfo, |
| 590 | skip_crash_collection=skip_crash_collection, |
| 591 | job_labels=job_labels, |
| 592 | use_packaging=(not no_use_packaging)) |
| 593 | c['success'] = True |
| 594 | |
Prathmesh Prabhu | 27bba96 | 2017-01-24 15:13:07 -0800 | [diff] [blame] | 595 | finally: |
Hidehiko Abe | 0689330 | 2017-06-24 07:32:38 +0900 | [diff] [blame] | 596 | job.close() |
Dan Shi | ffd5b82 | 2017-07-14 11:16:23 -0700 | [diff] [blame] | 597 | # Special task doesn't run parse, so result summary needs to be |
| 598 | # built here. |
| 599 | if results and (repair or verify or reset or cleanup or provision): |
Dan Shi | 4f8c024 | 2017-07-07 15:34:49 -0700 | [diff] [blame] | 600 | # Throttle the result on the server side. |
| 601 | try: |
| 602 | result_utils.execute( |
| 603 | results, control_data.DEFAULT_MAX_RESULT_SIZE_KB) |
| 604 | except: |
| 605 | logging.exception( |
| 606 | 'Non-critical failure: Failed to throttle results ' |
| 607 | 'in directory %s.', results) |
| 608 | # Build result view and report metrics for result sizes. |
Dan Shi | ffd5b82 | 2017-07-14 11:16:23 -0700 | [diff] [blame] | 609 | site_utils.collect_result_sizes(results) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 610 | except: |
jadmanski | 27b37ea | 2008-10-29 23:54:31 +0000 | [diff] [blame] | 611 | exit_code = 1 |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 612 | traceback.print_exc() |
Prathmesh Prabhu | 27bba96 | 2017-01-24 15:13:07 -0800 | [diff] [blame] | 613 | finally: |
| 614 | metrics.Flush() |
mbligh | a46678d | 2008-05-01 20:00:01 +0000 | [diff] [blame] | 615 | |
jadmanski | 27b37ea | 2008-10-29 23:54:31 +0000 | [diff] [blame] | 616 | sys.exit(exit_code) |
mbligh | a46678d | 2008-05-01 20:00:01 +0000 | [diff] [blame] | 617 | |
| 618 | |
Prathmesh Prabhu | 9a63108 | 2018-05-11 17:30:09 -0700 | [diff] [blame] | 619 | # Job breakdown statuses |
| 620 | _hs = host_states.Status |
| 621 | _qs = host_queue_entry_states.Status |
| 622 | _status_list = [ |
| 623 | _qs.QUEUED, _qs.RESETTING, _qs.VERIFYING, |
| 624 | _qs.PROVISIONING, _hs.REPAIRING, _qs.CLEANING, |
| 625 | _qs.RUNNING, _qs.GATHERING, _qs.PARSING] |
| 626 | _JOB_OVERHEAD_STATUS = enum.Enum(*_status_list, string_values=True) |
Paul Hobbs | 68d9859 | 2017-08-22 02:22:49 -0700 | [diff] [blame] | 627 | |
| 628 | |
| 629 | def get_job_status(options): |
| 630 | """Returns the HQE Status for this run. |
| 631 | |
| 632 | @param options: parser options. |
| 633 | """ |
Prathmesh Prabhu | 9a63108 | 2018-05-11 17:30:09 -0700 | [diff] [blame] | 634 | s = _JOB_OVERHEAD_STATUS |
Fang Deng | 042c147 | 2014-10-23 13:56:41 -0700 | [diff] [blame] | 635 | task_mapping = { |
| 636 | 'reset': s.RESETTING, 'verify': s.VERIFYING, |
| 637 | 'provision': s.PROVISIONING, 'repair': s.REPAIRING, |
| 638 | 'cleanup': s.CLEANING, 'collect_crashinfo': s.GATHERING} |
Paul Hobbs | 68d9859 | 2017-08-22 02:22:49 -0700 | [diff] [blame] | 639 | match = [task for task in task_mapping if getattr(options, task, False)] |
| 640 | return task_mapping[match[0]] if match else s.RUNNING |
Fang Deng | 042c147 | 2014-10-23 13:56:41 -0700 | [diff] [blame] | 641 | |
| 642 | |
Prathmesh Prabhu | 83b598c | 2018-08-13 17:03:07 -0700 | [diff] [blame] | 643 | def _require_ssp_from_control(control_name): |
| 644 | """Read the value of REQUIRE_SSP from test control file. |
| 645 | |
| 646 | This reads the control file from the prod checkout of autotest and uses that |
| 647 | to determine whether to even stage the SSP package on a devserver. |
| 648 | |
| 649 | This means: |
| 650 | [1] Any change in REQUIRE_SSP directive in a test requires a prod-push to go |
| 651 | live. |
| 652 | [2] This function may find that the control file does not exist but the SSP |
| 653 | package may contain the test file. This function conservatively returns True |
| 654 | in that case. |
| 655 | |
| 656 | This function is called very early in autoserv, before logging is setup. |
| 657 | """ |
| 658 | if not control_name: |
| 659 | return True |
Allen Li | dc1b98d | 2018-09-12 13:56:06 -0700 | [diff] [blame] | 660 | try: |
| 661 | path = _control_path_on_disk(control_name) |
Allen Li | aa24166 | 2018-09-24 13:47:04 -0700 | [diff] [blame] | 662 | except error.AutoservError as e: |
Allen Li | dc1b98d | 2018-09-12 13:56:06 -0700 | [diff] [blame] | 663 | sys.stderr.write("autoserv: Could not determine control file path," |
Allen Li | aa24166 | 2018-09-24 13:47:04 -0700 | [diff] [blame] | 664 | " assuming we need SSP: %s\n" % e) |
Allen Li | dc1b98d | 2018-09-12 13:56:06 -0700 | [diff] [blame] | 665 | sys.stderr.flush() |
| 666 | return True |
Prathmesh Prabhu | 83b598c | 2018-08-13 17:03:07 -0700 | [diff] [blame] | 667 | if not os.path.isfile(path): |
| 668 | return True |
| 669 | control = control_data.parse_control(path) |
| 670 | # There must be explicit directive in the control file to disable SSP. |
| 671 | if not control or control.require_ssp is None: |
| 672 | return True |
| 673 | return control.require_ssp |
| 674 | |
| 675 | |
mbligh | a46678d | 2008-05-01 20:00:01 +0000 | [diff] [blame] | 676 | def main(): |
Fang Deng | 042c147 | 2014-10-23 13:56:41 -0700 | [diff] [blame] | 677 | start_time = datetime.datetime.now() |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 678 | parser = autoserv_parser.autoserv_parser |
mbligh | a5cb406 | 2009-02-17 15:53:39 +0000 | [diff] [blame] | 679 | parser.parse_args() |
mbligh | a46678d | 2008-05-01 20:00:01 +0000 | [diff] [blame] | 680 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 681 | if len(sys.argv) == 1: |
| 682 | parser.parser.print_help() |
| 683 | sys.exit(1) |
mbligh | a6f1308 | 2008-06-05 23:53:46 +0000 | [diff] [blame] | 684 | |
showard | 75cdfee | 2009-06-10 17:40:41 +0000 | [diff] [blame] | 685 | if parser.options.no_logging: |
| 686 | results = None |
| 687 | else: |
| 688 | results = parser.options.results |
mbligh | 80e1eba | 2008-11-19 00:26:18 +0000 | [diff] [blame] | 689 | if not results: |
| 690 | results = 'results.' + time.strftime('%Y-%m-%d-%H.%M.%S') |
Dan Shi | 14de762 | 2016-08-22 11:09:06 -0700 | [diff] [blame] | 691 | results = os.path.abspath(results) |
showard | 566d3c0 | 2010-01-12 18:57:01 +0000 | [diff] [blame] | 692 | resultdir_exists = False |
| 693 | for filename in ('control.srv', 'status.log', '.autoserv_execute'): |
| 694 | if os.path.exists(os.path.join(results, filename)): |
| 695 | resultdir_exists = True |
mbligh | 4608b00 | 2010-01-05 18:22:35 +0000 | [diff] [blame] | 696 | if not parser.options.use_existing_results and resultdir_exists: |
mbligh | 80e1eba | 2008-11-19 00:26:18 +0000 | [diff] [blame] | 697 | error = "Error: results directory already exists: %s\n" % results |
| 698 | sys.stderr.write(error) |
| 699 | sys.exit(1) |
mbligh | a788dc4 | 2009-03-26 21:10:16 +0000 | [diff] [blame] | 700 | |
| 701 | # Now that we certified that there's no leftover results dir from |
| 702 | # previous jobs, lets create the result dir since the logging system |
| 703 | # needs to create the log file in there. |
| 704 | if not os.path.isdir(results): |
| 705 | os.makedirs(results) |
showard | 75cdfee | 2009-06-10 17:40:41 +0000 | [diff] [blame] | 706 | |
Prathmesh Prabhu | 83b598c | 2018-08-13 17:03:07 -0700 | [diff] [blame] | 707 | if parser.options.require_ssp: |
| 708 | # This is currently only used for skylab (i.e., when --control-name is |
| 709 | # used). |
| 710 | use_ssp = _require_ssp_from_control(parser.options.control_name) |
| 711 | else: |
| 712 | use_ssp = False |
| 713 | |
| 714 | |
Dan Shi | c68fefb | 2015-04-07 10:10:52 -0700 | [diff] [blame] | 715 | if use_ssp: |
Dan Shi | e28de55 | 2015-05-06 16:51:58 -0700 | [diff] [blame] | 716 | log_dir = os.path.join(results, 'ssp_logs') if results else None |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 717 | if log_dir and not os.path.exists(log_dir): |
| 718 | os.makedirs(log_dir) |
| 719 | else: |
| 720 | log_dir = results |
Dan Shi | 3f1b8a5 | 2015-04-21 11:11:06 -0700 | [diff] [blame] | 721 | |
showard | 75cdfee | 2009-06-10 17:40:41 +0000 | [diff] [blame] | 722 | logging_manager.configure_logging( |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 723 | server_logging_config.ServerLoggingConfig(), |
| 724 | results_dir=log_dir, |
showard | 10d8417 | 2009-06-18 23:16:50 +0000 | [diff] [blame] | 725 | use_console=not parser.options.no_tee, |
| 726 | verbose=parser.options.verbose, |
| 727 | no_console_prefix=parser.options.no_console_prefix) |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 728 | |
Prathmesh Prabhu | 80f9cb8 | 2018-08-11 18:32:08 -0700 | [diff] [blame] | 729 | logging.debug('autoserv is running in drone %s.', socket.gethostname()) |
| 730 | logging.debug('autoserv command was: %s', ' '.join(sys.argv)) |
| 731 | logging.debug('autoserv parsed options: %s', parser.options) |
| 732 | |
| 733 | if use_ssp: |
| 734 | ssp_url = _stage_ssp(parser, results) |
| 735 | else: |
| 736 | ssp_url = None |
Dan Shi | 0b754c5 | 2015-04-20 14:20:38 -0700 | [diff] [blame] | 737 | |
showard | 75cdfee | 2009-06-10 17:40:41 +0000 | [diff] [blame] | 738 | if results: |
mbligh | a788dc4 | 2009-03-26 21:10:16 +0000 | [diff] [blame] | 739 | logging.info("Results placed in %s" % results) |
mbligh | 1071763 | 2008-11-19 00:21:57 +0000 | [diff] [blame] | 740 | |
mbligh | 4608b00 | 2010-01-05 18:22:35 +0000 | [diff] [blame] | 741 | # wait until now to perform this check, so it get properly logged |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 742 | if (parser.options.use_existing_results and not resultdir_exists and |
Dan Shi | ff78f11 | 2015-06-12 13:34:02 -0700 | [diff] [blame] | 743 | not utils.is_in_container()): |
mbligh | 4608b00 | 2010-01-05 18:22:35 +0000 | [diff] [blame] | 744 | logging.error("No existing results directory found: %s", results) |
| 745 | sys.exit(1) |
| 746 | |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 747 | if parser.options.write_pidfile and results: |
mbligh | 4608b00 | 2010-01-05 18:22:35 +0000 | [diff] [blame] | 748 | pid_file_manager = pidfile.PidFileManager(parser.options.pidfile_label, |
| 749 | results) |
jadmanski | d5ab8c5 | 2008-12-03 16:27:07 +0000 | [diff] [blame] | 750 | pid_file_manager.open_file() |
mbligh | ff7d61f | 2008-12-22 14:53:35 +0000 | [diff] [blame] | 751 | else: |
| 752 | pid_file_manager = None |
mbligh | a46678d | 2008-05-01 20:00:01 +0000 | [diff] [blame] | 753 | |
Allen Li | d5abdab | 2017-02-07 16:03:43 -0800 | [diff] [blame] | 754 | autotest.Autotest.set_install_in_tmpdir( |
jadmanski | f22fea8 | 2008-11-26 20:57:07 +0000 | [diff] [blame] | 755 | parser.options.install_in_tmpdir) |
| 756 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 757 | exit_code = 0 |
Allen Li | f146e87 | 2017-08-15 18:24:31 -0700 | [diff] [blame] | 758 | # TODO(beeps): Extend this to cover different failure modes. |
| 759 | # Testing exceptions are matched against labels sent to autoserv. Eg, |
| 760 | # to allow only the hostless job to run, specify |
| 761 | # testing_exceptions: test_suite in the shadow_config. To allow both |
| 762 | # the hostless job and dummy_Pass to run, specify |
| 763 | # testing_exceptions: test_suite,dummy_Pass. You can figure out |
| 764 | # what label autoserv is invoked with by looking through the logs of a test |
| 765 | # for the autoserv command's -l option. |
| 766 | testing_exceptions = _CONFIG.get_config_value( |
| 767 | 'AUTOSERV', 'testing_exceptions', type=list, default=[]) |
| 768 | test_mode = _CONFIG.get_config_value( |
| 769 | 'AUTOSERV', 'testing_mode', type=bool, default=False) |
| 770 | test_mode = (results_mocker and test_mode and not |
| 771 | any([ex in parser.options.label |
| 772 | for ex in testing_exceptions])) |
| 773 | is_task = (parser.options.verify or parser.options.repair or |
| 774 | parser.options.provision or parser.options.reset or |
| 775 | parser.options.cleanup or parser.options.collect_crashinfo) |
Paul Hobbs | e9fd557 | 2017-08-22 02:48:25 -0700 | [diff] [blame] | 776 | |
| 777 | trace_labels = { |
| 778 | 'job_id': job_directories.get_job_id_or_task_id( |
| 779 | parser.options.results) |
| 780 | } |
| 781 | trace = cloud_trace.SpanStack( |
| 782 | labels=trace_labels, |
| 783 | global_context=parser.options.cloud_trace_context) |
| 784 | trace.enabled = parser.options.cloud_trace_context_enabled == 'True' |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 785 | try: |
| 786 | try: |
Allen Li | f146e87 | 2017-08-15 18:24:31 -0700 | [diff] [blame] | 787 | if test_mode: |
| 788 | # The parser doesn't run on tasks anyway, so we can just return |
| 789 | # happy signals without faking results. |
| 790 | if not is_task: |
| 791 | machine = parser.options.results.split('/')[-1] |
| 792 | |
| 793 | # TODO(beeps): The proper way to do this would be to |
| 794 | # refactor job creation so we can invoke job.record |
| 795 | # directly. To do that one needs to pipe the test_name |
| 796 | # through run_autoserv and bail just before invoking |
| 797 | # the server job. See the comment in |
| 798 | # puppylab/results_mocker for more context. |
| 799 | results_mocker.ResultsMocker( |
| 800 | 'unknown-test', parser.options.results, machine |
| 801 | ).mock_results() |
| 802 | return |
| 803 | else: |
Paul Hobbs | e9fd557 | 2017-08-22 02:48:25 -0700 | [diff] [blame] | 804 | with trace.Span(get_job_status(parser.options)): |
| 805 | run_autoserv(pid_file_manager, results, parser, ssp_url, |
| 806 | use_ssp) |
Aviv Keshet | 5c40ec6 | 2013-08-20 12:11:12 -0700 | [diff] [blame] | 807 | except SystemExit as e: |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 808 | exit_code = e.code |
Aviv Keshet | 5c40ec6 | 2013-08-20 12:11:12 -0700 | [diff] [blame] | 809 | if exit_code: |
Aviv Keshet | 5ae0a00 | 2017-05-05 10:23:33 -0700 | [diff] [blame] | 810 | logging.exception('Uncaught SystemExit with code %s', exit_code) |
| 811 | except Exception: |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 812 | # If we don't know what happened, we'll classify it as |
| 813 | # an 'abort' and return 1. |
Aviv Keshet | 5ae0a00 | 2017-05-05 10:23:33 -0700 | [diff] [blame] | 814 | logging.exception('Uncaught Exception, exit_code = 1.') |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 815 | exit_code = 1 |
| 816 | finally: |
mbligh | ff7d61f | 2008-12-22 14:53:35 +0000 | [diff] [blame] | 817 | if pid_file_manager: |
| 818 | pid_file_manager.close_file(exit_code) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 819 | sys.exit(exit_code) |
mbligh | faf0cd4 | 2007-11-19 16:00:24 +0000 | [diff] [blame] | 820 | |
mbligh | bb42185 | 2008-03-11 22:36:16 +0000 | [diff] [blame] | 821 | |
mbligh | a46678d | 2008-05-01 20:00:01 +0000 | [diff] [blame] | 822 | if __name__ == '__main__': |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 823 | main() |