J. Richard Barnette | 24adbf4 | 2012-04-11 15:04:53 -0700 | [diff] [blame] | 1 | # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. |
Dale Curtis | aa5eedb | 2011-08-23 16:18:52 -0700 | [diff] [blame] | 2 | # Use of this source code is governed by a BSD-style license that can be |
| 3 | # found in the LICENSE file. |
| 4 | |
mussa | 584b446 | 2014-06-20 15:13:28 -0700 | [diff] [blame] | 5 | import ConfigParser |
Aviv Keshet | 74c89a9 | 2013-02-04 15:18:30 -0800 | [diff] [blame] | 6 | import functools |
Christopher Wiley | 0ed712b | 2013-04-09 15:25:12 -0700 | [diff] [blame] | 7 | import httplib |
J. Richard Barnette | 1d78b01 | 2012-05-15 13:56:30 -0700 | [diff] [blame] | 8 | import logging |
Dan Shi | 0f466e8 | 2013-02-22 15:44:58 -0800 | [diff] [blame] | 9 | import os |
Simran Basi | d5e5e27 | 2012-09-24 15:23:59 -0700 | [diff] [blame] | 10 | import re |
Christopher Wiley | d78249a | 2013-03-01 13:05:31 -0800 | [diff] [blame] | 11 | import socket |
J. Richard Barnette | 1d78b01 | 2012-05-15 13:56:30 -0700 | [diff] [blame] | 12 | import subprocess |
J. Richard Barnette | 134ec2c | 2012-04-25 12:59:37 -0700 | [diff] [blame] | 13 | import time |
J. Richard Barnette | 1d78b01 | 2012-05-15 13:56:30 -0700 | [diff] [blame] | 14 | import xmlrpclib |
J. Richard Barnette | 134ec2c | 2012-04-25 12:59:37 -0700 | [diff] [blame] | 15 | |
mussa | 584b446 | 2014-06-20 15:13:28 -0700 | [diff] [blame] | 16 | import common |
J. Richard Barnette | 45e93de | 2012-04-11 17:24:15 -0700 | [diff] [blame] | 17 | from autotest_lib.client.bin import utils |
Richard Barnette | 0c73ffc | 2012-11-19 15:21:18 -0800 | [diff] [blame] | 18 | from autotest_lib.client.common_lib import error |
| 19 | from autotest_lib.client.common_lib import global_config |
J. Richard Barnette | 45e93de | 2012-04-11 17:24:15 -0700 | [diff] [blame] | 20 | from autotest_lib.client.common_lib.cros import autoupdater |
Richard Barnette | 03a0c13 | 2012-11-05 12:40:35 -0800 | [diff] [blame] | 21 | from autotest_lib.client.common_lib.cros import dev_server |
Christopher Wiley | d78249a | 2013-03-01 13:05:31 -0800 | [diff] [blame] | 22 | from autotest_lib.client.common_lib.cros import retry |
Michael Liang | da8c60a | 2014-06-03 13:24:51 -0700 | [diff] [blame] | 23 | from autotest_lib.client.common_lib.cros.graphite import stats |
Richard Barnette | 82c3591 | 2012-11-20 10:09:10 -0800 | [diff] [blame] | 24 | from autotest_lib.client.cros import constants |
J. Richard Barnette | 84890bd | 2014-02-21 11:05:47 -0800 | [diff] [blame] | 25 | from autotest_lib.client.cros import cros_ui |
Simran Basi | 263a9d3 | 2014-08-19 11:16:51 -0700 | [diff] [blame^] | 26 | from autotest_lib.server import autoserv_parser, autotest |
Dan Shi | a1ecd5c | 2013-06-06 11:21:31 -0700 | [diff] [blame] | 27 | from autotest_lib.server import utils as server_utils |
Scott Zawalski | 89c44dd | 2013-02-26 09:28:02 -0500 | [diff] [blame] | 28 | from autotest_lib.server.cros.dynamic_suite import constants as ds_constants |
Simran Basi | 5e6339a | 2013-03-21 11:34:32 -0700 | [diff] [blame] | 29 | from autotest_lib.server.cros.dynamic_suite import tools, frontend_wrappers |
Fang Deng | 96667ca | 2013-08-01 17:46:18 -0700 | [diff] [blame] | 30 | from autotest_lib.server.hosts import abstract_ssh |
Tom Wai-Hong Tam | efe1c7f | 2014-01-02 14:00:11 +0800 | [diff] [blame] | 31 | from autotest_lib.server.hosts import chameleon_host |
Fang Deng | 5d518f4 | 2013-08-02 14:04:32 -0700 | [diff] [blame] | 32 | from autotest_lib.server.hosts import servo_host |
Simran Basi | dcff425 | 2012-11-20 16:13:20 -0800 | [diff] [blame] | 33 | from autotest_lib.site_utils.rpm_control_system import rpm_client |
Simran Basi | d5e5e27 | 2012-09-24 15:23:59 -0700 | [diff] [blame] | 34 | |
| 35 | |
beeps | 32a6308 | 2013-08-22 14:02:29 -0700 | [diff] [blame] | 36 | try: |
| 37 | import jsonrpclib |
| 38 | except ImportError: |
| 39 | jsonrpclib = None |
Fang Deng | 96667ca | 2013-08-01 17:46:18 -0700 | [diff] [blame] | 40 | |
Fang Deng | d1c2b73 | 2013-08-20 12:59:46 -0700 | [diff] [blame] | 41 | |
beeps | c87ff60 | 2013-07-31 21:53:00 -0700 | [diff] [blame] | 42 | class FactoryImageCheckerException(error.AutoservError): |
| 43 | """Exception raised when an image is a factory image.""" |
| 44 | pass |
| 45 | |
| 46 | |
Aviv Keshet | 74c89a9 | 2013-02-04 15:18:30 -0800 | [diff] [blame] | 47 | def add_label_detector(label_function_list, label_list=None, label=None): |
| 48 | """Decorator used to group functions together into the provided list. |
| 49 | @param label_function_list: List of label detecting functions to add |
| 50 | decorated function to. |
| 51 | @param label_list: List of detectable labels to add detectable labels to. |
| 52 | (Default: None) |
| 53 | @param label: Label string that is detectable by this detection function |
| 54 | (Default: None) |
J. Richard Barnette | 7214e0b | 2013-02-06 15:20:49 -0800 | [diff] [blame] | 55 | """ |
Simran Basi | c6f1f7a | 2012-10-16 10:47:46 -0700 | [diff] [blame] | 56 | def add_func(func): |
Aviv Keshet | 74c89a9 | 2013-02-04 15:18:30 -0800 | [diff] [blame] | 57 | """ |
| 58 | @param func: The function to be added as a detector. |
| 59 | """ |
| 60 | label_function_list.append(func) |
| 61 | if label and label_list is not None: |
| 62 | label_list.append(label) |
Simran Basi | c6f1f7a | 2012-10-16 10:47:46 -0700 | [diff] [blame] | 63 | return func |
| 64 | return add_func |
| 65 | |
| 66 | |
Fang Deng | 0ca40e2 | 2013-08-27 17:47:44 -0700 | [diff] [blame] | 67 | class CrosHost(abstract_ssh.AbstractSSHHost): |
J. Richard Barnette | 45e93de | 2012-04-11 17:24:15 -0700 | [diff] [blame] | 68 | """Chromium OS specific subclass of Host.""" |
| 69 | |
| 70 | _parser = autoserv_parser.autoserv_parser |
Scott Zawalski | 62bacae | 2013-03-05 10:40:32 -0500 | [diff] [blame] | 71 | _AFE = frontend_wrappers.RetryingAFE(timeout_min=5, delay_sec=10) |
J. Richard Barnette | 45e93de | 2012-04-11 17:24:15 -0700 | [diff] [blame] | 72 | |
Richard Barnette | 03a0c13 | 2012-11-05 12:40:35 -0800 | [diff] [blame] | 73 | # Timeout values (in seconds) associated with various Chrome OS |
| 74 | # state changes. |
J. Richard Barnette | 134ec2c | 2012-04-25 12:59:37 -0700 | [diff] [blame] | 75 | # |
Richard Barnette | 0c73ffc | 2012-11-19 15:21:18 -0800 | [diff] [blame] | 76 | # In general, a good rule of thumb is that the timeout can be up |
| 77 | # to twice the typical measured value on the slowest platform. |
| 78 | # The times here have not necessarily been empirically tested to |
| 79 | # meet this criterion. |
J. Richard Barnette | eb69d72 | 2012-06-18 17:29:44 -0700 | [diff] [blame] | 80 | # |
| 81 | # SLEEP_TIMEOUT: Time to allow for suspend to memory. |
Richard Barnette | 0c73ffc | 2012-11-19 15:21:18 -0800 | [diff] [blame] | 82 | # RESUME_TIMEOUT: Time to allow for resume after suspend, plus |
| 83 | # time to restart the netwowrk. |
J. Richard Barnette | 84890bd | 2014-02-21 11:05:47 -0800 | [diff] [blame] | 84 | # SHUTDOWN_TIMEOUT: Time to allow for shut down. |
J. Richard Barnette | eb69d72 | 2012-06-18 17:29:44 -0700 | [diff] [blame] | 85 | # BOOT_TIMEOUT: Time to allow for boot from power off. Among |
Richard Barnette | 0c73ffc | 2012-11-19 15:21:18 -0800 | [diff] [blame] | 86 | # other things, this must account for the 30 second dev-mode |
J. Richard Barnette | d4649c6 | 2013-03-06 17:42:27 -0800 | [diff] [blame] | 87 | # screen delay and time to start the network. |
J. Richard Barnette | eb69d72 | 2012-06-18 17:29:44 -0700 | [diff] [blame] | 88 | # USB_BOOT_TIMEOUT: Time to allow for boot from a USB device, |
Richard Barnette | 0c73ffc | 2012-11-19 15:21:18 -0800 | [diff] [blame] | 89 | # including the 30 second dev-mode delay and time to start the |
J. Richard Barnette | d4649c6 | 2013-03-06 17:42:27 -0800 | [diff] [blame] | 90 | # network. |
beeps | f079cfb | 2013-09-18 17:49:51 -0700 | [diff] [blame] | 91 | # INSTALL_TIMEOUT: Time to allow for chromeos-install. |
J. Richard Barnette | 84890bd | 2014-02-21 11:05:47 -0800 | [diff] [blame] | 92 | # POWERWASH_BOOT_TIMEOUT: Time to allow for a reboot that |
| 93 | # includes powerwash. |
J. Richard Barnette | eb69d72 | 2012-06-18 17:29:44 -0700 | [diff] [blame] | 94 | |
| 95 | SLEEP_TIMEOUT = 2 |
J. Richard Barnette | d4649c6 | 2013-03-06 17:42:27 -0800 | [diff] [blame] | 96 | RESUME_TIMEOUT = 10 |
Tom Wai-Hong Tam | 4d169ed | 2014-02-14 11:05:40 +0800 | [diff] [blame] | 97 | SHUTDOWN_TIMEOUT = 5 |
J. Richard Barnette | fbcc712 | 2013-07-24 18:24:59 -0700 | [diff] [blame] | 98 | BOOT_TIMEOUT = 60 |
J. Richard Barnette | eb69d72 | 2012-06-18 17:29:44 -0700 | [diff] [blame] | 99 | USB_BOOT_TIMEOUT = 150 |
J. Richard Barnette | 84890bd | 2014-02-21 11:05:47 -0800 | [diff] [blame] | 100 | INSTALL_TIMEOUT = 240 |
Dan Shi | 2c88eed | 2013-11-12 10:18:38 -0800 | [diff] [blame] | 101 | POWERWASH_BOOT_TIMEOUT = 60 |
Chris Sosa | b76e0ee | 2013-05-22 16:55:41 -0700 | [diff] [blame] | 102 | |
J. Richard Barnette | 84890bd | 2014-02-21 11:05:47 -0800 | [diff] [blame] | 103 | # REBOOT_TIMEOUT: How long to wait for a reboot. |
| 104 | # |
Chris Sosa | b76e0ee | 2013-05-22 16:55:41 -0700 | [diff] [blame] | 105 | # We have a long timeout to ensure we don't flakily fail due to other |
| 106 | # issues. Shorter timeouts are vetted in platform_RebootAfterUpdate. |
Simran Basi | 1160e2c | 2013-10-04 16:00:24 -0700 | [diff] [blame] | 107 | # TODO(sbasi - crbug.com/276094) Restore to 5 mins once the 'host did not |
| 108 | # return from reboot' bug is solved. |
| 109 | REBOOT_TIMEOUT = 480 |
Chris Sosa | b76e0ee | 2013-05-22 16:55:41 -0700 | [diff] [blame] | 110 | |
Ismail Noorbasha | 07fdb61 | 2013-02-14 14:13:31 -0800 | [diff] [blame] | 111 | # _USB_POWER_TIMEOUT: Time to allow for USB to power toggle ON and OFF. |
| 112 | # _POWER_CYCLE_TIMEOUT: Time to allow for manual power cycle. |
| 113 | _USB_POWER_TIMEOUT = 5 |
| 114 | _POWER_CYCLE_TIMEOUT = 10 |
| 115 | |
beeps | 32a6308 | 2013-08-22 14:02:29 -0700 | [diff] [blame] | 116 | _RPC_PROXY_URL = 'http://localhost:%d' |
Christopher Wiley | dd18185 | 2013-10-10 19:56:58 -0700 | [diff] [blame] | 117 | _RPC_SHUTDOWN_POLLING_PERIOD_SECONDS = 2 |
Peter Qiu | 4410db7 | 2014-06-05 10:32:41 -0700 | [diff] [blame] | 118 | # Set shutdown timeout to account for the time for restarting the UI. |
| 119 | _RPC_SHUTDOWN_TIMEOUT_SECONDS = cros_ui.RESTART_UI_TIMEOUT |
Richard Barnette | 0c73ffc | 2012-11-19 15:21:18 -0800 | [diff] [blame] | 120 | |
Richard Barnette | 82c3591 | 2012-11-20 10:09:10 -0800 | [diff] [blame] | 121 | _RPM_RECOVERY_BOARDS = global_config.global_config.get_config_value('CROS', |
| 122 | 'rpm_recovery_boards', type=str).split(',') |
| 123 | |
| 124 | _MAX_POWER_CYCLE_ATTEMPTS = 6 |
| 125 | _LAB_MACHINE_FILE = '/mnt/stateful_partition/.labmachine' |
Simran Basi | 9a479b2 | 2014-07-09 11:18:40 -0700 | [diff] [blame] | 126 | # TODO (sbasi) crbug.com/392548 - renable support for chromeos 4 once the |
| 127 | # rpm work is done. |
| 128 | _RPM_HOSTNAME_REGEX = ('chromeos[0-3|5-9]+(-row[0-9]+)?-rack[0-9]+[a-z]*-' |
Richard Barnette | 82c3591 | 2012-11-20 10:09:10 -0800 | [diff] [blame] | 129 | 'host[0-9]+') |
| 130 | _LIGHTSENSOR_FILES = ['in_illuminance0_input', |
| 131 | 'in_illuminance0_raw', |
| 132 | 'illuminance0_input'] |
| 133 | _LIGHTSENSOR_SEARCH_DIR = '/sys/bus/iio/devices' |
| 134 | _LABEL_FUNCTIONS = [] |
Aviv Keshet | 74c89a9 | 2013-02-04 15:18:30 -0800 | [diff] [blame] | 135 | _DETECTABLE_LABELS = [] |
| 136 | label_decorator = functools.partial(add_label_detector, _LABEL_FUNCTIONS, |
| 137 | _DETECTABLE_LABELS) |
J. Richard Barnette | 134ec2c | 2012-04-25 12:59:37 -0700 | [diff] [blame] | 138 | |
J. Richard Barnette | b6de7e3 | 2013-02-14 13:28:04 -0800 | [diff] [blame] | 139 | # Constants used in ping_wait_up() and ping_wait_down(). |
| 140 | # |
| 141 | # _PING_WAIT_COUNT is the approximate number of polling |
| 142 | # cycles to use when waiting for a host state change. |
| 143 | # |
| 144 | # _PING_STATUS_DOWN and _PING_STATUS_UP are names used |
| 145 | # for arguments to the internal _ping_wait_for_status() |
| 146 | # method. |
| 147 | _PING_WAIT_COUNT = 40 |
| 148 | _PING_STATUS_DOWN = False |
| 149 | _PING_STATUS_UP = True |
| 150 | |
Ismail Noorbasha | 07fdb61 | 2013-02-14 14:13:31 -0800 | [diff] [blame] | 151 | # Allowed values for the power_method argument. |
| 152 | |
| 153 | # POWER_CONTROL_RPM: Passed as default arg for power_off/on/cycle() methods. |
| 154 | # POWER_CONTROL_SERVO: Used in set_power() and power_cycle() methods. |
| 155 | # POWER_CONTROL_MANUAL: Used in set_power() and power_cycle() methods. |
| 156 | POWER_CONTROL_RPM = 'RPM' |
| 157 | POWER_CONTROL_SERVO = 'servoj10' |
| 158 | POWER_CONTROL_MANUAL = 'manual' |
| 159 | |
| 160 | POWER_CONTROL_VALID_ARGS = (POWER_CONTROL_RPM, |
| 161 | POWER_CONTROL_SERVO, |
| 162 | POWER_CONTROL_MANUAL) |
Richard Barnette | 0c73ffc | 2012-11-19 15:21:18 -0800 | [diff] [blame] | 163 | |
Simran Basi | 5e6339a | 2013-03-21 11:34:32 -0700 | [diff] [blame] | 164 | _RPM_OUTLET_CHANGED = 'outlet_changed' |
| 165 | |
beeps | 687243d | 2013-07-18 15:29:27 -0700 | [diff] [blame] | 166 | |
J. Richard Barnette | 964fba0 | 2012-10-24 17:34:29 -0700 | [diff] [blame] | 167 | @staticmethod |
beeps | 46dadc9 | 2013-11-07 14:07:10 -0800 | [diff] [blame] | 168 | def check_host(host, timeout=10): |
| 169 | """ |
| 170 | Check if the given host is a chrome-os host. |
| 171 | |
| 172 | @param host: An ssh host representing a device. |
| 173 | @param timeout: The timeout for the run command. |
| 174 | |
| 175 | @return: True if the host device is chromeos. |
| 176 | |
beeps | 46dadc9 | 2013-11-07 14:07:10 -0800 | [diff] [blame] | 177 | """ |
| 178 | try: |
Christopher Wiley | 1ea8094 | 2014-02-26 16:45:08 -0800 | [diff] [blame] | 179 | result = host.run('grep -q CHROMEOS /etc/lsb-release && ' |
Simran Basi | e5f7ae4 | 2014-06-26 15:44:06 -0700 | [diff] [blame] | 180 | '! which adb >/dev/null 2>&1 && ' |
| 181 | '! grep -q moblab /etc/lsb-release', |
Christopher Wiley | fc3eac0 | 2013-11-21 16:24:57 -0800 | [diff] [blame] | 182 | ignore_status=True, timeout=timeout) |
beeps | 46dadc9 | 2013-11-07 14:07:10 -0800 | [diff] [blame] | 183 | except (error.AutoservRunError, error.AutoservSSHTimeout): |
| 184 | return False |
| 185 | return result.exit_status == 0 |
| 186 | |
| 187 | |
| 188 | @staticmethod |
Tom Wai-Hong Tam | efe1c7f | 2014-01-02 14:00:11 +0800 | [diff] [blame] | 189 | def _extract_arguments(args_dict, key_subset): |
| 190 | """Extract options from `args_dict` and return a subset result. |
J. Richard Barnette | 7214e0b | 2013-02-06 15:20:49 -0800 | [diff] [blame] | 191 | |
| 192 | Take the provided dictionary of argument options and return |
Tom Wai-Hong Tam | efe1c7f | 2014-01-02 14:00:11 +0800 | [diff] [blame] | 193 | a subset that represent standard arguments needed to construct |
| 194 | a test-assistant object (chameleon or servo) for a host. The |
| 195 | intent is to provide standard argument processing from |
| 196 | run_remote_tests for tests that require a test-assistant board |
| 197 | to operate. |
| 198 | |
| 199 | @param args_dict Dictionary from which to extract the arguments. |
| 200 | @param key_subset Tuple of keys to extract from the args_dict, e.g. |
| 201 | ('servo_host', 'servo_port'). |
| 202 | """ |
| 203 | result = {} |
| 204 | for arg in key_subset: |
| 205 | if arg in args_dict: |
| 206 | result[arg] = args_dict[arg] |
| 207 | return result |
| 208 | |
| 209 | |
| 210 | @staticmethod |
| 211 | def get_chameleon_arguments(args_dict): |
| 212 | """Extract chameleon options from `args_dict` and return the result. |
| 213 | |
| 214 | Recommended usage: |
| 215 | ~~~~~~~~ |
| 216 | args_dict = utils.args_to_dict(args) |
| 217 | chameleon_args = hosts.CrosHost.get_chameleon_arguments(args_dict) |
| 218 | host = hosts.create_host(machine, chameleon_args=chameleon_args) |
| 219 | ~~~~~~~~ |
| 220 | |
| 221 | @param args_dict Dictionary from which to extract the chameleon |
| 222 | arguments. |
| 223 | """ |
| 224 | return CrosHost._extract_arguments( |
| 225 | args_dict, ('chameleon_host', 'chameleon_port')) |
| 226 | |
| 227 | |
| 228 | @staticmethod |
| 229 | def get_servo_arguments(args_dict): |
| 230 | """Extract servo options from `args_dict` and return the result. |
J. Richard Barnette | 7214e0b | 2013-02-06 15:20:49 -0800 | [diff] [blame] | 231 | |
| 232 | Recommended usage: |
| 233 | ~~~~~~~~ |
| 234 | args_dict = utils.args_to_dict(args) |
Fang Deng | 0ca40e2 | 2013-08-27 17:47:44 -0700 | [diff] [blame] | 235 | servo_args = hosts.CrosHost.get_servo_arguments(args_dict) |
J. Richard Barnette | 7214e0b | 2013-02-06 15:20:49 -0800 | [diff] [blame] | 236 | host = hosts.create_host(machine, servo_args=servo_args) |
| 237 | ~~~~~~~~ |
| 238 | |
| 239 | @param args_dict Dictionary from which to extract the servo |
| 240 | arguments. |
| 241 | """ |
Tom Wai-Hong Tam | efe1c7f | 2014-01-02 14:00:11 +0800 | [diff] [blame] | 242 | return CrosHost._extract_arguments( |
| 243 | args_dict, ('servo_host', 'servo_port')) |
J. Richard Barnette | 134ec2c | 2012-04-25 12:59:37 -0700 | [diff] [blame] | 244 | |
J. Richard Barnette | 964fba0 | 2012-10-24 17:34:29 -0700 | [diff] [blame] | 245 | |
Tom Wai-Hong Tam | efe1c7f | 2014-01-02 14:00:11 +0800 | [diff] [blame] | 246 | def _initialize(self, hostname, chameleon_args=None, servo_args=None, |
| 247 | ssh_verbosity_flag='', ssh_options='', |
Fang Deng | d1c2b73 | 2013-08-20 12:59:46 -0700 | [diff] [blame] | 248 | *args, **dargs): |
Tom Wai-Hong Tam | efe1c7f | 2014-01-02 14:00:11 +0800 | [diff] [blame] | 249 | """Initialize superclasses, |self.chameleon|, and |self.servo|. |
J. Richard Barnette | 67ccb87 | 2012-04-19 16:34:56 -0700 | [diff] [blame] | 250 | |
Tom Wai-Hong Tam | efe1c7f | 2014-01-02 14:00:11 +0800 | [diff] [blame] | 251 | This method checks whether a chameleon/servo (aka |
| 252 | test-assistant objects) is required by checking whether |
| 253 | chameleon_args/servo_args is None. This method will only |
| 254 | attempt to create the test-assistant object when it is |
| 255 | required by the test. |
Fang Deng | 5d518f4 | 2013-08-02 14:04:32 -0700 | [diff] [blame] | 256 | |
Tom Wai-Hong Tam | efe1c7f | 2014-01-02 14:00:11 +0800 | [diff] [blame] | 257 | For creating the test-assistant object, there are three |
| 258 | possibilities: First, if the host is a lab system known to have |
| 259 | a test-assistant board, we connect to that board unconditionally. |
J. Richard Barnette | 67ccb87 | 2012-04-19 16:34:56 -0700 | [diff] [blame] | 260 | Second, if we're called from a control file that requires |
Tom Wai-Hong Tam | efe1c7f | 2014-01-02 14:00:11 +0800 | [diff] [blame] | 261 | test-assistant features for testing, it will pass settings from |
| 262 | the arguments, like `servo_host`, `servo_port`. If neither of |
| 263 | these cases apply, the test-assistant object will be `None`. |
J. Richard Barnette | 67ccb87 | 2012-04-19 16:34:56 -0700 | [diff] [blame] | 264 | |
| 265 | """ |
Fang Deng | 0ca40e2 | 2013-08-27 17:47:44 -0700 | [diff] [blame] | 266 | super(CrosHost, self)._initialize(hostname=hostname, |
J. Richard Barnette | 45e93de | 2012-04-11 17:24:15 -0700 | [diff] [blame] | 267 | *args, **dargs) |
J. Richard Barnette | f085985 | 2012-08-20 14:55:50 -0700 | [diff] [blame] | 268 | # self.env is a dictionary of environment variable settings |
| 269 | # to be exported for commands run on the host. |
| 270 | # LIBC_FATAL_STDERR_ can be useful for diagnosing certain |
| 271 | # errors that might happen. |
| 272 | self.env['LIBC_FATAL_STDERR_'] = '1' |
beeps | 32a6308 | 2013-08-22 14:02:29 -0700 | [diff] [blame] | 273 | self._rpc_proxy_map = {} |
Fang Deng | d1c2b73 | 2013-08-20 12:59:46 -0700 | [diff] [blame] | 274 | self._ssh_verbosity_flag = ssh_verbosity_flag |
Aviv Keshet | c5947fa | 2013-09-04 14:06:29 -0700 | [diff] [blame] | 275 | self._ssh_options = ssh_options |
Fang Deng | 5d518f4 | 2013-08-02 14:04:32 -0700 | [diff] [blame] | 276 | # TODO(fdeng): We need to simplify the |
| 277 | # process of servo and servo_host initialization. |
| 278 | # crbug.com/298432 |
Dan Shi | 4d47852 | 2014-02-14 13:46:32 -0800 | [diff] [blame] | 279 | self._servo_host = servo_host.create_servo_host(dut=self.hostname, |
| 280 | servo_args=servo_args) |
Tom Wai-Hong Tam | efe1c7f | 2014-01-02 14:00:11 +0800 | [diff] [blame] | 281 | # TODO(waihong): Do the simplication on Chameleon too. |
Tom Wai-Hong Tam | 3d6790d | 2014-04-14 16:15:47 +0800 | [diff] [blame] | 282 | self._chameleon_host = chameleon_host.create_chameleon_host( |
| 283 | dut=self.hostname, chameleon_args=chameleon_args) |
| 284 | |
Dan Shi | 4d47852 | 2014-02-14 13:46:32 -0800 | [diff] [blame] | 285 | if self._servo_host is not None: |
| 286 | self.servo = self._servo_host.get_servo() |
| 287 | else: |
| 288 | self.servo = None |
| 289 | |
Tom Wai-Hong Tam | 3d6790d | 2014-04-14 16:15:47 +0800 | [diff] [blame] | 290 | if self._chameleon_host: |
Tom Wai-Hong Tam | eaee340 | 2014-01-22 08:52:10 +0800 | [diff] [blame] | 291 | self.chameleon = self._chameleon_host.create_chameleon_board() |
Tom Wai-Hong Tam | efe1c7f | 2014-01-02 14:00:11 +0800 | [diff] [blame] | 292 | else: |
Tom Wai-Hong Tam | 3d6790d | 2014-04-14 16:15:47 +0800 | [diff] [blame] | 293 | self.chameleon = None |
Fang Deng | 5d518f4 | 2013-08-02 14:04:32 -0700 | [diff] [blame] | 294 | |
| 295 | |
Scott Zawalski | 89c44dd | 2013-02-26 09:28:02 -0500 | [diff] [blame] | 296 | def get_repair_image_name(self): |
| 297 | """Generate a image_name from variables in the global config. |
| 298 | |
| 299 | @returns a str of $board-version/$BUILD. |
| 300 | |
| 301 | """ |
| 302 | stable_version = global_config.global_config.get_config_value( |
| 303 | 'CROS', 'stable_cros_version') |
| 304 | build_pattern = global_config.global_config.get_config_value( |
| 305 | 'CROS', 'stable_build_pattern') |
| 306 | board = self._get_board_from_afe() |
| 307 | if board is None: |
| 308 | raise error.AutoservError('DUT has no board attribute, ' |
| 309 | 'cannot be repaired.') |
| 310 | return build_pattern % (board, stable_version) |
| 311 | |
| 312 | |
Scott Zawalski | 62bacae | 2013-03-05 10:40:32 -0500 | [diff] [blame] | 313 | def _host_in_AFE(self): |
| 314 | """Check if the host is an object the AFE knows. |
| 315 | |
| 316 | @returns the host object. |
| 317 | """ |
| 318 | return self._AFE.get_hosts(hostname=self.hostname) |
| 319 | |
| 320 | |
Chris Sosa | b76e0ee | 2013-05-22 16:55:41 -0700 | [diff] [blame] | 321 | def lookup_job_repo_url(self): |
| 322 | """Looks up the job_repo_url for the host. |
| 323 | |
| 324 | @returns job_repo_url from AFE or None if not found. |
| 325 | |
| 326 | @raises KeyError if the host does not have a job_repo_url |
| 327 | """ |
| 328 | if not self._host_in_AFE(): |
| 329 | return None |
| 330 | |
| 331 | hosts = self._AFE.get_hosts(hostname=self.hostname) |
beeps | b5efc53 | 2013-06-04 11:29:34 -0700 | [diff] [blame] | 332 | if hosts and ds_constants.JOB_REPO_URL in hosts[0].attributes: |
| 333 | return hosts[0].attributes[ds_constants.JOB_REPO_URL] |
Chris Sosa | b76e0ee | 2013-05-22 16:55:41 -0700 | [diff] [blame] | 334 | |
| 335 | |
Scott Zawalski | 89c44dd | 2013-02-26 09:28:02 -0500 | [diff] [blame] | 336 | def clear_cros_version_labels_and_job_repo_url(self): |
| 337 | """Clear cros_version labels and host attribute job_repo_url.""" |
Scott Zawalski | 62bacae | 2013-03-05 10:40:32 -0500 | [diff] [blame] | 338 | if not self._host_in_AFE(): |
Scott Zawalski | eadbf70 | 2013-03-14 09:23:06 -0400 | [diff] [blame] | 339 | return |
| 340 | |
Scott Zawalski | 62bacae | 2013-03-05 10:40:32 -0500 | [diff] [blame] | 341 | host_list = [self.hostname] |
| 342 | labels = self._AFE.get_labels( |
| 343 | name__startswith=ds_constants.VERSION_PREFIX, |
| 344 | host__hostname=self.hostname) |
Dan Shi | 0f466e8 | 2013-02-22 15:44:58 -0800 | [diff] [blame] | 345 | |
Scott Zawalski | 62bacae | 2013-03-05 10:40:32 -0500 | [diff] [blame] | 346 | for label in labels: |
| 347 | label.remove_hosts(hosts=host_list) |
Scott Zawalski | 89c44dd | 2013-02-26 09:28:02 -0500 | [diff] [blame] | 348 | |
beeps | cb6f1e2 | 2013-06-28 19:14:10 -0700 | [diff] [blame] | 349 | self.update_job_repo_url(None, None) |
| 350 | |
| 351 | |
| 352 | def update_job_repo_url(self, devserver_url, image_name): |
| 353 | """ |
| 354 | Updates the job_repo_url host attribute and asserts it's value. |
| 355 | |
| 356 | @param devserver_url: The devserver to use in the job_repo_url. |
| 357 | @param image_name: The name of the image to use in the job_repo_url. |
| 358 | |
| 359 | @raises AutoservError: If we failed to update the job_repo_url. |
| 360 | """ |
| 361 | repo_url = None |
| 362 | if devserver_url and image_name: |
| 363 | repo_url = tools.get_package_url(devserver_url, image_name) |
| 364 | self._AFE.set_host_attribute(ds_constants.JOB_REPO_URL, repo_url, |
Scott Zawalski | 62bacae | 2013-03-05 10:40:32 -0500 | [diff] [blame] | 365 | hostname=self.hostname) |
beeps | cb6f1e2 | 2013-06-28 19:14:10 -0700 | [diff] [blame] | 366 | if self.lookup_job_repo_url() != repo_url: |
| 367 | raise error.AutoservError('Failed to update job_repo_url with %s, ' |
| 368 | 'host %s' % (repo_url, self.hostname)) |
Scott Zawalski | 89c44dd | 2013-02-26 09:28:02 -0500 | [diff] [blame] | 369 | |
| 370 | |
Dan Shi | e930926 | 2013-06-19 22:50:21 -0700 | [diff] [blame] | 371 | def add_cros_version_labels_and_job_repo_url(self, image_name): |
Scott Zawalski | eadbf70 | 2013-03-14 09:23:06 -0400 | [diff] [blame] | 372 | """Add cros_version labels and host attribute job_repo_url. |
| 373 | |
| 374 | @param image_name: The name of the image e.g. |
| 375 | lumpy-release/R27-3837.0.0 |
Dan Shi | 7458bf6 | 2013-06-10 12:50:16 -0700 | [diff] [blame] | 376 | |
Scott Zawalski | eadbf70 | 2013-03-14 09:23:06 -0400 | [diff] [blame] | 377 | """ |
Scott Zawalski | 62bacae | 2013-03-05 10:40:32 -0500 | [diff] [blame] | 378 | if not self._host_in_AFE(): |
Scott Zawalski | eadbf70 | 2013-03-14 09:23:06 -0400 | [diff] [blame] | 379 | return |
Scott Zawalski | 62bacae | 2013-03-05 10:40:32 -0500 | [diff] [blame] | 380 | |
Scott Zawalski | eadbf70 | 2013-03-14 09:23:06 -0400 | [diff] [blame] | 381 | cros_label = '%s%s' % (ds_constants.VERSION_PREFIX, image_name) |
Dan Shi | e930926 | 2013-06-19 22:50:21 -0700 | [diff] [blame] | 382 | devserver_url = dev_server.ImageServer.resolve(image_name).url() |
Scott Zawalski | 62bacae | 2013-03-05 10:40:32 -0500 | [diff] [blame] | 383 | |
| 384 | labels = self._AFE.get_labels(name=cros_label) |
| 385 | if labels: |
| 386 | label = labels[0] |
| 387 | else: |
| 388 | label = self._AFE.create_label(name=cros_label) |
| 389 | |
| 390 | label.add_hosts([self.hostname]) |
beeps | cb6f1e2 | 2013-06-28 19:14:10 -0700 | [diff] [blame] | 391 | self.update_job_repo_url(devserver_url, image_name) |
| 392 | |
| 393 | |
beeps | dae65fd | 2013-07-26 16:24:41 -0700 | [diff] [blame] | 394 | def verify_job_repo_url(self, tag=''): |
beeps | cb6f1e2 | 2013-06-28 19:14:10 -0700 | [diff] [blame] | 395 | """ |
| 396 | Make sure job_repo_url of this host is valid. |
| 397 | |
joychen | 03eaad9 | 2013-06-26 09:55:21 -0700 | [diff] [blame] | 398 | Eg: The job_repo_url "http://lmn.cd.ab.xyx:8080/static/\ |
beeps | cb6f1e2 | 2013-06-28 19:14:10 -0700 | [diff] [blame] | 399 | lumpy-release/R29-4279.0.0/autotest/packages" claims to have the |
| 400 | autotest package for lumpy-release/R29-4279.0.0. If this isn't the case, |
| 401 | download and extract it. If the devserver embedded in the url is |
| 402 | unresponsive, update the job_repo_url of the host after staging it on |
| 403 | another devserver. |
| 404 | |
| 405 | @param job_repo_url: A url pointing to the devserver where the autotest |
| 406 | package for this build should be staged. |
beeps | dae65fd | 2013-07-26 16:24:41 -0700 | [diff] [blame] | 407 | @param tag: The tag from the server job, in the format |
| 408 | <job_id>-<user>/<hostname>, or <hostless> for a server job. |
beeps | cb6f1e2 | 2013-06-28 19:14:10 -0700 | [diff] [blame] | 409 | |
| 410 | @raises DevServerException: If we could not resolve a devserver. |
| 411 | @raises AutoservError: If we're unable to save the new job_repo_url as |
| 412 | a result of choosing a new devserver because the old one failed to |
| 413 | respond to a health check. |
beeps | 0c86503 | 2013-07-30 11:37:06 -0700 | [diff] [blame] | 414 | @raises urllib2.URLError: If the devserver embedded in job_repo_url |
| 415 | doesn't respond within the timeout. |
beeps | cb6f1e2 | 2013-06-28 19:14:10 -0700 | [diff] [blame] | 416 | """ |
| 417 | job_repo_url = self.lookup_job_repo_url() |
| 418 | if not job_repo_url: |
| 419 | logging.warning('No job repo url set on host %s', self.hostname) |
| 420 | return |
| 421 | |
| 422 | logging.info('Verifying job repo url %s', job_repo_url) |
| 423 | devserver_url, image_name = tools.get_devserver_build_from_package_url( |
| 424 | job_repo_url) |
| 425 | |
beeps | 0c86503 | 2013-07-30 11:37:06 -0700 | [diff] [blame] | 426 | ds = dev_server.ImageServer(devserver_url) |
beeps | cb6f1e2 | 2013-06-28 19:14:10 -0700 | [diff] [blame] | 427 | |
| 428 | logging.info('Staging autotest artifacts for %s on devserver %s', |
| 429 | image_name, ds.url()) |
beeps | 687243d | 2013-07-18 15:29:27 -0700 | [diff] [blame] | 430 | |
| 431 | start_time = time.time() |
beeps | cb6f1e2 | 2013-06-28 19:14:10 -0700 | [diff] [blame] | 432 | ds.stage_artifacts(image_name, ['autotest']) |
beeps | 687243d | 2013-07-18 15:29:27 -0700 | [diff] [blame] | 433 | stage_time = time.time() - start_time |
| 434 | |
| 435 | # Record how much of the verification time comes from a devserver |
| 436 | # restage. If we're doing things right we should not see multiple |
| 437 | # devservers for a given board/build/branch path. |
| 438 | try: |
J. Richard Barnette | 3cbd76b | 2013-11-27 12:11:25 -0800 | [diff] [blame] | 439 | board, build_type, branch = server_utils.ParseBuildName( |
beeps | 687243d | 2013-07-18 15:29:27 -0700 | [diff] [blame] | 440 | image_name)[:3] |
J. Richard Barnette | 3cbd76b | 2013-11-27 12:11:25 -0800 | [diff] [blame] | 441 | except server_utils.ParseBuildNameException: |
beeps | 687243d | 2013-07-18 15:29:27 -0700 | [diff] [blame] | 442 | pass |
| 443 | else: |
beeps | 0c86503 | 2013-07-30 11:37:06 -0700 | [diff] [blame] | 444 | devserver = devserver_url[ |
Chris Sosa | 6542508 | 2013-10-16 13:26:22 -0700 | [diff] [blame] | 445 | devserver_url.find('/') + 2:devserver_url.rfind(':')] |
beeps | 687243d | 2013-07-18 15:29:27 -0700 | [diff] [blame] | 446 | stats_key = { |
| 447 | 'board': board, |
| 448 | 'build_type': build_type, |
| 449 | 'branch': branch, |
beeps | 0c86503 | 2013-07-30 11:37:06 -0700 | [diff] [blame] | 450 | 'devserver': devserver.replace('.', '_'), |
beeps | 687243d | 2013-07-18 15:29:27 -0700 | [diff] [blame] | 451 | } |
| 452 | stats.Gauge('verify_job_repo_url').send( |
| 453 | '%(board)s.%(build_type)s.%(branch)s.%(devserver)s' % stats_key, |
| 454 | stage_time) |
beeps | cb6f1e2 | 2013-06-28 19:14:10 -0700 | [diff] [blame] | 455 | |
Scott Zawalski | eadbf70 | 2013-03-14 09:23:06 -0400 | [diff] [blame] | 456 | |
Dan Shi | 0f466e8 | 2013-02-22 15:44:58 -0800 | [diff] [blame] | 457 | def _try_stateful_update(self, update_url, force_update, updater): |
| 458 | """Try to use stateful update to initialize DUT. |
| 459 | |
| 460 | When DUT is already running the same version that machine_install |
| 461 | tries to install, stateful update is a much faster way to clean up |
| 462 | the DUT for testing, compared to a full reimage. It is implemeted |
| 463 | by calling autoupdater.run_update, but skipping updating root, as |
| 464 | updating the kernel is time consuming and not necessary. |
| 465 | |
| 466 | @param update_url: url of the image. |
| 467 | @param force_update: Set to True to update the image even if the DUT |
| 468 | is running the same version. |
| 469 | @param updater: ChromiumOSUpdater instance used to update the DUT. |
| 470 | @returns: True if the DUT was updated with stateful update. |
| 471 | |
| 472 | """ |
J. Richard Barnette | 3f73103 | 2014-04-07 17:42:59 -0700 | [diff] [blame] | 473 | # TODO(jrbarnette): Yes, I hate this re.match() test case. |
| 474 | # It's better than the alternative: see crbug.com/360944. |
| 475 | image_name = autoupdater.url_to_image_name(update_url) |
| 476 | release_pattern = r'^.*-release/R[0-9]+-[0-9]+\.[0-9]+\.0$' |
| 477 | if not re.match(release_pattern, image_name): |
| 478 | return False |
Dan Shi | 0f466e8 | 2013-02-22 15:44:58 -0800 | [diff] [blame] | 479 | if not updater.check_version(): |
| 480 | return False |
| 481 | if not force_update: |
| 482 | logging.info('Canceling stateful update because the new and ' |
| 483 | 'old versions are the same.') |
| 484 | return False |
| 485 | # Following folders should be rebuilt after stateful update. |
| 486 | # A test file is used to confirm each folder gets rebuilt after |
| 487 | # the stateful update. |
| 488 | folders_to_check = ['/var', '/home', '/mnt/stateful_partition'] |
| 489 | test_file = '.test_file_to_be_deleted' |
| 490 | for folder in folders_to_check: |
| 491 | touch_path = os.path.join(folder, test_file) |
| 492 | self.run('touch %s' % touch_path) |
| 493 | |
| 494 | if not updater.run_update(force_update=True, update_root=False): |
| 495 | return False |
| 496 | |
| 497 | # Reboot to complete stateful update. |
Chris Sosa | b76e0ee | 2013-05-22 16:55:41 -0700 | [diff] [blame] | 498 | self.reboot(timeout=self.REBOOT_TIMEOUT, wait=True) |
Dan Shi | 0f466e8 | 2013-02-22 15:44:58 -0800 | [diff] [blame] | 499 | check_file_cmd = 'test -f %s; echo $?' |
| 500 | for folder in folders_to_check: |
| 501 | test_file_path = os.path.join(folder, test_file) |
| 502 | result = self.run(check_file_cmd % test_file_path, |
| 503 | ignore_status=True) |
| 504 | if result.exit_status == 1: |
| 505 | return False |
| 506 | return True |
| 507 | |
| 508 | |
J. Richard Barnette | 7275b61 | 2013-06-04 18:13:11 -0700 | [diff] [blame] | 509 | def _post_update_processing(self, updater, expected_kernel=None): |
Dan Shi | 0f466e8 | 2013-02-22 15:44:58 -0800 | [diff] [blame] | 510 | """After the DUT is updated, confirm machine_install succeeded. |
| 511 | |
| 512 | @param updater: ChromiumOSUpdater instance used to update the DUT. |
J. Richard Barnette | 7275b61 | 2013-06-04 18:13:11 -0700 | [diff] [blame] | 513 | @param expected_kernel: kernel expected to be active after reboot, |
| 514 | or `None` to skip rollback checking. |
Dan Shi | 0f466e8 | 2013-02-22 15:44:58 -0800 | [diff] [blame] | 515 | |
| 516 | """ |
J. Richard Barnette | 7275b61 | 2013-06-04 18:13:11 -0700 | [diff] [blame] | 517 | # Touch the lab machine file to leave a marker that |
| 518 | # distinguishes this image from other test images. |
| 519 | # Afterwards, we must re-run the autoreboot script because |
| 520 | # it depends on the _LAB_MACHINE_FILE. |
Dan Shi | 0f466e8 | 2013-02-22 15:44:58 -0800 | [diff] [blame] | 521 | self.run('touch %s' % self._LAB_MACHINE_FILE) |
Dan Shi | 0f466e8 | 2013-02-22 15:44:58 -0800 | [diff] [blame] | 522 | self.run('start autoreboot') |
Chris Sosa | 6542508 | 2013-10-16 13:26:22 -0700 | [diff] [blame] | 523 | updater.verify_boot_expectations( |
| 524 | expected_kernel, rollback_message= |
| 525 | 'Build %s failed to boot on %s; system rolled back to previous' |
| 526 | 'build' % (updater.update_version, self.hostname)) |
J. Richard Barnette | 7275b61 | 2013-06-04 18:13:11 -0700 | [diff] [blame] | 527 | # Check that we've got the build we meant to install. |
| 528 | if not updater.check_version_to_confirm_install(): |
| 529 | raise autoupdater.ChromiumOSError( |
| 530 | 'Failed to update %s to build %s; found build ' |
| 531 | '%s instead' % (self.hostname, |
Chris Sosa | 6542508 | 2013-10-16 13:26:22 -0700 | [diff] [blame] | 532 | updater.update_version, |
| 533 | updater.get_build_id())) |
Dan Shi | 0f466e8 | 2013-02-22 15:44:58 -0800 | [diff] [blame] | 534 | |
| 535 | |
J. Richard Barnette | e4af8b9 | 2013-05-01 13:16:12 -0700 | [diff] [blame] | 536 | def _stage_image_for_update(self, image_name=None): |
Scott Zawalski | eadbf70 | 2013-03-14 09:23:06 -0400 | [diff] [blame] | 537 | """Stage a build on a devserver and return the update_url. |
| 538 | |
| 539 | @param image_name: a name like lumpy-release/R27-3837.0.0 |
| 540 | @returns an update URL like: |
| 541 | http://172.22.50.205:8082/update/lumpy-release/R27-3837.0.0 |
| 542 | """ |
J. Richard Barnette | e4af8b9 | 2013-05-01 13:16:12 -0700 | [diff] [blame] | 543 | if not image_name: |
| 544 | image_name = self.get_repair_image_name() |
| 545 | logging.info('Staging build for AU: %s', image_name) |
Scott Zawalski | eadbf70 | 2013-03-14 09:23:06 -0400 | [diff] [blame] | 546 | devserver = dev_server.ImageServer.resolve(image_name) |
| 547 | devserver.trigger_download(image_name, synchronous=False) |
| 548 | return tools.image_url_pattern() % (devserver.url(), image_name) |
| 549 | |
| 550 | |
J. Richard Barnette | e4af8b9 | 2013-05-01 13:16:12 -0700 | [diff] [blame] | 551 | def stage_image_for_servo(self, image_name=None): |
| 552 | """Stage a build on a devserver and return the update_url. |
| 553 | |
| 554 | @param image_name: a name like lumpy-release/R27-3837.0.0 |
| 555 | @returns an update URL like: |
| 556 | http://172.22.50.205:8082/update/lumpy-release/R27-3837.0.0 |
| 557 | """ |
| 558 | if not image_name: |
| 559 | image_name = self.get_repair_image_name() |
| 560 | logging.info('Staging build for servo install: %s', image_name) |
| 561 | devserver = dev_server.ImageServer.resolve(image_name) |
| 562 | devserver.stage_artifacts(image_name, ['test_image']) |
| 563 | return devserver.get_test_image_url(image_name) |
| 564 | |
| 565 | |
beeps | e539be0 | 2013-07-31 21:57:39 -0700 | [diff] [blame] | 566 | def stage_factory_image_for_servo(self, image_name): |
| 567 | """Stage a build on a devserver and return the update_url. |
| 568 | |
| 569 | @param image_name: a name like <baord>/4262.204.0 |
beeps | 12c0a3c | 2013-09-03 11:58:27 -0700 | [diff] [blame] | 570 | |
beeps | e539be0 | 2013-07-31 21:57:39 -0700 | [diff] [blame] | 571 | @return: An update URL, eg: |
| 572 | http://<devserver>/static/canary-channel/\ |
| 573 | <board>/4262.204.0/factory_test/chromiumos_factory_image.bin |
beeps | 12c0a3c | 2013-09-03 11:58:27 -0700 | [diff] [blame] | 574 | |
| 575 | @raises: ValueError if the factory artifact name is missing from |
| 576 | the config. |
| 577 | |
beeps | e539be0 | 2013-07-31 21:57:39 -0700 | [diff] [blame] | 578 | """ |
| 579 | if not image_name: |
| 580 | logging.error('Need an image_name to stage a factory image.') |
| 581 | return |
| 582 | |
beeps | 12c0a3c | 2013-09-03 11:58:27 -0700 | [diff] [blame] | 583 | factory_artifact = global_config.global_config.get_config_value( |
| 584 | 'CROS', 'factory_artifact', type=str, default='') |
| 585 | if not factory_artifact: |
| 586 | raise ValueError('Cannot retrieve the factory artifact name from ' |
| 587 | 'autotest config, and hence cannot stage factory ' |
| 588 | 'artifacts.') |
| 589 | |
beeps | e539be0 | 2013-07-31 21:57:39 -0700 | [diff] [blame] | 590 | logging.info('Staging build for servo install: %s', image_name) |
| 591 | devserver = dev_server.ImageServer.resolve(image_name) |
| 592 | devserver.stage_artifacts( |
| 593 | image_name, |
beeps | 12c0a3c | 2013-09-03 11:58:27 -0700 | [diff] [blame] | 594 | [factory_artifact], |
| 595 | archive_url=None) |
beeps | e539be0 | 2013-07-31 21:57:39 -0700 | [diff] [blame] | 596 | |
| 597 | return tools.factory_image_url_pattern() % (devserver.url(), image_name) |
| 598 | |
| 599 | |
Chris Sosa | a3ac215 | 2012-05-23 22:23:13 -0700 | [diff] [blame] | 600 | def machine_install(self, update_url=None, force_update=False, |
Scott Zawalski | 89c44dd | 2013-02-26 09:28:02 -0500 | [diff] [blame] | 601 | local_devserver=False, repair=False): |
| 602 | """Install the DUT. |
| 603 | |
Dan Shi | 0f466e8 | 2013-02-22 15:44:58 -0800 | [diff] [blame] | 604 | Use stateful update if the DUT is already running the same build. |
| 605 | Stateful update does not update kernel and tends to run much faster |
| 606 | than a full reimage. If the DUT is running a different build, or it |
| 607 | failed to do a stateful update, full update, including kernel update, |
| 608 | will be applied to the DUT. |
| 609 | |
Scott Zawalski | eadbf70 | 2013-03-14 09:23:06 -0400 | [diff] [blame] | 610 | Once a host enters machine_install its cros_version label will be |
| 611 | removed as well as its host attribute job_repo_url (used for |
| 612 | package install). |
| 613 | |
Scott Zawalski | 89c44dd | 2013-02-26 09:28:02 -0500 | [diff] [blame] | 614 | @param update_url: The url to use for the update |
| 615 | pattern: http://$devserver:###/update/$build |
| 616 | If update_url is None and repair is True we will install the |
| 617 | stable image listed in global_config under |
| 618 | CROS.stable_cros_version. |
| 619 | @param force_update: Force an update even if the version installed |
| 620 | is the same. Default:False |
| 621 | @param local_devserver: Used by run_remote_test to allow people to |
| 622 | use their local devserver. Default: False |
| 623 | @param repair: Whether or not we are in repair mode. This adds special |
| 624 | cases for repairing a machine like starting update_engine. |
| 625 | Setting repair to True sets force_update to True as well. |
| 626 | default: False |
| 627 | @raises autoupdater.ChromiumOSError |
| 628 | |
| 629 | """ |
Dan Shi | 7458bf6 | 2013-06-10 12:50:16 -0700 | [diff] [blame] | 630 | if update_url: |
| 631 | logging.debug('update url is set to %s', update_url) |
| 632 | else: |
| 633 | logging.debug('update url is not set, resolving...') |
J. Richard Barnette | e4af8b9 | 2013-05-01 13:16:12 -0700 | [diff] [blame] | 634 | if self._parser.options.image: |
| 635 | requested_build = self._parser.options.image |
| 636 | if requested_build.startswith('http://'): |
| 637 | update_url = requested_build |
Dan Shi | 7458bf6 | 2013-06-10 12:50:16 -0700 | [diff] [blame] | 638 | logging.debug('update url is retrieved from requested_build' |
| 639 | ': %s', update_url) |
J. Richard Barnette | e4af8b9 | 2013-05-01 13:16:12 -0700 | [diff] [blame] | 640 | else: |
| 641 | # Try to stage any build that does not start with |
| 642 | # http:// on the devservers defined in |
| 643 | # global_config.ini. |
Dan Shi | 7458bf6 | 2013-06-10 12:50:16 -0700 | [diff] [blame] | 644 | update_url = self._stage_image_for_update(requested_build) |
| 645 | logging.debug('Build staged, and update_url is set to: %s', |
| 646 | update_url) |
J. Richard Barnette | e4af8b9 | 2013-05-01 13:16:12 -0700 | [diff] [blame] | 647 | elif repair: |
| 648 | update_url = self._stage_image_for_update() |
Dan Shi | 7458bf6 | 2013-06-10 12:50:16 -0700 | [diff] [blame] | 649 | logging.debug('Build staged, and update_url is set to: %s', |
| 650 | update_url) |
Scott Zawalski | eadbf70 | 2013-03-14 09:23:06 -0400 | [diff] [blame] | 651 | else: |
J. Richard Barnette | e4af8b9 | 2013-05-01 13:16:12 -0700 | [diff] [blame] | 652 | raise autoupdater.ChromiumOSError( |
| 653 | 'Update failed. No update URL provided.') |
Scott Zawalski | 89c44dd | 2013-02-26 09:28:02 -0500 | [diff] [blame] | 654 | |
Scott Zawalski | 89c44dd | 2013-02-26 09:28:02 -0500 | [diff] [blame] | 655 | if repair: |
Dan Shi | 0f466e8 | 2013-02-22 15:44:58 -0800 | [diff] [blame] | 656 | # In case the system is in a bad state, we always reboot the machine |
| 657 | # before machine_install. |
Chris Sosa | b76e0ee | 2013-05-22 16:55:41 -0700 | [diff] [blame] | 658 | self.reboot(timeout=self.REBOOT_TIMEOUT, wait=True) |
Scott Zawalski | 89c44dd | 2013-02-26 09:28:02 -0500 | [diff] [blame] | 659 | self.run('stop update-engine; start update-engine') |
| 660 | force_update = True |
Dan Shi | 0f466e8 | 2013-02-22 15:44:58 -0800 | [diff] [blame] | 661 | |
Chris Sosa | a3ac215 | 2012-05-23 22:23:13 -0700 | [diff] [blame] | 662 | updater = autoupdater.ChromiumOSUpdater(update_url, host=self, |
Chris Sosa | 7231260 | 2013-04-16 15:01:56 -0700 | [diff] [blame] | 663 | local_devserver=local_devserver) |
Dan Shi | 0f466e8 | 2013-02-22 15:44:58 -0800 | [diff] [blame] | 664 | updated = False |
Scott Zawalski | eadbf70 | 2013-03-14 09:23:06 -0400 | [diff] [blame] | 665 | # Remove cros-version and job_repo_url host attribute from host. |
| 666 | self.clear_cros_version_labels_and_job_repo_url() |
Dan Shi | 0f466e8 | 2013-02-22 15:44:58 -0800 | [diff] [blame] | 667 | # If the DUT is already running the same build, try stateful update |
| 668 | # first. Stateful update does not update kernel and tends to run much |
| 669 | # faster than a full reimage. |
| 670 | try: |
Chris Sosa | b76e0ee | 2013-05-22 16:55:41 -0700 | [diff] [blame] | 671 | updated = self._try_stateful_update( |
| 672 | update_url, force_update, updater) |
Dan Shi | 0f466e8 | 2013-02-22 15:44:58 -0800 | [diff] [blame] | 673 | if updated: |
| 674 | logging.info('DUT is updated with stateful update.') |
| 675 | except Exception as e: |
| 676 | logging.exception(e) |
Ilja H. Friedel | 04be2bd | 2014-05-07 21:29:59 -0700 | [diff] [blame] | 677 | logging.warning('Failed to stateful update DUT, force to update.') |
J. Richard Barnette | 45e93de | 2012-04-11 17:24:15 -0700 | [diff] [blame] | 678 | |
Dan Shi | 0f466e8 | 2013-02-22 15:44:58 -0800 | [diff] [blame] | 679 | inactive_kernel = None |
| 680 | # Do a full update if stateful update is not applicable or failed. |
| 681 | if not updated: |
| 682 | # In case the system is in a bad state, we always reboot the |
| 683 | # machine before machine_install. |
Chris Sosa | b76e0ee | 2013-05-22 16:55:41 -0700 | [diff] [blame] | 684 | self.reboot(timeout=self.REBOOT_TIMEOUT, wait=True) |
Chris Sosa | b7612bc | 2013-03-21 10:32:37 -0700 | [diff] [blame] | 685 | |
| 686 | # TODO(sosa): Remove temporary hack to get rid of bricked machines |
| 687 | # that can't update due to a corrupted policy. |
| 688 | self.run('rm -rf /var/lib/whitelist') |
| 689 | self.run('touch /var/lib/whitelist') |
| 690 | self.run('chmod -w /var/lib/whitelist') |
Scott Zawalski | b550d5a | 2013-03-22 09:23:59 -0400 | [diff] [blame] | 691 | self.run('stop update-engine; start update-engine') |
Chris Sosa | b7612bc | 2013-03-21 10:32:37 -0700 | [diff] [blame] | 692 | |
Dan Shi | 0f466e8 | 2013-02-22 15:44:58 -0800 | [diff] [blame] | 693 | if updater.run_update(force_update): |
| 694 | updated = True |
| 695 | # Figure out active and inactive kernel. |
| 696 | active_kernel, inactive_kernel = updater.get_kernel_state() |
J. Richard Barnette | 45e93de | 2012-04-11 17:24:15 -0700 | [diff] [blame] | 697 | |
Dan Shi | 0f466e8 | 2013-02-22 15:44:58 -0800 | [diff] [blame] | 698 | # Ensure inactive kernel has higher priority than active. |
| 699 | if (updater.get_kernel_priority(inactive_kernel) |
| 700 | < updater.get_kernel_priority(active_kernel)): |
| 701 | raise autoupdater.ChromiumOSError( |
| 702 | 'Update failed. The priority of the inactive kernel' |
| 703 | ' partition is less than that of the active kernel' |
| 704 | ' partition.') |
J. Richard Barnette | 45e93de | 2012-04-11 17:24:15 -0700 | [diff] [blame] | 705 | |
Dan Shi | 0f466e8 | 2013-02-22 15:44:58 -0800 | [diff] [blame] | 706 | # Updater has returned successfully; reboot the host. |
Chris Sosa | b76e0ee | 2013-05-22 16:55:41 -0700 | [diff] [blame] | 707 | self.reboot(timeout=self.REBOOT_TIMEOUT, wait=True) |
J. Richard Barnette | 45e93de | 2012-04-11 17:24:15 -0700 | [diff] [blame] | 708 | |
Dan Shi | 0f466e8 | 2013-02-22 15:44:58 -0800 | [diff] [blame] | 709 | if updated: |
| 710 | self._post_update_processing(updater, inactive_kernel) |
Scott Zawalski | eadbf70 | 2013-03-14 09:23:06 -0400 | [diff] [blame] | 711 | image_name = autoupdater.url_to_image_name(update_url) |
Dan Shi | e930926 | 2013-06-19 22:50:21 -0700 | [diff] [blame] | 712 | self.add_cros_version_labels_and_job_repo_url(image_name) |
Simran Basi | 13fa1ba | 2013-03-04 10:56:47 -0800 | [diff] [blame] | 713 | |
J. Richard Barnette | 45e93de | 2012-04-11 17:24:15 -0700 | [diff] [blame] | 714 | # Clean up any old autotest directories which may be lying around. |
| 715 | for path in global_config.global_config.get_config_value( |
| 716 | 'AUTOSERV', 'client_autodir_paths', type=list): |
| 717 | self.run('rm -rf ' + path) |
| 718 | |
| 719 | |
Dan Shi | 10e992b | 2013-08-30 11:02:59 -0700 | [diff] [blame] | 720 | def show_update_engine_log(self): |
| 721 | """Output update engine log.""" |
| 722 | logging.debug('Dumping %s', constants.UPDATE_ENGINE_LOG) |
| 723 | self.run('cat %s' % constants.UPDATE_ENGINE_LOG) |
| 724 | |
| 725 | |
Richard Barnette | 82c3591 | 2012-11-20 10:09:10 -0800 | [diff] [blame] | 726 | def _get_board_from_afe(self): |
| 727 | """Retrieve this host's board from its labels in the AFE. |
| 728 | |
| 729 | Looks for a host label of the form "board:<board>", and |
| 730 | returns the "<board>" part of the label. `None` is returned |
| 731 | if there is not a single, unique label matching the pattern. |
| 732 | |
| 733 | @returns board from label, or `None`. |
| 734 | """ |
Dan Shi | a1ecd5c | 2013-06-06 11:21:31 -0700 | [diff] [blame] | 735 | return server_utils.get_board_from_afe(self.hostname, self._AFE) |
Simran Basi | 833814b | 2013-01-29 13:13:43 -0800 | [diff] [blame] | 736 | |
| 737 | |
| 738 | def get_build(self): |
| 739 | """Retrieve the current build for this Host from the AFE. |
| 740 | |
| 741 | Looks through this host's labels in the AFE to determine its build. |
| 742 | |
| 743 | @returns The current build or None if it could not find it or if there |
| 744 | were multiple build labels assigned to this host. |
| 745 | """ |
Dan Shi | a1ecd5c | 2013-06-06 11:21:31 -0700 | [diff] [blame] | 746 | return server_utils.get_build_from_afe(self.hostname, self._AFE) |
Richard Barnette | 82c3591 | 2012-11-20 10:09:10 -0800 | [diff] [blame] | 747 | |
| 748 | |
Scott Zawalski | 89c44dd | 2013-02-26 09:28:02 -0500 | [diff] [blame] | 749 | def _install_repair(self): |
| 750 | """Attempt to repair this host using upate-engine. |
| 751 | |
| 752 | If the host is up, try installing the DUT with a stable |
| 753 | "repair" version of Chrome OS as defined in the global_config |
| 754 | under CROS.stable_cros_version. |
| 755 | |
Scott Zawalski | 62bacae | 2013-03-05 10:40:32 -0500 | [diff] [blame] | 756 | @raises AutoservRepairMethodNA if the DUT is not reachable. |
| 757 | @raises ChromiumOSError if the install failed for some reason. |
Scott Zawalski | 89c44dd | 2013-02-26 09:28:02 -0500 | [diff] [blame] | 758 | |
| 759 | """ |
| 760 | if not self.is_up(): |
Scott Zawalski | 62bacae | 2013-03-05 10:40:32 -0500 | [diff] [blame] | 761 | raise error.AutoservRepairMethodNA('DUT unreachable for install.') |
Scott Zawalski | 89c44dd | 2013-02-26 09:28:02 -0500 | [diff] [blame] | 762 | logging.info('Attempting to reimage machine to repair image.') |
| 763 | try: |
| 764 | self.machine_install(repair=True) |
Fang Deng | d0672f3 | 2013-03-18 17:18:09 -0700 | [diff] [blame] | 765 | except autoupdater.ChromiumOSError as e: |
| 766 | logging.exception(e) |
Scott Zawalski | 89c44dd | 2013-02-26 09:28:02 -0500 | [diff] [blame] | 767 | logging.info('Repair via install failed.') |
Scott Zawalski | 62bacae | 2013-03-05 10:40:32 -0500 | [diff] [blame] | 768 | raise |
Scott Zawalski | 89c44dd | 2013-02-26 09:28:02 -0500 | [diff] [blame] | 769 | |
| 770 | |
Dan Shi | 2c88eed | 2013-11-12 10:18:38 -0800 | [diff] [blame] | 771 | def _install_repair_with_powerwash(self): |
Dan Shi | 9cc4845 | 2013-11-12 12:39:26 -0800 | [diff] [blame] | 772 | """Attempt to powerwash first then repair this host using update-engine. |
Dan Shi | 2c88eed | 2013-11-12 10:18:38 -0800 | [diff] [blame] | 773 | |
Dan Shi | 9cc4845 | 2013-11-12 12:39:26 -0800 | [diff] [blame] | 774 | update-engine may fail due to a bad image. In such case, powerwash |
| 775 | may help to cleanup the DUT for update-engine to work again. |
Dan Shi | 2c88eed | 2013-11-12 10:18:38 -0800 | [diff] [blame] | 776 | |
| 777 | @raises AutoservRepairMethodNA if the DUT is not reachable. |
| 778 | @raises ChromiumOSError if the install failed for some reason. |
| 779 | |
| 780 | """ |
| 781 | if not self.is_up(): |
| 782 | raise error.AutoservRepairMethodNA('DUT unreachable for install.') |
| 783 | |
| 784 | logging.info('Attempting to powerwash the DUT.') |
| 785 | self.run('echo "fast safe" > ' |
| 786 | '/mnt/stateful_partition/factory_install_reset') |
| 787 | self.reboot(timeout=self.POWERWASH_BOOT_TIMEOUT, wait=True) |
| 788 | if not self.is_up(): |
Dan Shi | 9cc4845 | 2013-11-12 12:39:26 -0800 | [diff] [blame] | 789 | logging.error('Powerwash failed. DUT did not come back after ' |
Dan Shi | 2c88eed | 2013-11-12 10:18:38 -0800 | [diff] [blame] | 790 | 'reboot.') |
| 791 | raise error.AutoservRepairFailure( |
| 792 | 'DUT failed to boot from powerwash after %d seconds' % |
| 793 | self.POWERWASH_BOOT_TIMEOUT) |
| 794 | |
| 795 | logging.info('Powerwash succeeded.') |
| 796 | self._install_repair() |
| 797 | |
| 798 | |
beeps | f079cfb | 2013-09-18 17:49:51 -0700 | [diff] [blame] | 799 | def servo_install(self, image_url=None, usb_boot_timeout=USB_BOOT_TIMEOUT, |
| 800 | install_timeout=INSTALL_TIMEOUT): |
Scott Zawalski | 62bacae | 2013-03-05 10:40:32 -0500 | [diff] [blame] | 801 | """ |
| 802 | Re-install the OS on the DUT by: |
| 803 | 1) installing a test image on a USB storage device attached to the Servo |
| 804 | board, |
Richard Barnette | 03a0c13 | 2012-11-05 12:40:35 -0800 | [diff] [blame] | 805 | 2) booting that image in recovery mode, and then |
J. Richard Barnette | 31b2e31 | 2013-04-04 16:05:22 -0700 | [diff] [blame] | 806 | 3) installing the image with chromeos-install. |
| 807 | |
Scott Zawalski | 62bacae | 2013-03-05 10:40:32 -0500 | [diff] [blame] | 808 | @param image_url: If specified use as the url to install on the DUT. |
| 809 | otherwise boot the currently staged image on the USB stick. |
beeps | f079cfb | 2013-09-18 17:49:51 -0700 | [diff] [blame] | 810 | @param usb_boot_timeout: The usb_boot_timeout to use during reimage. |
| 811 | Factory images need a longer usb_boot_timeout than regular |
| 812 | cros images. |
| 813 | @param install_timeout: The timeout to use when installing the chromeos |
| 814 | image. Factory images need a longer install_timeout. |
Richard Barnette | 03a0c13 | 2012-11-05 12:40:35 -0800 | [diff] [blame] | 815 | |
Scott Zawalski | 62bacae | 2013-03-05 10:40:32 -0500 | [diff] [blame] | 816 | @raises AutoservError if the image fails to boot. |
Richard Barnette | 03a0c13 | 2012-11-05 12:40:35 -0800 | [diff] [blame] | 817 | """ |
beeps | f079cfb | 2013-09-18 17:49:51 -0700 | [diff] [blame] | 818 | |
| 819 | usb_boot_timer_key = ('servo_install.usb_boot_timeout_%s' |
| 820 | % usb_boot_timeout) |
| 821 | logging.info('Downloading image to USB, then booting from it. Usb boot ' |
| 822 | 'timeout = %s', usb_boot_timeout) |
| 823 | timer = stats.Timer(usb_boot_timer_key) |
| 824 | timer.start() |
J. Richard Barnette | 31b2e31 | 2013-04-04 16:05:22 -0700 | [diff] [blame] | 825 | self.servo.install_recovery_image(image_url) |
beeps | f079cfb | 2013-09-18 17:49:51 -0700 | [diff] [blame] | 826 | if not self.wait_up(timeout=usb_boot_timeout): |
Scott Zawalski | 62bacae | 2013-03-05 10:40:32 -0500 | [diff] [blame] | 827 | raise error.AutoservRepairFailure( |
| 828 | 'DUT failed to boot from USB after %d seconds' % |
beeps | f079cfb | 2013-09-18 17:49:51 -0700 | [diff] [blame] | 829 | usb_boot_timeout) |
| 830 | timer.stop() |
Scott Zawalski | 62bacae | 2013-03-05 10:40:32 -0500 | [diff] [blame] | 831 | |
beeps | f079cfb | 2013-09-18 17:49:51 -0700 | [diff] [blame] | 832 | install_timer_key = ('servo_install.install_timeout_%s' |
| 833 | % install_timeout) |
| 834 | timer = stats.Timer(install_timer_key) |
| 835 | timer.start() |
| 836 | logging.info('Installing image through chromeos-install.') |
| 837 | self.run('chromeos-install --yes', timeout=install_timeout) |
| 838 | timer.stop() |
| 839 | |
| 840 | logging.info('Power cycling DUT through servo.') |
Richard Barnette | 03a0c13 | 2012-11-05 12:40:35 -0800 | [diff] [blame] | 841 | self.servo.power_long_press() |
Fang Deng | afb8814 | 2013-05-30 17:44:31 -0700 | [diff] [blame] | 842 | self.servo.switch_usbkey('off') |
J. Richard Barnette | fbcc712 | 2013-07-24 18:24:59 -0700 | [diff] [blame] | 843 | # We *must* use power_on() here; on Parrot it's how we get |
| 844 | # out of recovery mode. |
| 845 | self.servo.get_power_state_controller().power_on() |
beeps | f079cfb | 2013-09-18 17:49:51 -0700 | [diff] [blame] | 846 | |
| 847 | logging.info('Waiting for DUT to come back up.') |
Richard Barnette | 03a0c13 | 2012-11-05 12:40:35 -0800 | [diff] [blame] | 848 | if not self.wait_up(timeout=self.BOOT_TIMEOUT): |
| 849 | raise error.AutoservError('DUT failed to reboot installed ' |
| 850 | 'test image after %d seconds' % |
Scott Zawalski | 62bacae | 2013-03-05 10:40:32 -0500 | [diff] [blame] | 851 | self.BOOT_TIMEOUT) |
| 852 | |
| 853 | |
J. Richard Barnette | e4af8b9 | 2013-05-01 13:16:12 -0700 | [diff] [blame] | 854 | def _servo_repair_reinstall(self): |
Scott Zawalski | 62bacae | 2013-03-05 10:40:32 -0500 | [diff] [blame] | 855 | """Reinstall the DUT utilizing servo and a test image. |
| 856 | |
| 857 | Re-install the OS on the DUT by: |
| 858 | 1) installing a test image on a USB storage device attached to the Servo |
| 859 | board, |
| 860 | 2) booting that image in recovery mode, and then |
| 861 | 3) installing the image with chromeos-install. |
| 862 | |
Scott Zawalski | 62bacae | 2013-03-05 10:40:32 -0500 | [diff] [blame] | 863 | @raises AutoservRepairMethodNA if the device does not have servo |
| 864 | support. |
| 865 | |
| 866 | """ |
| 867 | if not self.servo: |
| 868 | raise error.AutoservRepairMethodNA('Repair Reinstall NA: ' |
| 869 | 'DUT has no servo support.') |
| 870 | |
| 871 | logging.info('Attempting to recovery servo enabled device with ' |
| 872 | 'servo_repair_reinstall') |
| 873 | |
J. Richard Barnette | e4af8b9 | 2013-05-01 13:16:12 -0700 | [diff] [blame] | 874 | image_url = self.stage_image_for_servo() |
Scott Zawalski | 62bacae | 2013-03-05 10:40:32 -0500 | [diff] [blame] | 875 | self.servo_install(image_url) |
| 876 | |
| 877 | |
| 878 | def _servo_repair_power(self): |
| 879 | """Attempt to repair DUT using an attached Servo. |
| 880 | |
| 881 | Attempt to power on the DUT via power_long_press. |
| 882 | |
| 883 | @raises AutoservRepairMethodNA if the device does not have servo |
| 884 | support. |
| 885 | @raises AutoservRepairFailure if the repair fails for any reason. |
| 886 | """ |
| 887 | if not self.servo: |
| 888 | raise error.AutoservRepairMethodNA('Repair Power NA: ' |
| 889 | 'DUT has no servo support.') |
| 890 | |
| 891 | logging.info('Attempting to recover servo enabled device by ' |
| 892 | 'powering it off and on.') |
| 893 | self.servo.get_power_state_controller().power_off() |
| 894 | self.servo.get_power_state_controller().power_on() |
| 895 | if self.wait_up(self.BOOT_TIMEOUT): |
| 896 | return |
| 897 | |
| 898 | raise error.AutoservRepairFailure('DUT did not boot after long_press.') |
Richard Barnette | 03a0c13 | 2012-11-05 12:40:35 -0800 | [diff] [blame] | 899 | |
| 900 | |
Richard Barnette | 82c3591 | 2012-11-20 10:09:10 -0800 | [diff] [blame] | 901 | def _powercycle_to_repair(self): |
| 902 | """Utilize the RPM Infrastructure to bring the host back up. |
| 903 | |
| 904 | If the host is not up/repaired after the first powercycle we utilize |
| 905 | auto fallback to the last good install by powercycling and rebooting the |
| 906 | host 6 times. |
Scott Zawalski | 62bacae | 2013-03-05 10:40:32 -0500 | [diff] [blame] | 907 | |
| 908 | @raises AutoservRepairMethodNA if the device does not support remote |
| 909 | power. |
| 910 | @raises AutoservRepairFailure if the repair fails for any reason. |
| 911 | |
Richard Barnette | 82c3591 | 2012-11-20 10:09:10 -0800 | [diff] [blame] | 912 | """ |
Scott Zawalski | 62bacae | 2013-03-05 10:40:32 -0500 | [diff] [blame] | 913 | if not self.has_power(): |
| 914 | raise error.AutoservRepairMethodNA('Device does not support power.') |
| 915 | |
Richard Barnette | 82c3591 | 2012-11-20 10:09:10 -0800 | [diff] [blame] | 916 | logging.info('Attempting repair via RPM powercycle.') |
| 917 | failed_cycles = 0 |
| 918 | self.power_cycle() |
| 919 | while not self.wait_up(timeout=self.BOOT_TIMEOUT): |
| 920 | failed_cycles += 1 |
| 921 | if failed_cycles >= self._MAX_POWER_CYCLE_ATTEMPTS: |
Scott Zawalski | 62bacae | 2013-03-05 10:40:32 -0500 | [diff] [blame] | 922 | raise error.AutoservRepairFailure( |
| 923 | 'Powercycled host %s %d times; device did not come back' |
| 924 | ' online.' % (self.hostname, failed_cycles)) |
Richard Barnette | 82c3591 | 2012-11-20 10:09:10 -0800 | [diff] [blame] | 925 | self.power_cycle() |
| 926 | if failed_cycles == 0: |
| 927 | logging.info('Powercycling was successful first time.') |
| 928 | else: |
| 929 | logging.info('Powercycling was successful after %d failures.', |
| 930 | failed_cycles) |
| 931 | |
| 932 | |
Prashanth B | 4d8184f | 2014-05-05 12:22:02 -0700 | [diff] [blame] | 933 | def check_device(self): |
| 934 | """Check if a device is ssh-able, and if so, clean and verify it. |
| 935 | |
| 936 | @raise AutoservSSHTimeout: If the ssh ping times out. |
| 937 | @raise AutoservSshPermissionDeniedError: If ssh ping fails due to |
| 938 | permissions. |
| 939 | @raise AutoservSshPingHostError: For other AutoservRunErrors during |
| 940 | ssh_ping. |
| 941 | @raises AutoservError: As appropriate, during cleanup and verify. |
| 942 | """ |
| 943 | self.ssh_ping() |
| 944 | self.cleanup() |
| 945 | self.verify() |
| 946 | |
| 947 | |
Richard Barnette | 82c3591 | 2012-11-20 10:09:10 -0800 | [diff] [blame] | 948 | def repair_full(self): |
| 949 | """Repair a host for repair level NO_PROTECTION. |
| 950 | |
| 951 | This overrides the base class function for repair; it does |
| 952 | not call back to the parent class, but instead offers a |
| 953 | simplified implementation based on the capabilities in the |
| 954 | Chrome OS test lab. |
| 955 | |
Fang Deng | 5d518f4 | 2013-08-02 14:04:32 -0700 | [diff] [blame] | 956 | It first verifies and repairs servo if it is a DUT in CrOS |
Fang Deng | 03590af | 2013-10-07 17:34:20 -0700 | [diff] [blame] | 957 | lab and a servo is attached. |
Fang Deng | 5d518f4 | 2013-08-02 14:04:32 -0700 | [diff] [blame] | 958 | |
Prashanth B | 4d8184f | 2014-05-05 12:22:02 -0700 | [diff] [blame] | 959 | If `self.check_device()` fails, the following procedures are |
J. Richard Barnette | fde55fc | 2013-03-15 17:47:01 -0700 | [diff] [blame] | 960 | attempted: |
| 961 | 1. Try to re-install to a known stable image using |
| 962 | auto-update. |
Scott Zawalski | 62bacae | 2013-03-05 10:40:32 -0500 | [diff] [blame] | 963 | 2. If there's a servo for the DUT, try to power the DUT off and |
| 964 | on. |
| 965 | 3. If there's a servo for the DUT, try to re-install via |
J. Richard Barnette | fde55fc | 2013-03-15 17:47:01 -0700 | [diff] [blame] | 966 | the servo. |
Scott Zawalski | 62bacae | 2013-03-05 10:40:32 -0500 | [diff] [blame] | 967 | 4. If the DUT can be power-cycled via RPM, try to repair |
Richard Barnette | 82c3591 | 2012-11-20 10:09:10 -0800 | [diff] [blame] | 968 | by power-cycling. |
| 969 | |
| 970 | As with the parent method, the last operation performed on |
Prashanth B | 4d8184f | 2014-05-05 12:22:02 -0700 | [diff] [blame] | 971 | the DUT must be to call `self.check_device()`; If that call fails the |
| 972 | exception it raises is passed back to the caller. |
J. Richard Barnette | fde55fc | 2013-03-15 17:47:01 -0700 | [diff] [blame] | 973 | |
Scott Zawalski | 62bacae | 2013-03-05 10:40:32 -0500 | [diff] [blame] | 974 | @raises AutoservRepairTotalFailure if the repair process fails to |
| 975 | fix the DUT. |
Fang Deng | 5d518f4 | 2013-08-02 14:04:32 -0700 | [diff] [blame] | 976 | @raises ServoHostRepairTotalFailure if the repair process fails to |
| 977 | fix the servo host if one is attached to the DUT. |
| 978 | @raises AutoservSshPermissionDeniedError if it is unable |
| 979 | to ssh to the servo host due to permission error. |
| 980 | |
Richard Barnette | 82c3591 | 2012-11-20 10:09:10 -0800 | [diff] [blame] | 981 | """ |
Dan Shi | 4d47852 | 2014-02-14 13:46:32 -0800 | [diff] [blame] | 982 | if self._servo_host and not self.servo: |
Fang Deng | 03590af | 2013-10-07 17:34:20 -0700 | [diff] [blame] | 983 | try: |
Dan Shi | 4d47852 | 2014-02-14 13:46:32 -0800 | [diff] [blame] | 984 | self._servo_host.repair_full() |
Fang Deng | 03590af | 2013-10-07 17:34:20 -0700 | [diff] [blame] | 985 | except Exception as e: |
Fang Deng | 03590af | 2013-10-07 17:34:20 -0700 | [diff] [blame] | 986 | logging.error('Could not create a healthy servo: %s', e) |
Dan Shi | 4d47852 | 2014-02-14 13:46:32 -0800 | [diff] [blame] | 987 | self.servo = self._servo_host.get_servo() |
Fang Deng | 5d518f4 | 2013-08-02 14:04:32 -0700 | [diff] [blame] | 988 | |
Scott Zawalski | 62bacae | 2013-03-05 10:40:32 -0500 | [diff] [blame] | 989 | # TODO(scottz): This should use something similar to label_decorator, |
| 990 | # but needs to be populated in order so DUTs are repaired with the |
| 991 | # least amount of effort. |
Dan Shi | 849a1c4 | 2014-03-05 11:10:43 -0800 | [diff] [blame] | 992 | repair_funcs = [self._servo_repair_power, |
| 993 | self._install_repair, |
Dan Shi | 2c88eed | 2013-11-12 10:18:38 -0800 | [diff] [blame] | 994 | self._install_repair_with_powerwash, |
J. Richard Barnette | e4af8b9 | 2013-05-01 13:16:12 -0700 | [diff] [blame] | 995 | self._servo_repair_reinstall, |
Scott Zawalski | 62bacae | 2013-03-05 10:40:32 -0500 | [diff] [blame] | 996 | self._powercycle_to_repair] |
| 997 | errors = [] |
Simran Basi | e613093 | 2013-10-01 14:07:52 -0700 | [diff] [blame] | 998 | board = self._get_board_from_afe() |
Scott Zawalski | 62bacae | 2013-03-05 10:40:32 -0500 | [diff] [blame] | 999 | for repair_func in repair_funcs: |
| 1000 | try: |
| 1001 | repair_func() |
Prashanth B | 4d8184f | 2014-05-05 12:22:02 -0700 | [diff] [blame] | 1002 | self.check_device() |
Simran Basi | e613093 | 2013-10-01 14:07:52 -0700 | [diff] [blame] | 1003 | stats.Counter( |
| 1004 | '%s.SUCCEEDED' % repair_func.__name__).increment() |
| 1005 | if board: |
| 1006 | stats.Counter( |
Dan Shi | b87c3aa | 2014-02-12 15:40:31 -0800 | [diff] [blame] | 1007 | '%s.%s.SUCCEEDED' % (repair_func.__name__, |
Simran Basi | e613093 | 2013-10-01 14:07:52 -0700 | [diff] [blame] | 1008 | board)).increment() |
Scott Zawalski | 62bacae | 2013-03-05 10:40:32 -0500 | [diff] [blame] | 1009 | return |
Simran Basi | e613093 | 2013-10-01 14:07:52 -0700 | [diff] [blame] | 1010 | except error.AutoservRepairMethodNA as e: |
| 1011 | stats.Counter( |
| 1012 | '%s.RepairNA' % repair_func.__name__).increment() |
| 1013 | if board: |
| 1014 | stats.Counter( |
Dan Shi | b87c3aa | 2014-02-12 15:40:31 -0800 | [diff] [blame] | 1015 | '%s.%s.RepairNA' % (repair_func.__name__, |
Simran Basi | e613093 | 2013-10-01 14:07:52 -0700 | [diff] [blame] | 1016 | board)).increment() |
Ilja H. Friedel | 04be2bd | 2014-05-07 21:29:59 -0700 | [diff] [blame] | 1017 | logging.warning('Repair function NA: %s', e) |
Simran Basi | e613093 | 2013-10-01 14:07:52 -0700 | [diff] [blame] | 1018 | errors.append(str(e)) |
Scott Zawalski | 62bacae | 2013-03-05 10:40:32 -0500 | [diff] [blame] | 1019 | except Exception as e: |
Simran Basi | e613093 | 2013-10-01 14:07:52 -0700 | [diff] [blame] | 1020 | stats.Counter( |
| 1021 | '%s.FAILED' % repair_func.__name__).increment() |
| 1022 | if board: |
| 1023 | stats.Counter( |
Dan Shi | b87c3aa | 2014-02-12 15:40:31 -0800 | [diff] [blame] | 1024 | '%s.%s.FAILED' % (repair_func.__name__, |
Simran Basi | e613093 | 2013-10-01 14:07:52 -0700 | [diff] [blame] | 1025 | board)).increment() |
Ilja H. Friedel | 04be2bd | 2014-05-07 21:29:59 -0700 | [diff] [blame] | 1026 | logging.warning('Failed to repair device: %s', e) |
Scott Zawalski | 62bacae | 2013-03-05 10:40:32 -0500 | [diff] [blame] | 1027 | errors.append(str(e)) |
Scott Zawalski | 89c44dd | 2013-02-26 09:28:02 -0500 | [diff] [blame] | 1028 | |
Simran Basi | e613093 | 2013-10-01 14:07:52 -0700 | [diff] [blame] | 1029 | stats.Counter('Full_Repair_Failed').increment() |
| 1030 | if board: |
| 1031 | stats.Counter( |
| 1032 | 'Full_Repair_Failed.%s' % board).increment() |
Scott Zawalski | 62bacae | 2013-03-05 10:40:32 -0500 | [diff] [blame] | 1033 | raise error.AutoservRepairTotalFailure( |
| 1034 | 'All attempts at repairing the device failed:\n%s' % |
| 1035 | '\n'.join(errors)) |
Richard Barnette | 82c3591 | 2012-11-20 10:09:10 -0800 | [diff] [blame] | 1036 | |
| 1037 | |
J. Richard Barnette | 1d78b01 | 2012-05-15 13:56:30 -0700 | [diff] [blame] | 1038 | def close(self): |
beeps | 32a6308 | 2013-08-22 14:02:29 -0700 | [diff] [blame] | 1039 | self.rpc_disconnect_all() |
Fang Deng | 0ca40e2 | 2013-08-27 17:47:44 -0700 | [diff] [blame] | 1040 | super(CrosHost, self).close() |
J. Richard Barnette | 1d78b01 | 2012-05-15 13:56:30 -0700 | [diff] [blame] | 1041 | |
| 1042 | |
Simran Basi | 5e6339a | 2013-03-21 11:34:32 -0700 | [diff] [blame] | 1043 | def _cleanup_poweron(self): |
| 1044 | """Special cleanup method to make sure hosts always get power back.""" |
| 1045 | afe = frontend_wrappers.RetryingAFE(timeout_min=5, delay_sec=10) |
| 1046 | hosts = afe.get_hosts(hostname=self.hostname) |
| 1047 | if not hosts or not (self._RPM_OUTLET_CHANGED in |
| 1048 | hosts[0].attributes): |
| 1049 | return |
| 1050 | logging.debug('This host has recently interacted with the RPM' |
| 1051 | ' Infrastructure. Ensuring power is on.') |
| 1052 | try: |
| 1053 | self.power_on() |
| 1054 | except rpm_client.RemotePowerException: |
| 1055 | # If cleanup has completed but there was an issue with the RPM |
| 1056 | # Infrastructure, log an error message rather than fail cleanup |
| 1057 | logging.error('Failed to turn Power On for this host after ' |
| 1058 | 'cleanup through the RPM Infrastructure.') |
| 1059 | afe.set_host_attribute(self._RPM_OUTLET_CHANGED, None, |
| 1060 | hostname=self.hostname) |
| 1061 | |
| 1062 | |
beeps | c87ff60 | 2013-07-31 21:53:00 -0700 | [diff] [blame] | 1063 | def _is_factory_image(self): |
| 1064 | """Checks if the image on the DUT is a factory image. |
| 1065 | |
| 1066 | @return: True if the image on the DUT is a factory image. |
| 1067 | False otherwise. |
| 1068 | """ |
| 1069 | result = self.run('[ -f /root/.factory_test ]', ignore_status=True) |
| 1070 | return result.exit_status == 0 |
| 1071 | |
| 1072 | |
| 1073 | def _restart_ui(self): |
J. Richard Barnette | 84890bd | 2014-02-21 11:05:47 -0800 | [diff] [blame] | 1074 | """Restart the Chrome UI. |
beeps | c87ff60 | 2013-07-31 21:53:00 -0700 | [diff] [blame] | 1075 | |
| 1076 | @raises: FactoryImageCheckerException for factory images, since |
| 1077 | we cannot attempt to restart ui on them. |
| 1078 | error.AutoservRunError for any other type of error that |
| 1079 | occurs while restarting ui. |
| 1080 | """ |
| 1081 | if self._is_factory_image(): |
| 1082 | raise FactoryImageCheckerException('Cannot restart ui on factory ' |
| 1083 | 'images') |
| 1084 | |
J. Richard Barnette | 84890bd | 2014-02-21 11:05:47 -0800 | [diff] [blame] | 1085 | # TODO(jrbarnette): The command to stop/start the ui job |
| 1086 | # should live inside cros_ui, too. However that would seem |
| 1087 | # to imply interface changes to the existing start()/restart() |
| 1088 | # functions, which is a bridge too far (for now). |
| 1089 | prompt = cros_ui.get_login_prompt_state(self) |
| 1090 | self.run('stop ui; start ui') |
| 1091 | cros_ui.wait_for_chrome_ready(prompt, self) |
beeps | c87ff60 | 2013-07-31 21:53:00 -0700 | [diff] [blame] | 1092 | |
| 1093 | |
| 1094 | def cleanup(self): |
Richard Barnette | 82c3591 | 2012-11-20 10:09:10 -0800 | [diff] [blame] | 1095 | self.run('rm -f %s' % constants.CLEANUP_LOGS_PAUSED_FILE) |
Scott Zawalski | ddbc31e | 2012-11-15 11:29:01 -0500 | [diff] [blame] | 1096 | try: |
beeps | c87ff60 | 2013-07-31 21:53:00 -0700 | [diff] [blame] | 1097 | self._restart_ui() |
| 1098 | except (error.AutotestRunError, error.AutoservRunError, |
| 1099 | FactoryImageCheckerException): |
Ilja H. Friedel | 04be2bd | 2014-05-07 21:29:59 -0700 | [diff] [blame] | 1100 | logging.warning('Unable to restart ui, rebooting device.') |
Scott Zawalski | ddbc31e | 2012-11-15 11:29:01 -0500 | [diff] [blame] | 1101 | # Since restarting the UI fails fall back to normal Autotest |
| 1102 | # cleanup routines, i.e. reboot the machine. |
Fang Deng | 0ca40e2 | 2013-08-27 17:47:44 -0700 | [diff] [blame] | 1103 | super(CrosHost, self).cleanup() |
Simran Basi | 5e6339a | 2013-03-21 11:34:32 -0700 | [diff] [blame] | 1104 | # Check if the rpm outlet was manipulated. |
Simran Basi | d5e5e27 | 2012-09-24 15:23:59 -0700 | [diff] [blame] | 1105 | if self.has_power(): |
Simran Basi | 5e6339a | 2013-03-21 11:34:32 -0700 | [diff] [blame] | 1106 | self._cleanup_poweron() |
J. Richard Barnette | 45e93de | 2012-04-11 17:24:15 -0700 | [diff] [blame] | 1107 | |
| 1108 | |
Yu-Ju Hong | a2be94a | 2012-07-31 09:48:52 -0700 | [diff] [blame] | 1109 | def reboot(self, **dargs): |
| 1110 | """ |
| 1111 | This function reboots the site host. The more generic |
| 1112 | RemoteHost.reboot() performs sync and sleeps for 5 |
| 1113 | seconds. This is not necessary for Chrome OS devices as the |
| 1114 | sync should be finished in a short time during the reboot |
| 1115 | command. |
| 1116 | """ |
Tom Wai-Hong Tam | f5cd1d4 | 2012-08-13 12:04:08 +0800 | [diff] [blame] | 1117 | if 'reboot_cmd' not in dargs: |
Doug Anderson | 7d5aeb2 | 2014-02-27 15:12:17 -0800 | [diff] [blame] | 1118 | reboot_timeout = dargs.get('reboot_timeout', 10) |
| 1119 | dargs['reboot_cmd'] = ('((reboot & sleep %d; reboot -f &)' |
| 1120 | ' </dev/null >/dev/null 2>&1 &)' % |
| 1121 | reboot_timeout) |
Yu-Ju Hong | a2be94a | 2012-07-31 09:48:52 -0700 | [diff] [blame] | 1122 | # Enable fastsync to avoid running extra sync commands before reboot. |
Tom Wai-Hong Tam | f5cd1d4 | 2012-08-13 12:04:08 +0800 | [diff] [blame] | 1123 | if 'fastsync' not in dargs: |
| 1124 | dargs['fastsync'] = True |
Michael Liang | da8c60a | 2014-06-03 13:24:51 -0700 | [diff] [blame] | 1125 | |
Charlie Mooney | a8e6dab | 2014-05-29 14:37:55 -0700 | [diff] [blame] | 1126 | # For purposes of logging reboot times: |
| 1127 | # Get the board name i.e. 'daisy_spring' |
Michael Liang | ca4f5a6 | 2014-07-10 15:45:13 -0700 | [diff] [blame] | 1128 | board_fullname = self.get_board() |
| 1129 | |
| 1130 | # Strip the prefix and add it to dargs. |
| 1131 | dargs['board'] = board_fullname[board_fullname.find(':')+1:] |
Fang Deng | 0ca40e2 | 2013-08-27 17:47:44 -0700 | [diff] [blame] | 1132 | super(CrosHost, self).reboot(**dargs) |
Yu-Ju Hong | a2be94a | 2012-07-31 09:48:52 -0700 | [diff] [blame] | 1133 | |
| 1134 | |
Gwendal Grignou | 7a61d2f | 2014-05-23 11:05:51 -0700 | [diff] [blame] | 1135 | def suspend(self, **dargs): |
| 1136 | """ |
| 1137 | This function suspends the site host. |
| 1138 | """ |
| 1139 | suspend_time = dargs.get('suspend_time', 60) |
| 1140 | dargs['timeout'] = suspend_time |
| 1141 | if 'suspend_cmd' not in dargs: |
| 1142 | cmd = ' && '.join(['echo 0 > /sys/class/rtc/rtc0/wakealarm', |
| 1143 | 'echo +%d > /sys/class/rtc/rtc0/wakealarm' % suspend_time, |
| 1144 | 'powerd_dbus_suspend --delay=0 &']) |
| 1145 | dargs['suspend_cmd'] = ('(( %s )' |
| 1146 | '< /dev/null >/dev/null 2>&1 &)' % cmd) |
| 1147 | super(CrosHost, self).suspend(**dargs) |
| 1148 | |
| 1149 | |
J. Richard Barnette | 45e93de | 2012-04-11 17:24:15 -0700 | [diff] [blame] | 1150 | def verify_software(self): |
Richard Barnette | b2bc13c | 2013-01-08 17:32:51 -0800 | [diff] [blame] | 1151 | """Verify working software on a Chrome OS system. |
J. Richard Barnette | 45e93de | 2012-04-11 17:24:15 -0700 | [diff] [blame] | 1152 | |
Richard Barnette | b2bc13c | 2013-01-08 17:32:51 -0800 | [diff] [blame] | 1153 | Tests for the following conditions: |
| 1154 | 1. All conditions tested by the parent version of this |
| 1155 | function. |
| 1156 | 2. Sufficient space in /mnt/stateful_partition. |
Fang Deng | 6b05f5b | 2013-03-20 13:42:11 -0700 | [diff] [blame] | 1157 | 3. Sufficient space in /mnt/stateful_partition/encrypted. |
| 1158 | 4. update_engine answers a simple status request over DBus. |
J. Richard Barnette | 45e93de | 2012-04-11 17:24:15 -0700 | [diff] [blame] | 1159 | |
J. Richard Barnette | 45e93de | 2012-04-11 17:24:15 -0700 | [diff] [blame] | 1160 | """ |
Fang Deng | 0ca40e2 | 2013-08-27 17:47:44 -0700 | [diff] [blame] | 1161 | super(CrosHost, self).verify_software() |
J. Richard Barnette | 45e93de | 2012-04-11 17:24:15 -0700 | [diff] [blame] | 1162 | self.check_diskspace( |
| 1163 | '/mnt/stateful_partition', |
| 1164 | global_config.global_config.get_config_value( |
Fang Deng | 6b05f5b | 2013-03-20 13:42:11 -0700 | [diff] [blame] | 1165 | 'SERVER', 'gb_diskspace_required', type=float, |
| 1166 | default=20.0)) |
Gaurav Shah | e448af8 | 2014-06-19 15:18:59 -0700 | [diff] [blame] | 1167 | encrypted_stateful_path = '/mnt/stateful_partition/encrypted' |
| 1168 | # Not all targets build with encrypted stateful support. |
| 1169 | if self.path_exists(encrypted_stateful_path): |
| 1170 | self.check_diskspace( |
| 1171 | encrypted_stateful_path, |
| 1172 | global_config.global_config.get_config_value( |
| 1173 | 'SERVER', 'gb_encrypted_diskspace_required', type=float, |
| 1174 | default=0.1)) |
beeps | c87ff60 | 2013-07-31 21:53:00 -0700 | [diff] [blame] | 1175 | |
Prashanth B | 5d0a051 | 2014-04-25 12:26:08 -0700 | [diff] [blame] | 1176 | services_status = self.run('status system-services').stdout |
| 1177 | if services_status != 'system-services start/running\n': |
| 1178 | raise error.AutoservError('Chrome failed to reach login. ' |
| 1179 | 'System services not running.') |
| 1180 | |
beeps | c87ff60 | 2013-07-31 21:53:00 -0700 | [diff] [blame] | 1181 | # Factory images don't run update engine, |
| 1182 | # goofy controls dbus on these DUTs. |
| 1183 | if not self._is_factory_image(): |
| 1184 | self.run('update_engine_client --status') |
Scott Zawalski | fbca4a9 | 2013-03-04 15:56:42 -0500 | [diff] [blame] | 1185 | # Makes sure python is present, loads and can use built in functions. |
| 1186 | # We have seen cases where importing cPickle fails with undefined |
| 1187 | # symbols in cPickle.so. |
| 1188 | self.run('python -c "import cPickle"') |
J. Richard Barnette | 134ec2c | 2012-04-25 12:59:37 -0700 | [diff] [blame] | 1189 | |
| 1190 | |
Fang Deng | 96667ca | 2013-08-01 17:46:18 -0700 | [diff] [blame] | 1191 | def make_ssh_command(self, user='root', port=22, opts='', hosts_file=None, |
| 1192 | connect_timeout=None, alive_interval=None): |
| 1193 | """Override default make_ssh_command to use options tuned for Chrome OS. |
| 1194 | |
| 1195 | Tuning changes: |
| 1196 | - ConnectTimeout=30; maximum of 30 seconds allowed for an SSH |
| 1197 | connection failure. Consistency with remote_access.sh. |
| 1198 | |
| 1199 | - ServerAliveInterval=180; which causes SSH to ping connection every |
| 1200 | 180 seconds. In conjunction with ServerAliveCountMax ensures |
| 1201 | that if the connection dies, Autotest will bail out quickly. |
| 1202 | Originally tried 60 secs, but saw frequent job ABORTS where |
| 1203 | the test completed successfully. |
| 1204 | |
| 1205 | - ServerAliveCountMax=3; consistency with remote_access.sh. |
| 1206 | |
| 1207 | - ConnectAttempts=4; reduce flakiness in connection errors; |
| 1208 | consistency with remote_access.sh. |
| 1209 | |
| 1210 | - UserKnownHostsFile=/dev/null; we don't care about the keys. |
| 1211 | Host keys change with every new installation, don't waste |
| 1212 | memory/space saving them. |
| 1213 | |
| 1214 | - SSH protocol forced to 2; needed for ServerAliveInterval. |
| 1215 | |
| 1216 | @param user User name to use for the ssh connection. |
| 1217 | @param port Port on the target host to use for ssh connection. |
| 1218 | @param opts Additional options to the ssh command. |
| 1219 | @param hosts_file Ignored. |
| 1220 | @param connect_timeout Ignored. |
| 1221 | @param alive_interval Ignored. |
| 1222 | """ |
Aviv Keshet | c5947fa | 2013-09-04 14:06:29 -0700 | [diff] [blame] | 1223 | base_command = ('/usr/bin/ssh -a -x %s %s %s' |
| 1224 | ' -o StrictHostKeyChecking=no' |
Fang Deng | 96667ca | 2013-08-01 17:46:18 -0700 | [diff] [blame] | 1225 | ' -o UserKnownHostsFile=/dev/null -o BatchMode=yes' |
| 1226 | ' -o ConnectTimeout=30 -o ServerAliveInterval=180' |
| 1227 | ' -o ServerAliveCountMax=3 -o ConnectionAttempts=4' |
| 1228 | ' -o Protocol=2 -l %s -p %d') |
Aviv Keshet | c5947fa | 2013-09-04 14:06:29 -0700 | [diff] [blame] | 1229 | return base_command % (self._ssh_verbosity_flag, self._ssh_options, |
| 1230 | opts, user, port) |
Fang Deng | 96667ca | 2013-08-01 17:46:18 -0700 | [diff] [blame] | 1231 | |
| 1232 | |
beeps | 32a6308 | 2013-08-22 14:02:29 -0700 | [diff] [blame] | 1233 | def _create_ssh_tunnel(self, port, local_port): |
| 1234 | """Create an ssh tunnel from local_port to port. |
| 1235 | |
| 1236 | @param port: remote port on the host. |
| 1237 | @param local_port: local forwarding port. |
| 1238 | |
| 1239 | @return: the tunnel process. |
| 1240 | """ |
| 1241 | # Chrome OS on the target closes down most external ports |
| 1242 | # for security. We could open the port, but doing that |
| 1243 | # would conflict with security tests that check that only |
| 1244 | # expected ports are open. So, to get to the port on the |
| 1245 | # target we use an ssh tunnel. |
| 1246 | tunnel_options = '-n -N -q -L %d:localhost:%d' % (local_port, port) |
| 1247 | ssh_cmd = self.make_ssh_command(opts=tunnel_options) |
| 1248 | tunnel_cmd = '%s %s' % (ssh_cmd, self.hostname) |
| 1249 | logging.debug('Full tunnel command: %s', tunnel_cmd) |
| 1250 | tunnel_proc = subprocess.Popen(tunnel_cmd, shell=True, close_fds=True) |
| 1251 | logging.debug('Started ssh tunnel, local = %d' |
| 1252 | ' remote = %d, pid = %d', |
| 1253 | local_port, port, tunnel_proc.pid) |
| 1254 | return tunnel_proc |
| 1255 | |
| 1256 | |
Christopher Wiley | dd18185 | 2013-10-10 19:56:58 -0700 | [diff] [blame] | 1257 | def _setup_rpc(self, port, command_name, remote_pid=None): |
beeps | 32a6308 | 2013-08-22 14:02:29 -0700 | [diff] [blame] | 1258 | """Sets up a tunnel process and performs rpc connection book keeping. |
| 1259 | |
| 1260 | This method assumes that xmlrpc and jsonrpc never conflict, since |
| 1261 | we can only either have an xmlrpc or a jsonrpc server listening on |
| 1262 | a remote port. As such, it enforces a single proxy->remote port |
| 1263 | policy, i.e if one starts a jsonrpc proxy/server from port A->B, |
| 1264 | and then tries to start an xmlrpc proxy forwarded to the same port, |
| 1265 | the xmlrpc proxy will override the jsonrpc tunnel process, however: |
| 1266 | |
| 1267 | 1. None of the methods on the xmlrpc proxy will work because |
| 1268 | the server listening on B is jsonrpc. |
| 1269 | |
| 1270 | 2. The xmlrpc client cannot initiate a termination of the JsonRPC |
| 1271 | server, as the only use case currently is goofy, which is tied to |
| 1272 | the factory image. It is much easier to handle a failed xmlrpc |
| 1273 | call on the client than it is to terminate goofy in this scenario, |
| 1274 | as doing the latter might leave the DUT in a hard to recover state. |
| 1275 | |
| 1276 | With the current implementation newer rpc proxy connections will |
| 1277 | terminate the tunnel processes of older rpc connections tunneling |
| 1278 | to the same remote port. If methods are invoked on the client |
| 1279 | after this has happened they will fail with connection closed errors. |
| 1280 | |
| 1281 | @param port: The remote forwarding port. |
| 1282 | @param command_name: The name of the remote process, to terminate |
| 1283 | using pkill. |
| 1284 | |
| 1285 | @return A url that we can use to initiate the rpc connection. |
| 1286 | """ |
| 1287 | self.rpc_disconnect(port) |
| 1288 | local_port = utils.get_unused_port() |
| 1289 | tunnel_proc = self._create_ssh_tunnel(port, local_port) |
Christopher Wiley | dd18185 | 2013-10-10 19:56:58 -0700 | [diff] [blame] | 1290 | self._rpc_proxy_map[port] = (command_name, tunnel_proc, remote_pid) |
beeps | 32a6308 | 2013-08-22 14:02:29 -0700 | [diff] [blame] | 1291 | return self._RPC_PROXY_URL % local_port |
| 1292 | |
| 1293 | |
Christopher Wiley | d78249a | 2013-03-01 13:05:31 -0800 | [diff] [blame] | 1294 | def xmlrpc_connect(self, command, port, command_name=None, |
Yusuf Mohsinally | fff89d6 | 2013-11-18 16:34:07 -0800 | [diff] [blame] | 1295 | ready_test_name=None, timeout_seconds=10, |
| 1296 | logfile='/dev/null'): |
J. Richard Barnette | 1d78b01 | 2012-05-15 13:56:30 -0700 | [diff] [blame] | 1297 | """Connect to an XMLRPC server on the host. |
| 1298 | |
| 1299 | The `command` argument should be a simple shell command that |
| 1300 | starts an XMLRPC server on the given `port`. The command |
| 1301 | must not daemonize, and must terminate cleanly on SIGTERM. |
| 1302 | The command is started in the background on the host, and a |
| 1303 | local XMLRPC client for the server is created and returned |
| 1304 | to the caller. |
| 1305 | |
| 1306 | Note that the process of creating an XMLRPC client makes no |
| 1307 | attempt to connect to the remote server; the caller is |
| 1308 | responsible for determining whether the server is running |
| 1309 | correctly, and is ready to serve requests. |
| 1310 | |
Christopher Wiley | d78249a | 2013-03-01 13:05:31 -0800 | [diff] [blame] | 1311 | Optionally, the caller can pass ready_test_name, a string |
| 1312 | containing the name of a method to call on the proxy. This |
| 1313 | method should take no parameters and return successfully only |
| 1314 | when the server is ready to process client requests. When |
| 1315 | ready_test_name is set, xmlrpc_connect will block until the |
| 1316 | proxy is ready, and throw a TestError if the server isn't |
| 1317 | ready by timeout_seconds. |
| 1318 | |
beeps | 32a6308 | 2013-08-22 14:02:29 -0700 | [diff] [blame] | 1319 | If a server is already running on the remote port, this |
| 1320 | method will kill it and disconnect the tunnel process |
| 1321 | associated with the connection before establishing a new one, |
| 1322 | by consulting the rpc_proxy_map in rpc_disconnect. |
| 1323 | |
J. Richard Barnette | 1d78b01 | 2012-05-15 13:56:30 -0700 | [diff] [blame] | 1324 | @param command Shell command to start the server. |
| 1325 | @param port Port number on which the server is expected to |
| 1326 | be serving. |
J. Richard Barnette | 7214e0b | 2013-02-06 15:20:49 -0800 | [diff] [blame] | 1327 | @param command_name String to use as input to `pkill` to |
| 1328 | terminate the XMLRPC server on the host. |
Christopher Wiley | d78249a | 2013-03-01 13:05:31 -0800 | [diff] [blame] | 1329 | @param ready_test_name String containing the name of a |
| 1330 | method defined on the XMLRPC server. |
| 1331 | @param timeout_seconds Number of seconds to wait |
| 1332 | for the server to become 'ready.' Will throw a |
| 1333 | TestFail error if server is not ready in time. |
Yusuf Mohsinally | fff89d6 | 2013-11-18 16:34:07 -0800 | [diff] [blame] | 1334 | @param logfile Logfile to send output when running |
| 1335 | 'command' argument. |
Yusuf Mohsinally | 8d19e3c | 2013-11-21 14:25:45 -0800 | [diff] [blame] | 1336 | |
J. Richard Barnette | 1d78b01 | 2012-05-15 13:56:30 -0700 | [diff] [blame] | 1337 | """ |
Christopher Wiley | c14f06a | 2013-10-16 13:55:39 -0700 | [diff] [blame] | 1338 | # Clean up any existing state. If the caller is willing |
| 1339 | # to believe their server is down, we ought to clean up |
| 1340 | # any tunnels we might have sitting around. |
| 1341 | self.rpc_disconnect(port) |
J. Richard Barnette | 1d78b01 | 2012-05-15 13:56:30 -0700 | [diff] [blame] | 1342 | # Start the server on the host. Redirection in the command |
| 1343 | # below is necessary, because 'ssh' won't terminate until |
| 1344 | # background child processes close stdin, stdout, and |
| 1345 | # stderr. |
Yusuf Mohsinally | fff89d6 | 2013-11-18 16:34:07 -0800 | [diff] [blame] | 1346 | remote_cmd = '%s </dev/null >%s 2>&1 & echo $!' % (command, logfile) |
Christopher Wiley | dd18185 | 2013-10-10 19:56:58 -0700 | [diff] [blame] | 1347 | remote_pid = self.run(remote_cmd).stdout.rstrip('\n') |
J. Richard Barnette | 1d78b01 | 2012-05-15 13:56:30 -0700 | [diff] [blame] | 1348 | logging.debug('Started XMLRPC server on host %s, pid = %s', |
| 1349 | self.hostname, remote_pid) |
| 1350 | |
Christopher Wiley | dd18185 | 2013-10-10 19:56:58 -0700 | [diff] [blame] | 1351 | # Tunnel through SSH to be able to reach that remote port. |
| 1352 | rpc_url = self._setup_rpc(port, command_name, remote_pid=remote_pid) |
Christopher Wiley | d78249a | 2013-03-01 13:05:31 -0800 | [diff] [blame] | 1353 | proxy = xmlrpclib.ServerProxy(rpc_url, allow_none=True) |
Christopher Wiley | dd18185 | 2013-10-10 19:56:58 -0700 | [diff] [blame] | 1354 | |
Christopher Wiley | d78249a | 2013-03-01 13:05:31 -0800 | [diff] [blame] | 1355 | if ready_test_name is not None: |
J. Richard Barnette | 13eb7c0 | 2013-03-07 12:06:29 -0800 | [diff] [blame] | 1356 | # retry.retry logs each attempt; calculate delay_sec to |
| 1357 | # keep log spam to a dull roar. |
Christopher Wiley | 0ed712b | 2013-04-09 15:25:12 -0700 | [diff] [blame] | 1358 | @retry.retry((socket.error, |
| 1359 | xmlrpclib.ProtocolError, |
| 1360 | httplib.BadStatusLine), |
Chris Sosa | 6542508 | 2013-10-16 13:26:22 -0700 | [diff] [blame] | 1361 | timeout_min=timeout_seconds / 60.0, |
| 1362 | delay_sec=min(max(timeout_seconds / 20.0, 0.1), 1)) |
Christopher Wiley | d78249a | 2013-03-01 13:05:31 -0800 | [diff] [blame] | 1363 | def ready_test(): |
| 1364 | """ Call proxy.ready_test_name(). """ |
| 1365 | getattr(proxy, ready_test_name)() |
| 1366 | successful = False |
| 1367 | try: |
| 1368 | logging.info('Waiting %d seconds for XMLRPC server ' |
| 1369 | 'to start.', timeout_seconds) |
| 1370 | ready_test() |
| 1371 | successful = True |
Christopher Wiley | d78249a | 2013-03-01 13:05:31 -0800 | [diff] [blame] | 1372 | finally: |
| 1373 | if not successful: |
| 1374 | logging.error('Failed to start XMLRPC server.') |
beeps | 32a6308 | 2013-08-22 14:02:29 -0700 | [diff] [blame] | 1375 | self.rpc_disconnect(port) |
Christopher Wiley | d78249a | 2013-03-01 13:05:31 -0800 | [diff] [blame] | 1376 | logging.info('XMLRPC server started successfully.') |
| 1377 | return proxy |
J. Richard Barnette | 1d78b01 | 2012-05-15 13:56:30 -0700 | [diff] [blame] | 1378 | |
J. Richard Barnette | 1d78b01 | 2012-05-15 13:56:30 -0700 | [diff] [blame] | 1379 | |
Jason Abele | b6f924f | 2013-11-13 16:01:54 -0800 | [diff] [blame] | 1380 | def syslog(self, message, tag='autotest'): |
| 1381 | """Logs a message to syslog on host. |
| 1382 | |
| 1383 | @param message String message to log into syslog |
| 1384 | @param tag String tag prefix for syslog |
| 1385 | |
| 1386 | """ |
| 1387 | self.run('logger -t "%s" "%s"' % (tag, message)) |
| 1388 | |
| 1389 | |
beeps | 32a6308 | 2013-08-22 14:02:29 -0700 | [diff] [blame] | 1390 | def jsonrpc_connect(self, port): |
| 1391 | """Creates a jsonrpc proxy connection through an ssh tunnel. |
| 1392 | |
| 1393 | This method exists to facilitate communication with goofy (which is |
| 1394 | the default system manager on all factory images) and as such, leaves |
| 1395 | most of the rpc server sanity checking to the caller. Unlike |
| 1396 | xmlrpc_connect, this method does not facilitate the creation of a remote |
| 1397 | jsonrpc server, as the only clients of this code are factory tests, |
| 1398 | for which the goofy system manager is built in to the image and starts |
| 1399 | when the target boots. |
| 1400 | |
| 1401 | One can theoretically create multiple jsonrpc proxies all forwarded |
| 1402 | to the same remote port, provided the remote port has an rpc server |
| 1403 | listening. However, in doing so we stand the risk of leaking an |
| 1404 | existing tunnel process, so we always disconnect any older tunnels |
| 1405 | we might have through rpc_disconnect. |
| 1406 | |
| 1407 | @param port: port on the remote host that is serving this proxy. |
| 1408 | |
| 1409 | @return: The client proxy. |
| 1410 | """ |
| 1411 | if not jsonrpclib: |
| 1412 | logging.warning('Jsonrpclib could not be imported. Check that ' |
| 1413 | 'site-packages contains jsonrpclib.') |
| 1414 | return None |
| 1415 | |
| 1416 | proxy = jsonrpclib.jsonrpc.ServerProxy(self._setup_rpc(port, None)) |
| 1417 | |
| 1418 | logging.info('Established a jsonrpc connection through port %s.', port) |
| 1419 | return proxy |
| 1420 | |
| 1421 | |
| 1422 | def rpc_disconnect(self, port): |
| 1423 | """Disconnect from an RPC server on the host. |
| 1424 | |
| 1425 | Terminates the remote RPC server previously started for |
J. Richard Barnette | 1d78b01 | 2012-05-15 13:56:30 -0700 | [diff] [blame] | 1426 | the given `port`. Also closes the local ssh tunnel created |
| 1427 | for the connection to the host. This function does not |
beeps | 32a6308 | 2013-08-22 14:02:29 -0700 | [diff] [blame] | 1428 | directly alter the state of a previously returned RPC |
J. Richard Barnette | 1d78b01 | 2012-05-15 13:56:30 -0700 | [diff] [blame] | 1429 | client object; however disconnection will cause all |
| 1430 | subsequent calls to methods on the object to fail. |
| 1431 | |
| 1432 | This function does nothing if requested to disconnect a port |
beeps | 32a6308 | 2013-08-22 14:02:29 -0700 | [diff] [blame] | 1433 | that was not previously connected via _setup_rpc. |
J. Richard Barnette | 1d78b01 | 2012-05-15 13:56:30 -0700 | [diff] [blame] | 1434 | |
| 1435 | @param port Port number passed to a previous call to |
beeps | 32a6308 | 2013-08-22 14:02:29 -0700 | [diff] [blame] | 1436 | `_setup_rpc()`. |
J. Richard Barnette | 1d78b01 | 2012-05-15 13:56:30 -0700 | [diff] [blame] | 1437 | """ |
beeps | 32a6308 | 2013-08-22 14:02:29 -0700 | [diff] [blame] | 1438 | if port not in self._rpc_proxy_map: |
J. Richard Barnette | 1d78b01 | 2012-05-15 13:56:30 -0700 | [diff] [blame] | 1439 | return |
Christopher Wiley | dd18185 | 2013-10-10 19:56:58 -0700 | [diff] [blame] | 1440 | remote_name, tunnel_proc, remote_pid = self._rpc_proxy_map[port] |
J. Richard Barnette | 1d78b01 | 2012-05-15 13:56:30 -0700 | [diff] [blame] | 1441 | if remote_name: |
| 1442 | # We use 'pkill' to find our target process rather than |
| 1443 | # a PID, because the host may have rebooted since |
| 1444 | # connecting, and we don't want to kill an innocent |
| 1445 | # process with the same PID. |
| 1446 | # |
| 1447 | # 'pkill' helpfully exits with status 1 if no target |
| 1448 | # process is found, for which run() will throw an |
Simran Basi | d5e5e27 | 2012-09-24 15:23:59 -0700 | [diff] [blame] | 1449 | # exception. We don't want that, so we the ignore |
J. Richard Barnette | 1d78b01 | 2012-05-15 13:56:30 -0700 | [diff] [blame] | 1450 | # status. |
| 1451 | self.run("pkill -f '%s'" % remote_name, ignore_status=True) |
Christopher Wiley | dd18185 | 2013-10-10 19:56:58 -0700 | [diff] [blame] | 1452 | if remote_pid: |
| 1453 | logging.info('Waiting for RPC server "%s" shutdown', |
| 1454 | remote_name) |
| 1455 | start_time = time.time() |
| 1456 | while (time.time() - start_time < |
| 1457 | self._RPC_SHUTDOWN_TIMEOUT_SECONDS): |
| 1458 | running_processes = self.run( |
| 1459 | "pgrep -f '%s'" % remote_name, |
| 1460 | ignore_status=True).stdout.split() |
| 1461 | if not remote_pid in running_processes: |
| 1462 | logging.info('Shut down RPC server.') |
| 1463 | break |
| 1464 | time.sleep(self._RPC_SHUTDOWN_POLLING_PERIOD_SECONDS) |
| 1465 | else: |
| 1466 | raise error.TestError('Failed to shutdown RPC server %s' % |
| 1467 | remote_name) |
J. Richard Barnette | 1d78b01 | 2012-05-15 13:56:30 -0700 | [diff] [blame] | 1468 | |
| 1469 | if tunnel_proc.poll() is None: |
| 1470 | tunnel_proc.terminate() |
| 1471 | logging.debug('Terminated tunnel, pid %d', tunnel_proc.pid) |
| 1472 | else: |
| 1473 | logging.debug('Tunnel pid %d terminated early, status %d', |
| 1474 | tunnel_proc.pid, tunnel_proc.returncode) |
beeps | 32a6308 | 2013-08-22 14:02:29 -0700 | [diff] [blame] | 1475 | del self._rpc_proxy_map[port] |
J. Richard Barnette | 1d78b01 | 2012-05-15 13:56:30 -0700 | [diff] [blame] | 1476 | |
| 1477 | |
beeps | 32a6308 | 2013-08-22 14:02:29 -0700 | [diff] [blame] | 1478 | def rpc_disconnect_all(self): |
| 1479 | """Disconnect all known RPC proxy ports.""" |
| 1480 | for port in self._rpc_proxy_map.keys(): |
| 1481 | self.rpc_disconnect(port) |
J. Richard Barnette | 1d78b01 | 2012-05-15 13:56:30 -0700 | [diff] [blame] | 1482 | |
| 1483 | |
Ilja H. Friedel | 1232e8a | 2014-06-17 21:30:48 -0700 | [diff] [blame] | 1484 | def poor_mans_rpc(self, fun): |
| 1485 | """ |
| 1486 | Calls a function from client utils on the host and returns a string. |
| 1487 | |
| 1488 | @param fun function in client utils namespace. |
| 1489 | @return output string from calling fun. |
| 1490 | """ |
Simran Basi | 263a9d3 | 2014-08-19 11:16:51 -0700 | [diff] [blame^] | 1491 | script = 'cd %s/bin; ' % autotest.Autotest.get_installed_autodir(self) |
Ilja H. Friedel | 1232e8a | 2014-06-17 21:30:48 -0700 | [diff] [blame] | 1492 | script += 'python -c "import common; import utils;' |
| 1493 | script += 'print utils.%s"' % fun |
| 1494 | return script |
| 1495 | |
| 1496 | |
J. Richard Barnette | b6de7e3 | 2013-02-14 13:28:04 -0800 | [diff] [blame] | 1497 | def _ping_check_status(self, status): |
| 1498 | """Ping the host once, and return whether it has a given status. |
J. Richard Barnette | 134ec2c | 2012-04-25 12:59:37 -0700 | [diff] [blame] | 1499 | |
J. Richard Barnette | b6de7e3 | 2013-02-14 13:28:04 -0800 | [diff] [blame] | 1500 | @param status Check the ping status against this value. |
| 1501 | @return True iff `status` and the result of ping are the same |
| 1502 | (i.e. both True or both False). |
| 1503 | |
| 1504 | """ |
| 1505 | ping_val = utils.ping(self.hostname, tries=1, deadline=1) |
| 1506 | return not (status ^ (ping_val == 0)) |
| 1507 | |
| 1508 | def _ping_wait_for_status(self, status, timeout): |
| 1509 | """Wait for the host to have a given status (UP or DOWN). |
| 1510 | |
| 1511 | Status is checked by polling. Polling will not last longer |
| 1512 | than the number of seconds in `timeout`. The polling |
| 1513 | interval will be long enough that only approximately |
| 1514 | _PING_WAIT_COUNT polling cycles will be executed, subject |
| 1515 | to a maximum interval of about one minute. |
| 1516 | |
| 1517 | @param status Waiting will stop immediately if `ping` of the |
| 1518 | host returns this status. |
| 1519 | @param timeout Poll for at most this many seconds. |
| 1520 | @return True iff the host status from `ping` matched the |
| 1521 | requested status at the time of return. |
| 1522 | |
| 1523 | """ |
| 1524 | # _ping_check_status() takes about 1 second, hence the |
| 1525 | # "- 1" in the formula below. |
| 1526 | poll_interval = min(int(timeout / self._PING_WAIT_COUNT), 60) - 1 |
| 1527 | end_time = time.time() + timeout |
| 1528 | while time.time() <= end_time: |
| 1529 | if self._ping_check_status(status): |
| 1530 | return True |
| 1531 | if poll_interval > 0: |
| 1532 | time.sleep(poll_interval) |
| 1533 | |
| 1534 | # The last thing we did was sleep(poll_interval), so it may |
| 1535 | # have been too long since the last `ping`. Check one more |
| 1536 | # time, just to be sure. |
| 1537 | return self._ping_check_status(status) |
| 1538 | |
| 1539 | def ping_wait_up(self, timeout): |
| 1540 | """Wait for the host to respond to `ping`. |
| 1541 | |
| 1542 | N.B. This method is not a reliable substitute for |
| 1543 | `wait_up()`, because a host that responds to ping will not |
| 1544 | necessarily respond to ssh. This method should only be used |
| 1545 | if the target DUT can be considered functional even if it |
| 1546 | can't be reached via ssh. |
| 1547 | |
| 1548 | @param timeout Minimum time to allow before declaring the |
| 1549 | host to be non-responsive. |
| 1550 | @return True iff the host answered to ping before the timeout. |
| 1551 | |
| 1552 | """ |
| 1553 | return self._ping_wait_for_status(self._PING_STATUS_UP, timeout) |
J. Richard Barnette | 134ec2c | 2012-04-25 12:59:37 -0700 | [diff] [blame] | 1554 | |
Andrew Bresticker | 678c0c7 | 2013-01-22 10:44:09 -0800 | [diff] [blame] | 1555 | def ping_wait_down(self, timeout): |
J. Richard Barnette | 134ec2c | 2012-04-25 12:59:37 -0700 | [diff] [blame] | 1556 | """Wait until the host no longer responds to `ping`. |
| 1557 | |
J. Richard Barnette | b6de7e3 | 2013-02-14 13:28:04 -0800 | [diff] [blame] | 1558 | This function can be used as a slightly faster version of |
| 1559 | `wait_down()`, by avoiding potentially long ssh timeouts. |
| 1560 | |
| 1561 | @param timeout Minimum time to allow for the host to become |
| 1562 | non-responsive. |
| 1563 | @return True iff the host quit answering ping before the |
| 1564 | timeout. |
| 1565 | |
J. Richard Barnette | 134ec2c | 2012-04-25 12:59:37 -0700 | [diff] [blame] | 1566 | """ |
J. Richard Barnette | b6de7e3 | 2013-02-14 13:28:04 -0800 | [diff] [blame] | 1567 | return self._ping_wait_for_status(self._PING_STATUS_DOWN, timeout) |
J. Richard Barnette | 134ec2c | 2012-04-25 12:59:37 -0700 | [diff] [blame] | 1568 | |
Tom Wai-Hong Tam | fced4f6 | 2014-04-17 10:56:30 +0800 | [diff] [blame] | 1569 | def test_wait_for_sleep(self, sleep_timeout=None): |
J. Richard Barnette | 134ec2c | 2012-04-25 12:59:37 -0700 | [diff] [blame] | 1570 | """Wait for the client to enter low-power sleep mode. |
| 1571 | |
| 1572 | The test for "is asleep" can't distinguish a system that is |
| 1573 | powered off; to confirm that the unit was asleep, it is |
| 1574 | necessary to force resume, and then call |
| 1575 | `test_wait_for_resume()`. |
| 1576 | |
| 1577 | This function is expected to be called from a test as part |
| 1578 | of a sequence like the following: |
| 1579 | |
| 1580 | ~~~~~~~~ |
| 1581 | boot_id = host.get_boot_id() |
| 1582 | # trigger sleep on the host |
| 1583 | host.test_wait_for_sleep() |
| 1584 | # trigger resume on the host |
| 1585 | host.test_wait_for_resume(boot_id) |
| 1586 | ~~~~~~~~ |
| 1587 | |
Tom Wai-Hong Tam | fced4f6 | 2014-04-17 10:56:30 +0800 | [diff] [blame] | 1588 | @param sleep_timeout time limit in seconds to allow the host sleep. |
| 1589 | |
J. Richard Barnette | 134ec2c | 2012-04-25 12:59:37 -0700 | [diff] [blame] | 1590 | @exception TestFail The host did not go to sleep within |
| 1591 | the allowed time. |
| 1592 | """ |
Tom Wai-Hong Tam | fced4f6 | 2014-04-17 10:56:30 +0800 | [diff] [blame] | 1593 | if sleep_timeout is None: |
| 1594 | sleep_timeout = self.SLEEP_TIMEOUT |
| 1595 | |
| 1596 | if not self.ping_wait_down(timeout=sleep_timeout): |
J. Richard Barnette | 134ec2c | 2012-04-25 12:59:37 -0700 | [diff] [blame] | 1597 | raise error.TestFail( |
Tom Wai-Hong Tam | fced4f6 | 2014-04-17 10:56:30 +0800 | [diff] [blame] | 1598 | 'client failed to sleep after %d seconds' % sleep_timeout) |
J. Richard Barnette | 134ec2c | 2012-04-25 12:59:37 -0700 | [diff] [blame] | 1599 | |
| 1600 | |
Tom Wai-Hong Tam | fced4f6 | 2014-04-17 10:56:30 +0800 | [diff] [blame] | 1601 | def test_wait_for_resume(self, old_boot_id, resume_timeout=None): |
J. Richard Barnette | 134ec2c | 2012-04-25 12:59:37 -0700 | [diff] [blame] | 1602 | """Wait for the client to resume from low-power sleep mode. |
| 1603 | |
| 1604 | The `old_boot_id` parameter should be the value from |
| 1605 | `get_boot_id()` obtained prior to entering sleep mode. A |
| 1606 | `TestFail` exception is raised if the boot id changes. |
| 1607 | |
| 1608 | See @ref test_wait_for_sleep for more on this function's |
| 1609 | usage. |
| 1610 | |
J. Richard Barnette | 7214e0b | 2013-02-06 15:20:49 -0800 | [diff] [blame] | 1611 | @param old_boot_id A boot id value obtained before the |
J. Richard Barnette | 134ec2c | 2012-04-25 12:59:37 -0700 | [diff] [blame] | 1612 | target host went to sleep. |
Tom Wai-Hong Tam | fced4f6 | 2014-04-17 10:56:30 +0800 | [diff] [blame] | 1613 | @param resume_timeout time limit in seconds to allow the host up. |
J. Richard Barnette | 134ec2c | 2012-04-25 12:59:37 -0700 | [diff] [blame] | 1614 | |
| 1615 | @exception TestFail The host did not respond within the |
| 1616 | allowed time. |
| 1617 | @exception TestFail The host responded, but the boot id test |
| 1618 | indicated a reboot rather than a sleep |
| 1619 | cycle. |
| 1620 | """ |
Tom Wai-Hong Tam | fced4f6 | 2014-04-17 10:56:30 +0800 | [diff] [blame] | 1621 | if resume_timeout is None: |
| 1622 | resume_timeout = self.RESUME_TIMEOUT |
| 1623 | |
| 1624 | if not self.wait_up(timeout=resume_timeout): |
J. Richard Barnette | 134ec2c | 2012-04-25 12:59:37 -0700 | [diff] [blame] | 1625 | raise error.TestFail( |
| 1626 | 'client failed to resume from sleep after %d seconds' % |
Tom Wai-Hong Tam | fced4f6 | 2014-04-17 10:56:30 +0800 | [diff] [blame] | 1627 | resume_timeout) |
J. Richard Barnette | 134ec2c | 2012-04-25 12:59:37 -0700 | [diff] [blame] | 1628 | else: |
| 1629 | new_boot_id = self.get_boot_id() |
| 1630 | if new_boot_id != old_boot_id: |
| 1631 | raise error.TestFail( |
| 1632 | 'client rebooted, but sleep was expected' |
| 1633 | ' (old boot %s, new boot %s)' |
| 1634 | % (old_boot_id, new_boot_id)) |
| 1635 | |
| 1636 | |
| 1637 | def test_wait_for_shutdown(self): |
| 1638 | """Wait for the client to shut down. |
| 1639 | |
| 1640 | The test for "has shut down" can't distinguish a system that |
| 1641 | is merely asleep; to confirm that the unit was down, it is |
| 1642 | necessary to force boot, and then call test_wait_for_boot(). |
| 1643 | |
| 1644 | This function is expected to be called from a test as part |
| 1645 | of a sequence like the following: |
| 1646 | |
| 1647 | ~~~~~~~~ |
| 1648 | boot_id = host.get_boot_id() |
| 1649 | # trigger shutdown on the host |
| 1650 | host.test_wait_for_shutdown() |
| 1651 | # trigger boot on the host |
| 1652 | host.test_wait_for_boot(boot_id) |
| 1653 | ~~~~~~~~ |
| 1654 | |
| 1655 | @exception TestFail The host did not shut down within the |
| 1656 | allowed time. |
| 1657 | """ |
Andrew Bresticker | 678c0c7 | 2013-01-22 10:44:09 -0800 | [diff] [blame] | 1658 | if not self.ping_wait_down(timeout=self.SHUTDOWN_TIMEOUT): |
J. Richard Barnette | 134ec2c | 2012-04-25 12:59:37 -0700 | [diff] [blame] | 1659 | raise error.TestFail( |
| 1660 | 'client failed to shut down after %d seconds' % |
J. Richard Barnette | eb69d72 | 2012-06-18 17:29:44 -0700 | [diff] [blame] | 1661 | self.SHUTDOWN_TIMEOUT) |
J. Richard Barnette | 134ec2c | 2012-04-25 12:59:37 -0700 | [diff] [blame] | 1662 | |
| 1663 | |
| 1664 | def test_wait_for_boot(self, old_boot_id=None): |
| 1665 | """Wait for the client to boot from cold power. |
| 1666 | |
| 1667 | The `old_boot_id` parameter should be the value from |
| 1668 | `get_boot_id()` obtained prior to shutting down. A |
| 1669 | `TestFail` exception is raised if the boot id does not |
| 1670 | change. The boot id test is omitted if `old_boot_id` is not |
| 1671 | specified. |
| 1672 | |
| 1673 | See @ref test_wait_for_shutdown for more on this function's |
| 1674 | usage. |
| 1675 | |
J. Richard Barnette | 7214e0b | 2013-02-06 15:20:49 -0800 | [diff] [blame] | 1676 | @param old_boot_id A boot id value obtained before the |
J. Richard Barnette | 134ec2c | 2012-04-25 12:59:37 -0700 | [diff] [blame] | 1677 | shut down. |
| 1678 | |
| 1679 | @exception TestFail The host did not respond within the |
| 1680 | allowed time. |
| 1681 | @exception TestFail The host responded, but the boot id test |
| 1682 | indicated that there was no reboot. |
| 1683 | """ |
J. Richard Barnette | eb69d72 | 2012-06-18 17:29:44 -0700 | [diff] [blame] | 1684 | if not self.wait_up(timeout=self.REBOOT_TIMEOUT): |
J. Richard Barnette | 134ec2c | 2012-04-25 12:59:37 -0700 | [diff] [blame] | 1685 | raise error.TestFail( |
| 1686 | 'client failed to reboot after %d seconds' % |
J. Richard Barnette | eb69d72 | 2012-06-18 17:29:44 -0700 | [diff] [blame] | 1687 | self.REBOOT_TIMEOUT) |
J. Richard Barnette | 134ec2c | 2012-04-25 12:59:37 -0700 | [diff] [blame] | 1688 | elif old_boot_id: |
| 1689 | if self.get_boot_id() == old_boot_id: |
| 1690 | raise error.TestFail( |
| 1691 | 'client is back up, but did not reboot' |
| 1692 | ' (boot %s)' % old_boot_id) |
Simran Basi | d5e5e27 | 2012-09-24 15:23:59 -0700 | [diff] [blame] | 1693 | |
| 1694 | |
| 1695 | @staticmethod |
| 1696 | def check_for_rpm_support(hostname): |
| 1697 | """For a given hostname, return whether or not it is powered by an RPM. |
| 1698 | |
Simran Basi | 1df5511 | 2013-09-06 11:25:09 -0700 | [diff] [blame] | 1699 | @param hostname: hostname to check for rpm support. |
| 1700 | |
Simran Basi | d5e5e27 | 2012-09-24 15:23:59 -0700 | [diff] [blame] | 1701 | @return None if this host does not follows the defined naming format |
| 1702 | for RPM powered DUT's in the lab. If it does follow the format, |
| 1703 | it returns a regular expression MatchObject instead. |
| 1704 | """ |
Fang Deng | 0ca40e2 | 2013-08-27 17:47:44 -0700 | [diff] [blame] | 1705 | return re.match(CrosHost._RPM_HOSTNAME_REGEX, hostname) |
Simran Basi | d5e5e27 | 2012-09-24 15:23:59 -0700 | [diff] [blame] | 1706 | |
| 1707 | |
| 1708 | def has_power(self): |
| 1709 | """For this host, return whether or not it is powered by an RPM. |
| 1710 | |
| 1711 | @return True if this host is in the CROS lab and follows the defined |
| 1712 | naming format. |
| 1713 | """ |
Fang Deng | 0ca40e2 | 2013-08-27 17:47:44 -0700 | [diff] [blame] | 1714 | return CrosHost.check_for_rpm_support(self.hostname) |
Simran Basi | d5e5e27 | 2012-09-24 15:23:59 -0700 | [diff] [blame] | 1715 | |
| 1716 | |
Ismail Noorbasha | 07fdb61 | 2013-02-14 14:13:31 -0800 | [diff] [blame] | 1717 | def _set_power(self, state, power_method): |
| 1718 | """Sets the power to the host via RPM, Servo or manual. |
| 1719 | |
| 1720 | @param state Specifies which power state to set to DUT |
| 1721 | @param power_method Specifies which method of power control to |
| 1722 | use. By default "RPM" will be used. Valid values |
| 1723 | are the strings "RPM", "manual", "servoj10". |
| 1724 | |
| 1725 | """ |
| 1726 | ACCEPTABLE_STATES = ['ON', 'OFF'] |
| 1727 | |
| 1728 | if state.upper() not in ACCEPTABLE_STATES: |
| 1729 | raise error.TestError('State must be one of: %s.' |
| 1730 | % (ACCEPTABLE_STATES,)) |
| 1731 | |
| 1732 | if power_method == self.POWER_CONTROL_SERVO: |
| 1733 | logging.info('Setting servo port J10 to %s', state) |
| 1734 | self.servo.set('prtctl3_pwren', state.lower()) |
| 1735 | time.sleep(self._USB_POWER_TIMEOUT) |
| 1736 | elif power_method == self.POWER_CONTROL_MANUAL: |
| 1737 | logging.info('You have %d seconds to set the AC power to %s.', |
| 1738 | self._POWER_CYCLE_TIMEOUT, state) |
| 1739 | time.sleep(self._POWER_CYCLE_TIMEOUT) |
| 1740 | else: |
| 1741 | if not self.has_power(): |
| 1742 | raise error.TestFail('DUT does not have RPM connected.') |
Simran Basi | 5e6339a | 2013-03-21 11:34:32 -0700 | [diff] [blame] | 1743 | afe = frontend_wrappers.RetryingAFE(timeout_min=5, delay_sec=10) |
| 1744 | afe.set_host_attribute(self._RPM_OUTLET_CHANGED, True, |
| 1745 | hostname=self.hostname) |
Simran Basi | 1df5511 | 2013-09-06 11:25:09 -0700 | [diff] [blame] | 1746 | rpm_client.set_power(self.hostname, state.upper(), timeout_mins=5) |
Simran Basi | d5e5e27 | 2012-09-24 15:23:59 -0700 | [diff] [blame] | 1747 | |
| 1748 | |
Ismail Noorbasha | 07fdb61 | 2013-02-14 14:13:31 -0800 | [diff] [blame] | 1749 | def power_off(self, power_method=POWER_CONTROL_RPM): |
| 1750 | """Turn off power to this host via RPM, Servo or manual. |
| 1751 | |
| 1752 | @param power_method Specifies which method of power control to |
| 1753 | use. By default "RPM" will be used. Valid values |
| 1754 | are the strings "RPM", "manual", "servoj10". |
| 1755 | |
| 1756 | """ |
| 1757 | self._set_power('OFF', power_method) |
Simran Basi | d5e5e27 | 2012-09-24 15:23:59 -0700 | [diff] [blame] | 1758 | |
| 1759 | |
Ismail Noorbasha | 07fdb61 | 2013-02-14 14:13:31 -0800 | [diff] [blame] | 1760 | def power_on(self, power_method=POWER_CONTROL_RPM): |
| 1761 | """Turn on power to this host via RPM, Servo or manual. |
| 1762 | |
| 1763 | @param power_method Specifies which method of power control to |
| 1764 | use. By default "RPM" will be used. Valid values |
| 1765 | are the strings "RPM", "manual", "servoj10". |
| 1766 | |
| 1767 | """ |
| 1768 | self._set_power('ON', power_method) |
| 1769 | |
| 1770 | |
| 1771 | def power_cycle(self, power_method=POWER_CONTROL_RPM): |
| 1772 | """Cycle power to this host by turning it OFF, then ON. |
| 1773 | |
| 1774 | @param power_method Specifies which method of power control to |
| 1775 | use. By default "RPM" will be used. Valid values |
| 1776 | are the strings "RPM", "manual", "servoj10". |
| 1777 | |
| 1778 | """ |
| 1779 | if power_method in (self.POWER_CONTROL_SERVO, |
| 1780 | self.POWER_CONTROL_MANUAL): |
| 1781 | self.power_off(power_method=power_method) |
| 1782 | time.sleep(self._POWER_CYCLE_TIMEOUT) |
| 1783 | self.power_on(power_method=power_method) |
| 1784 | else: |
| 1785 | rpm_client.set_power(self.hostname, 'CYCLE') |
Simran Basi | c6f1f7a | 2012-10-16 10:47:46 -0700 | [diff] [blame] | 1786 | |
| 1787 | |
| 1788 | def get_platform(self): |
| 1789 | """Determine the correct platform label for this host. |
| 1790 | |
| 1791 | @returns a string representing this host's platform. |
| 1792 | """ |
| 1793 | crossystem = utils.Crossystem(self) |
| 1794 | crossystem.init() |
| 1795 | # Extract fwid value and use the leading part as the platform id. |
| 1796 | # fwid generally follow the format of {platform}.{firmware version} |
| 1797 | # Example: Alex.X.YYY.Z or Google_Alex.X.YYY.Z |
| 1798 | platform = crossystem.fwid().split('.')[0].lower() |
| 1799 | # Newer platforms start with 'Google_' while the older ones do not. |
| 1800 | return platform.replace('google_', '') |
| 1801 | |
| 1802 | |
Hung-ying Tyan | b132803 | 2014-04-01 14:18:54 +0800 | [diff] [blame] | 1803 | def get_architecture(self): |
| 1804 | """Determine the correct architecture label for this host. |
| 1805 | |
| 1806 | @returns a string representing this host's architecture. |
| 1807 | """ |
| 1808 | crossystem = utils.Crossystem(self) |
| 1809 | crossystem.init() |
| 1810 | return crossystem.arch() |
| 1811 | |
| 1812 | |
Luis Lozano | 40b7d0d | 2014-01-17 15:12:06 -0800 | [diff] [blame] | 1813 | def get_chrome_version(self): |
| 1814 | """Gets the Chrome version number and milestone as strings. |
| 1815 | |
| 1816 | Invokes "chrome --version" to get the version number and milestone. |
| 1817 | |
| 1818 | @return A tuple (chrome_ver, milestone) where "chrome_ver" is the |
| 1819 | current Chrome version number as a string (in the form "W.X.Y.Z") |
| 1820 | and "milestone" is the first component of the version number |
| 1821 | (the "W" from "W.X.Y.Z"). If the version number cannot be parsed |
| 1822 | in the "W.X.Y.Z" format, the "chrome_ver" will be the full output |
| 1823 | of "chrome --version" and the milestone will be the empty string. |
| 1824 | |
| 1825 | """ |
| 1826 | version_string = self.run(constants.CHROME_VERSION_COMMAND).stdout |
| 1827 | return utils.parse_chrome_version(version_string) |
| 1828 | |
Aviv Keshet | 74c89a9 | 2013-02-04 15:18:30 -0800 | [diff] [blame] | 1829 | @label_decorator() |
Simran Basi | c6f1f7a | 2012-10-16 10:47:46 -0700 | [diff] [blame] | 1830 | def get_board(self): |
| 1831 | """Determine the correct board label for this host. |
| 1832 | |
| 1833 | @returns a string representing this host's board. |
| 1834 | """ |
| 1835 | release_info = utils.parse_cmd_output('cat /etc/lsb-release', |
| 1836 | run_method=self.run) |
| 1837 | board = release_info['CHROMEOS_RELEASE_BOARD'] |
| 1838 | # Devices in the lab generally have the correct board name but our own |
| 1839 | # development devices have {board_name}-signed-{key_type}. The board |
| 1840 | # name may also begin with 'x86-' which we need to keep. |
Simran Basi | 833814b | 2013-01-29 13:13:43 -0800 | [diff] [blame] | 1841 | board_format_string = ds_constants.BOARD_PREFIX + '%s' |
Simran Basi | c6f1f7a | 2012-10-16 10:47:46 -0700 | [diff] [blame] | 1842 | if 'x86' not in board: |
Simran Basi | 833814b | 2013-01-29 13:13:43 -0800 | [diff] [blame] | 1843 | return board_format_string % board.split('-')[0] |
| 1844 | return board_format_string % '-'.join(board.split('-')[0:2]) |
Simran Basi | c6f1f7a | 2012-10-16 10:47:46 -0700 | [diff] [blame] | 1845 | |
| 1846 | |
Ilja H. Friedel | 1232e8a | 2014-06-17 21:30:48 -0700 | [diff] [blame] | 1847 | @label_decorator('board_freq_mem') |
| 1848 | def get_board_with_frequency_and_memory(self): |
| 1849 | """ |
| 1850 | Determines the board name with frequency and memory. |
| 1851 | |
| 1852 | @returns a more detailed string representing the board. Examples are |
| 1853 | butterfly_1.1GHz_2GB, link_1.8GHz_4GB, x86-zgb_1.7GHz_2GB |
| 1854 | """ |
| 1855 | board = self.run(self.poor_mans_rpc( |
| 1856 | 'get_board_with_frequency_and_memory()')).stdout |
| 1857 | return 'board_freq_mem:%s' % str.strip(board) |
| 1858 | |
| 1859 | |
Aviv Keshet | 74c89a9 | 2013-02-04 15:18:30 -0800 | [diff] [blame] | 1860 | @label_decorator('lightsensor') |
Simran Basi | c6f1f7a | 2012-10-16 10:47:46 -0700 | [diff] [blame] | 1861 | def has_lightsensor(self): |
| 1862 | """Determine the correct board label for this host. |
| 1863 | |
| 1864 | @returns the string 'lightsensor' if this host has a lightsensor or |
| 1865 | None if it does not. |
| 1866 | """ |
| 1867 | search_cmd = "find -L %s -maxdepth 4 | egrep '%s'" % ( |
Richard Barnette | 82c3591 | 2012-11-20 10:09:10 -0800 | [diff] [blame] | 1868 | self._LIGHTSENSOR_SEARCH_DIR, '|'.join(self._LIGHTSENSOR_FILES)) |
Simran Basi | c6f1f7a | 2012-10-16 10:47:46 -0700 | [diff] [blame] | 1869 | try: |
| 1870 | # Run the search cmd following the symlinks. Stderr_tee is set to |
| 1871 | # None as there can be a symlink loop, but the command will still |
| 1872 | # execute correctly with a few messages printed to stderr. |
| 1873 | self.run(search_cmd, stdout_tee=None, stderr_tee=None) |
| 1874 | return 'lightsensor' |
| 1875 | except error.AutoservRunError: |
| 1876 | # egrep exited with a return code of 1 meaning none of the possible |
| 1877 | # lightsensor files existed. |
| 1878 | return None |
| 1879 | |
| 1880 | |
Aviv Keshet | 74c89a9 | 2013-02-04 15:18:30 -0800 | [diff] [blame] | 1881 | @label_decorator('bluetooth') |
Simran Basi | c6f1f7a | 2012-10-16 10:47:46 -0700 | [diff] [blame] | 1882 | def has_bluetooth(self): |
| 1883 | """Determine the correct board label for this host. |
| 1884 | |
| 1885 | @returns the string 'bluetooth' if this host has bluetooth or |
| 1886 | None if it does not. |
| 1887 | """ |
| 1888 | try: |
| 1889 | self.run('test -d /sys/class/bluetooth/hci0') |
| 1890 | # test exited with a return code of 0. |
| 1891 | return 'bluetooth' |
| 1892 | except error.AutoservRunError: |
| 1893 | # test exited with a return code 1 meaning the directory did not |
| 1894 | # exist. |
| 1895 | return None |
| 1896 | |
| 1897 | |
Ilja H. Friedel | 1232e8a | 2014-06-17 21:30:48 -0700 | [diff] [blame] | 1898 | @label_decorator('gpu_family') |
| 1899 | def get_gpu_family(self): |
| 1900 | """ |
| 1901 | Determine GPU family. |
| 1902 | |
| 1903 | @returns a string representing the gpu family. Examples are mali, tegra, |
| 1904 | pinetrail, sandybridge, ivybridge, haswell and baytrail. |
| 1905 | """ |
| 1906 | gpu_family = self.run(self.poor_mans_rpc('get_gpu_family()')).stdout |
| 1907 | return 'gpu_family:%s' % str.strip(gpu_family) |
| 1908 | |
| 1909 | |
Ilja Friedel | 0ce0b60 | 2013-08-15 18:45:27 -0700 | [diff] [blame] | 1910 | @label_decorator('graphics') |
| 1911 | def get_graphics(self): |
| 1912 | """ |
| 1913 | Determine the correct board label for this host. |
| 1914 | |
| 1915 | @returns a string representing this host's graphics. For now ARM boards |
| 1916 | return graphics:gles while all other boards return graphics:gl. This |
| 1917 | may change over time, but for robustness reasons this should avoid |
| 1918 | executing code in actual graphics libraries (which may not be ready and |
| 1919 | is tested by graphics_GLAPICheck). |
| 1920 | """ |
| 1921 | uname = self.run('uname -a').stdout.lower() |
| 1922 | if 'arm' in uname: |
| 1923 | return 'graphics:gles' |
| 1924 | return 'graphics:gl' |
| 1925 | |
| 1926 | |
Bill Richardson | 4f595f5 | 2014-02-13 16:20:26 -0800 | [diff] [blame] | 1927 | @label_decorator('ec') |
| 1928 | def get_ec(self): |
| 1929 | """ |
| 1930 | Determine the type of EC on this host. |
| 1931 | |
| 1932 | @returns a string representing this host's embedded controller type. |
| 1933 | At present, it only returns "ec:cros", for Chrome OS ECs. Other types |
| 1934 | of EC (or none) don't return any strings, since no tests depend on |
| 1935 | those. |
| 1936 | """ |
| 1937 | cmd = 'mosys ec info' |
| 1938 | # The output should look like these, so that the last field should |
| 1939 | # match our EC version scheme: |
| 1940 | # |
| 1941 | # stm | stm32f100 | snow_v1.3.139-375eb9f |
| 1942 | # ti | Unknown-10de | peppy_v1.5.114-5d52788 |
| 1943 | # |
| 1944 | # Non-Chrome OS ECs will look like these: |
| 1945 | # |
| 1946 | # ENE | KB932 | 00BE107A00 |
| 1947 | # ite | it8518 | 3.08 |
| 1948 | # |
| 1949 | # And some systems don't have ECs at all (Lumpy, for example). |
| 1950 | regexp = r'^.*\|\s*(\S+_v\d+\.\d+\.\d+-[0-9a-f]+)\s*$' |
| 1951 | |
| 1952 | ecinfo = self.run(command=cmd, ignore_status=True) |
| 1953 | if ecinfo.exit_status == 0: |
| 1954 | res = re.search(regexp, ecinfo.stdout) |
| 1955 | if res: |
| 1956 | logging.info("EC version is %s", res.groups()[0]) |
| 1957 | return 'ec:cros' |
| 1958 | logging.info("%s got: %s", cmd, ecinfo.stdout) |
| 1959 | # Has an EC, but it's not a Chrome OS EC |
| 1960 | return None |
| 1961 | logging.info("%s exited with status %d", cmd, ecinfo.exit_status) |
| 1962 | # No EC present |
| 1963 | return None |
| 1964 | |
| 1965 | |
Alec Berg | 31b932b | 2014-04-04 16:09:11 -0700 | [diff] [blame] | 1966 | @label_decorator('accels') |
| 1967 | def get_accels(self): |
| 1968 | """ |
| 1969 | Determine the type of accelerometers on this host. |
| 1970 | |
| 1971 | @returns a string representing this host's accelerometer type. |
| 1972 | At present, it only returns "accel:cros-ec", for accelerometers |
| 1973 | attached to a Chrome OS EC, or none, if no accelerometers. |
| 1974 | """ |
| 1975 | # Check to make sure we have ectool |
| 1976 | rv = self.run('which ectool', ignore_status=True) |
| 1977 | if rv.exit_status: |
| 1978 | logging.info("No ectool cmd found, assuming no EC accelerometers") |
| 1979 | return None |
| 1980 | |
| 1981 | # Check that the EC supports the motionsense command |
| 1982 | rv = self.run('ectool motionsense', ignore_status=True) |
| 1983 | if rv.exit_status: |
| 1984 | logging.info("EC does not support motionsense command " |
| 1985 | "assuming no EC accelerometers") |
| 1986 | return None |
| 1987 | |
| 1988 | # Check that EC motion sensors are active |
| 1989 | active = self.run('ectool motionsense active').stdout.split('\n') |
| 1990 | if active[0] == "0": |
| 1991 | logging.info("Motion sense inactive, assuming no EC accelerometers") |
| 1992 | return None |
| 1993 | |
| 1994 | logging.info("EC accelerometers found") |
| 1995 | return 'accel:cros-ec' |
| 1996 | |
| 1997 | |
Tom Wai-Hong Tam | 3d6790d | 2014-04-14 16:15:47 +0800 | [diff] [blame] | 1998 | @label_decorator('chameleon') |
| 1999 | def has_chameleon(self): |
| 2000 | """Determine if a Chameleon connected to this host. |
| 2001 | |
Tom Wai-Hong Tam | 3d75ebc | 2014-08-12 08:57:25 +0800 | [diff] [blame] | 2002 | @returns the string 'chameleon:' + label, e.g. 'chameleon:hdmi', |
| 2003 | if this host has a Chameleon or None if it has not. |
Tom Wai-Hong Tam | 3d6790d | 2014-04-14 16:15:47 +0800 | [diff] [blame] | 2004 | """ |
| 2005 | if self._chameleon_host: |
Tom Wai-Hong Tam | 3d75ebc | 2014-08-12 08:57:25 +0800 | [diff] [blame] | 2006 | return 'chameleon:' + self.chameleon.get_label() |
Tom Wai-Hong Tam | 3d6790d | 2014-04-14 16:15:47 +0800 | [diff] [blame] | 2007 | else: |
| 2008 | return None |
| 2009 | |
| 2010 | |
Derek Basehore | c71ff62 | 2014-07-07 15:18:40 -0700 | [diff] [blame] | 2011 | @label_decorator('power_supply') |
| 2012 | def get_power_supply(self): |
| 2013 | """ |
| 2014 | Determine what type of power supply the host has |
| 2015 | |
| 2016 | @returns a string representing this host's power supply. |
| 2017 | 'power:battery' when the device has a battery intended for |
| 2018 | extended use |
| 2019 | 'power:AC_primary' when the device has a battery not intended |
| 2020 | for extended use (for moving the machine, etc) |
| 2021 | 'power:AC_only' when the device has no battery at all. |
| 2022 | """ |
| 2023 | psu = self.run(command='mosys psu type', ignore_status=True) |
| 2024 | if psu.exit_status: |
| 2025 | # The psu command for mosys is not included for all platforms. The |
| 2026 | # assumption is that the device will have a battery if the command |
| 2027 | # is not found. |
| 2028 | return 'power:battery' |
| 2029 | |
| 2030 | psu_str = psu.stdout.strip() |
| 2031 | if psu_str == 'unknown': |
| 2032 | return None |
| 2033 | |
| 2034 | return 'power:%s' % psu_str |
| 2035 | |
| 2036 | |
Puthikorn Voravootivat | fa01124 | 2014-03-14 18:45:11 -0700 | [diff] [blame] | 2037 | @label_decorator('storage') |
| 2038 | def get_storage(self): |
| 2039 | """ |
| 2040 | Determine the type of boot device for this host. |
| 2041 | |
| 2042 | Determine if the internal device is SCSI or dw_mmc device. |
| 2043 | Then check that it is SSD or HDD or eMMC or something else. |
| 2044 | |
| 2045 | @returns a string representing this host's internal device type. |
| 2046 | 'storage:ssd' when internal device is solid state drive |
| 2047 | 'storage:hdd' when internal device is hard disk drive |
| 2048 | 'storage:mmc' when internal device is mmc drive |
| 2049 | None When internal device is something else or |
| 2050 | when we are unable to determine the type |
| 2051 | """ |
| 2052 | # The output should be /dev/mmcblk* for SD/eMMC or /dev/sd* for scsi |
| 2053 | rootdev_cmd = ' '.join(['. /usr/sbin/write_gpt.sh;', |
| 2054 | '. /usr/share/misc/chromeos-common.sh;', |
| 2055 | 'load_base_vars;', |
| 2056 | 'get_fixed_dst_drive']) |
Puthikorn Voravootivat | 03c5168 | 2014-04-24 13:52:12 -0700 | [diff] [blame] | 2057 | rootdev = self.run(command=rootdev_cmd, ignore_status=True) |
| 2058 | if rootdev.exit_status: |
| 2059 | logging.info("Fail to run %s", rootdev_cmd) |
| 2060 | return None |
Puthikorn Voravootivat | fa01124 | 2014-03-14 18:45:11 -0700 | [diff] [blame] | 2061 | rootdev_str = rootdev.stdout.strip() |
| 2062 | |
| 2063 | if not rootdev_str: |
| 2064 | return None |
| 2065 | |
| 2066 | rootdev_base = os.path.basename(rootdev_str) |
| 2067 | |
| 2068 | mmc_pattern = '/dev/mmcblk[0-9]' |
| 2069 | if re.match(mmc_pattern, rootdev_str): |
| 2070 | # Use type to determine if the internal device is eMMC or somthing |
| 2071 | # else. We can assume that MMC is always an internal device. |
| 2072 | type_cmd = 'cat /sys/block/%s/device/type' % rootdev_base |
Puthikorn Voravootivat | 03c5168 | 2014-04-24 13:52:12 -0700 | [diff] [blame] | 2073 | type = self.run(command=type_cmd, ignore_status=True) |
| 2074 | if type.exit_status: |
| 2075 | logging.info("Fail to run %s", type_cmd) |
| 2076 | return None |
Puthikorn Voravootivat | fa01124 | 2014-03-14 18:45:11 -0700 | [diff] [blame] | 2077 | type_str = type.stdout.strip() |
| 2078 | |
| 2079 | if type_str == 'MMC': |
| 2080 | return 'storage:mmc' |
| 2081 | |
| 2082 | scsi_pattern = '/dev/sd[a-z]+' |
| 2083 | if re.match(scsi_pattern, rootdev.stdout): |
| 2084 | # Read symlink for /sys/block/sd* to determine if the internal |
| 2085 | # device is connected via ata or usb. |
| 2086 | link_cmd = 'readlink /sys/block/%s' % rootdev_base |
Puthikorn Voravootivat | 03c5168 | 2014-04-24 13:52:12 -0700 | [diff] [blame] | 2087 | link = self.run(command=link_cmd, ignore_status=True) |
| 2088 | if link.exit_status: |
| 2089 | logging.info("Fail to run %s", link_cmd) |
| 2090 | return None |
Puthikorn Voravootivat | fa01124 | 2014-03-14 18:45:11 -0700 | [diff] [blame] | 2091 | link_str = link.stdout.strip() |
| 2092 | if 'usb' in link_str: |
| 2093 | return None |
| 2094 | |
| 2095 | # Read rotation to determine if the internal device is ssd or hdd. |
| 2096 | rotate_cmd = str('cat /sys/block/%s/queue/rotational' |
| 2097 | % rootdev_base) |
Puthikorn Voravootivat | 03c5168 | 2014-04-24 13:52:12 -0700 | [diff] [blame] | 2098 | rotate = self.run(command=rotate_cmd, ignore_status=True) |
| 2099 | if rotate.exit_status: |
| 2100 | logging.info("Fail to run %s", rotate_cmd) |
| 2101 | return None |
Puthikorn Voravootivat | fa01124 | 2014-03-14 18:45:11 -0700 | [diff] [blame] | 2102 | rotate_str = rotate.stdout.strip() |
| 2103 | |
| 2104 | rotate_dict = {'0':'storage:ssd', '1':'storage:hdd'} |
| 2105 | return rotate_dict.get(rotate_str) |
| 2106 | |
| 2107 | # All other internal device / error case will always fall here |
| 2108 | return None |
| 2109 | |
| 2110 | |
Dan Shi | 4e9a2aa | 2014-03-24 14:28:42 -0700 | [diff] [blame] | 2111 | @label_decorator('servo') |
| 2112 | def get_servo(self): |
| 2113 | """Determine if the host has a servo attached. |
| 2114 | |
| 2115 | If the host has a working servo attached, it should have a servo label. |
| 2116 | |
| 2117 | @return: string 'servo' if the host has servo attached. Otherwise, |
| 2118 | returns None. |
| 2119 | """ |
| 2120 | return 'servo' if self._servo_host else None |
| 2121 | |
| 2122 | |
Dan Shi | 5beba47 | 2014-05-28 22:46:07 -0700 | [diff] [blame] | 2123 | @label_decorator('video_labels') |
| 2124 | def get_video_labels(self): |
| 2125 | """Run /usr/local/bin/avtest_label_detect to get a list of video labels. |
| 2126 | |
| 2127 | Sample output of avtest_label_detect: |
| 2128 | Detected label: hw_video_acc_vp8 |
| 2129 | Detected label: webcam |
| 2130 | |
| 2131 | @return: A list of labels detected by tool avtest_label_detect. |
| 2132 | """ |
| 2133 | try: |
Simran Basi | 40ca818 | 2014-07-17 18:41:20 -0700 | [diff] [blame] | 2134 | # TODO (sbasi) crbug.com/391081 - Remove once the proper fix has |
| 2135 | # landed and supporting images older than the fix is no longer |
| 2136 | # necessary. |
| 2137 | # Change back to VT1 so avtest_label_detect does not get stuck. |
| 2138 | self.run('chvt 1') |
Dan Shi | 5beba47 | 2014-05-28 22:46:07 -0700 | [diff] [blame] | 2139 | result = self.run('/usr/local/bin/avtest_label_detect').stdout |
| 2140 | return re.findall('^Detected label: (\w+)$', result, re.M) |
| 2141 | except error.AutoservRunError: |
| 2142 | # The tool is not installed. |
| 2143 | return [] |
| 2144 | |
| 2145 | |
mussa | 584b446 | 2014-06-20 15:13:28 -0700 | [diff] [blame] | 2146 | @label_decorator('video_glitch_detection') |
| 2147 | def is_video_glitch_detection_supported(self): |
| 2148 | """ Determine if a board under test is supported for video glitch |
| 2149 | detection tests. |
| 2150 | |
| 2151 | @return: 'video_glitch_detection' if board is supported, None otherwise. |
| 2152 | """ |
| 2153 | parser = ConfigParser.SafeConfigParser() |
| 2154 | filename = os.path.join( |
| 2155 | common.autotest_dir, 'client/cros/video/device_spec.conf') |
| 2156 | |
| 2157 | dut = self.get_board().replace(ds_constants.BOARD_PREFIX, '') |
| 2158 | |
| 2159 | try: |
| 2160 | parser.read(filename) |
mussa | 584b446 | 2014-06-20 15:13:28 -0700 | [diff] [blame] | 2161 | supported_boards = parser.sections() |
| 2162 | |
Mussa | 2cba43a | 2014-07-24 10:38:08 -0700 | [diff] [blame] | 2163 | # Some boards have multiple resolutions. e.g: nyan_big has standard |
| 2164 | # and high definitions. The conf file has something like nyan_big_sd |
| 2165 | |
| 2166 | for board in supported_boards: |
| 2167 | if board.startswith(dut): |
| 2168 | return 'video_glitch_detection' |
| 2169 | |
| 2170 | return None |
mussa | 584b446 | 2014-06-20 15:13:28 -0700 | [diff] [blame] | 2171 | |
| 2172 | except ConfigParser.error: |
| 2173 | # something went wrong while parsing the conf file |
| 2174 | return None |
| 2175 | |
Katherine Threlkeld | 7b97a9f | 2014-06-24 13:47:14 -0700 | [diff] [blame] | 2176 | @label_decorator('touch_labels') |
| 2177 | def get_touch(self): |
| 2178 | """ |
| 2179 | Determine whether board under test has a touchpad or touchscreen. |
| 2180 | |
| 2181 | @return: A list of some combination of 'touchscreen' and 'touchpad', |
| 2182 | depending on what is present on the device. |
| 2183 | """ |
| 2184 | labels = [] |
| 2185 | input_cmd = '/opt/google/input/inputcontrol --names -t %s' |
| 2186 | for elt in ['touchpad', 'touchscreen']: |
| 2187 | if self.run(input_cmd % elt).stdout: |
| 2188 | labels.append(elt) |
| 2189 | return labels |
| 2190 | |
| 2191 | |
mussa | 584b446 | 2014-06-20 15:13:28 -0700 | [diff] [blame] | 2192 | |
Simran Basi | c6f1f7a | 2012-10-16 10:47:46 -0700 | [diff] [blame] | 2193 | def get_labels(self): |
| 2194 | """Return a list of labels for this given host. |
| 2195 | |
| 2196 | This is the main way to retrieve all the automatic labels for a host |
| 2197 | as it will run through all the currently implemented label functions. |
| 2198 | """ |
| 2199 | labels = [] |
Richard Barnette | 82c3591 | 2012-11-20 10:09:10 -0800 | [diff] [blame] | 2200 | for label_function in self._LABEL_FUNCTIONS: |
Simran Basi | c6f1f7a | 2012-10-16 10:47:46 -0700 | [diff] [blame] | 2201 | label = label_function(self) |
| 2202 | if label: |
Dan Shi | 5beba47 | 2014-05-28 22:46:07 -0700 | [diff] [blame] | 2203 | if type(label) is str: |
| 2204 | labels.append(label) |
| 2205 | elif type(label) is list: |
| 2206 | labels.extend(label) |
Simran Basi | c6f1f7a | 2012-10-16 10:47:46 -0700 | [diff] [blame] | 2207 | return labels |
Dan Shi | 85276d4 | 2014-04-08 22:11:45 -0700 | [diff] [blame] | 2208 | |
| 2209 | |
| 2210 | def is_boot_from_usb(self): |
| 2211 | """Check if DUT is boot from USB. |
| 2212 | |
| 2213 | @return: True if DUT is boot from usb. |
| 2214 | """ |
| 2215 | device = self.run('rootdev -s -d').stdout.strip() |
| 2216 | removable = int(self.run('cat /sys/block/%s/removable' % |
| 2217 | os.path.basename(device)).stdout.strip()) |
| 2218 | return removable == 1 |