Dan Shi | a1ecd5c | 2013-06-06 11:21:31 -0700 | [diff] [blame] | 1 | # Copyright (c) 2013 The Chromium Authors. All rights reserved. |
| 2 | # Use of this source code is governed by a BSD-style license that can be |
| 3 | # found in the LICENSE file. |
| 4 | |
Dan Shi | a1ecd5c | 2013-06-06 11:21:31 -0700 | [diff] [blame] | 5 | |
J. Richard Barnette | 3cbd76b | 2013-11-27 12:11:25 -0800 | [diff] [blame] | 6 | import httplib |
| 7 | import json |
Alex Miller | dadc2c2 | 2013-07-08 15:21:21 -0700 | [diff] [blame] | 8 | import logging |
MK Ryu | 35d661e | 2014-09-25 17:44:10 -0700 | [diff] [blame] | 9 | import os |
beeps | 023afc6 | 2014-02-04 16:59:22 -0800 | [diff] [blame] | 10 | import random |
Alex Miller | dadc2c2 | 2013-07-08 15:21:21 -0700 | [diff] [blame] | 11 | import re |
J. Richard Barnette | 3cbd76b | 2013-11-27 12:11:25 -0800 | [diff] [blame] | 12 | import time |
Paul Drews | bef578d | 2013-09-24 15:10:36 -0700 | [diff] [blame] | 13 | import urllib2 |
Alex Miller | dadc2c2 | 2013-07-08 15:21:21 -0700 | [diff] [blame] | 14 | |
J. Richard Barnette | 3cbd76b | 2013-11-27 12:11:25 -0800 | [diff] [blame] | 15 | import common |
beeps | 023afc6 | 2014-02-04 16:59:22 -0800 | [diff] [blame] | 16 | from autotest_lib.client.common_lib import base_utils |
| 17 | from autotest_lib.client.common_lib import error |
| 18 | from autotest_lib.client.common_lib import global_config |
MK Ryu | 0c1a37d | 2015-04-30 12:00:55 -0700 | [diff] [blame] | 19 | from autotest_lib.client.common_lib import host_queue_entry_states |
Dan Shi | a1ecd5c | 2013-06-06 11:21:31 -0700 | [diff] [blame] | 20 | from autotest_lib.server.cros.dynamic_suite import constants |
Dan Shi | 7e04fa8 | 2013-07-25 15:08:48 -0700 | [diff] [blame] | 21 | from autotest_lib.server.cros.dynamic_suite import job_status |
Dan Shi | a1ecd5c | 2013-06-06 11:21:31 -0700 | [diff] [blame] | 22 | |
Dan Shi | 82997b9 | 2015-05-06 12:08:02 -0700 | [diff] [blame^] | 23 | try: |
| 24 | from chromite.lib import cros_build_lib |
| 25 | except ImportError: |
| 26 | logging.warn('Unable to import chromite.') |
| 27 | # Init the module variable to None. Access to this module can check if it |
| 28 | # is not None before making calls. |
| 29 | cros_build_lib = None |
| 30 | |
Dan Shi | a1ecd5c | 2013-06-06 11:21:31 -0700 | [diff] [blame] | 31 | |
Alex Miller | dadc2c2 | 2013-07-08 15:21:21 -0700 | [diff] [blame] | 32 | _SHERIFF_JS = global_config.global_config.get_config_value( |
| 33 | 'NOTIFICATIONS', 'sheriffs', default='') |
Fang Deng | 3197b39 | 2013-06-26 11:42:02 -0700 | [diff] [blame] | 34 | _LAB_SHERIFF_JS = global_config.global_config.get_config_value( |
| 35 | 'NOTIFICATIONS', 'lab_sheriffs', default='') |
Alex Miller | dadc2c2 | 2013-07-08 15:21:21 -0700 | [diff] [blame] | 36 | _CHROMIUM_BUILD_URL = global_config.global_config.get_config_value( |
| 37 | 'NOTIFICATIONS', 'chromium_build_url', default='') |
| 38 | |
J. Richard Barnette | 3cbd76b | 2013-11-27 12:11:25 -0800 | [diff] [blame] | 39 | LAB_GOOD_STATES = ('open', 'throttled') |
| 40 | |
| 41 | |
J. Richard Barnette | abbe096 | 2013-12-10 18:15:44 -0800 | [diff] [blame] | 42 | class TestLabException(Exception): |
| 43 | """Exception raised when the Test Lab blocks a test or suite.""" |
J. Richard Barnette | 3cbd76b | 2013-11-27 12:11:25 -0800 | [diff] [blame] | 44 | pass |
| 45 | |
| 46 | |
| 47 | class ParseBuildNameException(Exception): |
| 48 | """Raised when ParseBuildName() cannot parse a build name.""" |
| 49 | pass |
| 50 | |
| 51 | |
| 52 | def ParseBuildName(name): |
| 53 | """Format a build name, given board, type, milestone, and manifest num. |
| 54 | |
Simran Basi | b7d2116 | 2014-05-21 15:26:16 -0700 | [diff] [blame] | 55 | @param name: a build name, e.g. 'x86-alex-release/R20-2015.0.0' or a |
| 56 | relative build name, e.g. 'x86-alex-release/LATEST' |
J. Richard Barnette | 3cbd76b | 2013-11-27 12:11:25 -0800 | [diff] [blame] | 57 | |
| 58 | @return board: board the manifest is for, e.g. x86-alex. |
| 59 | @return type: one of 'release', 'factory', or 'firmware' |
| 60 | @return milestone: (numeric) milestone the manifest was associated with. |
Simran Basi | b7d2116 | 2014-05-21 15:26:16 -0700 | [diff] [blame] | 61 | Will be None for relative build names. |
| 62 | @return manifest: manifest number, e.g. '2015.0.0'. |
| 63 | Will be None for relative build names. |
J. Richard Barnette | 3cbd76b | 2013-11-27 12:11:25 -0800 | [diff] [blame] | 64 | |
| 65 | """ |
Simran Basi | f8f648e | 2014-09-09 11:40:03 -0700 | [diff] [blame] | 66 | match = re.match(r'(trybot-)?(?P<board>[\w-]+)-(?P<type>\w+)/' |
| 67 | r'(R(?P<milestone>\d+)-(?P<manifest>[\d.ab-]+)|LATEST)', |
| 68 | name) |
| 69 | if match and len(match.groups()) >= 5: |
Simran Basi | b7d2116 | 2014-05-21 15:26:16 -0700 | [diff] [blame] | 70 | return (match.group('board'), match.group('type'), |
| 71 | match.group('milestone'), match.group('manifest')) |
J. Richard Barnette | 3cbd76b | 2013-11-27 12:11:25 -0800 | [diff] [blame] | 72 | raise ParseBuildNameException('%s is a malformed build name.' % name) |
| 73 | |
Alex Miller | dadc2c2 | 2013-07-08 15:21:21 -0700 | [diff] [blame] | 74 | |
Dan Shi | a1ecd5c | 2013-06-06 11:21:31 -0700 | [diff] [blame] | 75 | def get_label_from_afe(hostname, label_prefix, afe): |
| 76 | """Retrieve a host's specific label from the AFE. |
| 77 | |
| 78 | Looks for a host label that has the form <label_prefix>:<value> |
| 79 | and returns the "<value>" part of the label. None is returned |
| 80 | if there is not a label matching the pattern |
| 81 | |
| 82 | @param hostname: hostname of given DUT. |
| 83 | @param label_prefix: prefix of label to be matched, e.g., |board:| |
| 84 | @param afe: afe instance. |
| 85 | @returns the label that matches the prefix or 'None' |
| 86 | |
| 87 | """ |
| 88 | labels = afe.get_labels(name__startswith=label_prefix, |
| 89 | host__hostname__in=[hostname]) |
| 90 | if labels and len(labels) == 1: |
| 91 | return labels[0].name.split(label_prefix, 1)[1] |
| 92 | |
| 93 | |
| 94 | def get_board_from_afe(hostname, afe): |
| 95 | """Retrieve given host's board from its labels in the AFE. |
| 96 | |
| 97 | Looks for a host label of the form "board:<board>", and |
| 98 | returns the "<board>" part of the label. `None` is returned |
| 99 | if there is not a single, unique label matching the pattern. |
| 100 | |
| 101 | @param hostname: hostname of given DUT. |
| 102 | @param afe: afe instance. |
| 103 | @returns board from label, or `None`. |
| 104 | |
| 105 | """ |
| 106 | return get_label_from_afe(hostname, constants.BOARD_PREFIX, afe) |
| 107 | |
| 108 | |
| 109 | def get_build_from_afe(hostname, afe): |
| 110 | """Retrieve the current build for given host from the AFE. |
| 111 | |
| 112 | Looks through the host's labels in the AFE to determine its build. |
| 113 | |
| 114 | @param hostname: hostname of given DUT. |
| 115 | @param afe: afe instance. |
| 116 | @returns The current build or None if it could not find it or if there |
| 117 | were multiple build labels assigned to this host. |
| 118 | |
| 119 | """ |
| 120 | return get_label_from_afe(hostname, constants.VERSION_PREFIX, afe) |
| 121 | |
| 122 | |
Fang Deng | 3197b39 | 2013-06-26 11:42:02 -0700 | [diff] [blame] | 123 | def get_sheriffs(lab_only=False): |
Alex Miller | dadc2c2 | 2013-07-08 15:21:21 -0700 | [diff] [blame] | 124 | """ |
| 125 | Polls the javascript file that holds the identity of the sheriff and |
| 126 | parses it's output to return a list of chromium sheriff email addresses. |
| 127 | The javascript file can contain the ldap of more than one sheriff, eg: |
| 128 | document.write('sheriff_one, sheriff_two'). |
| 129 | |
Fang Deng | 3197b39 | 2013-06-26 11:42:02 -0700 | [diff] [blame] | 130 | @param lab_only: if True, only pulls lab sheriff. |
| 131 | @return: A list of chroium.org sheriff email addresses to cc on the bug. |
| 132 | An empty list if failed to parse the javascript. |
Alex Miller | dadc2c2 | 2013-07-08 15:21:21 -0700 | [diff] [blame] | 133 | """ |
| 134 | sheriff_ids = [] |
Fang Deng | 3197b39 | 2013-06-26 11:42:02 -0700 | [diff] [blame] | 135 | sheriff_js_list = _LAB_SHERIFF_JS.split(',') |
| 136 | if not lab_only: |
| 137 | sheriff_js_list.extend(_SHERIFF_JS.split(',')) |
| 138 | |
| 139 | for sheriff_js in sheriff_js_list: |
Alex Miller | dadc2c2 | 2013-07-08 15:21:21 -0700 | [diff] [blame] | 140 | try: |
| 141 | url_content = base_utils.urlopen('%s%s'% ( |
| 142 | _CHROMIUM_BUILD_URL, sheriff_js)).read() |
| 143 | except (ValueError, IOError) as e: |
beeps | 4efdf03 | 2013-09-17 11:27:14 -0700 | [diff] [blame] | 144 | logging.warning('could not parse sheriff from url %s%s: %s', |
| 145 | _CHROMIUM_BUILD_URL, sheriff_js, str(e)) |
Paul Drews | bef578d | 2013-09-24 15:10:36 -0700 | [diff] [blame] | 146 | except (urllib2.URLError, httplib.HTTPException) as e: |
| 147 | logging.warning('unexpected error reading from url "%s%s": %s', |
| 148 | _CHROMIUM_BUILD_URL, sheriff_js, str(e)) |
Alex Miller | dadc2c2 | 2013-07-08 15:21:21 -0700 | [diff] [blame] | 149 | else: |
| 150 | ldaps = re.search(r"document.write\('(.*)'\)", url_content) |
| 151 | if not ldaps: |
beeps | 4efdf03 | 2013-09-17 11:27:14 -0700 | [diff] [blame] | 152 | logging.warning('Could not retrieve sheriff ldaps for: %s', |
| 153 | url_content) |
Alex Miller | dadc2c2 | 2013-07-08 15:21:21 -0700 | [diff] [blame] | 154 | continue |
| 155 | sheriff_ids += ['%s@chromium.org' % alias.replace(' ', '') |
| 156 | for alias in ldaps.group(1).split(',')] |
| 157 | return sheriff_ids |
beeps | 46dadc9 | 2013-11-07 14:07:10 -0800 | [diff] [blame] | 158 | |
| 159 | |
| 160 | def remote_wget(source_url, dest_path, ssh_cmd): |
| 161 | """wget source_url from localhost to dest_path on remote host using ssh. |
| 162 | |
| 163 | @param source_url: The complete url of the source of the package to send. |
| 164 | @param dest_path: The path on the remote host's file system where we would |
| 165 | like to store the package. |
| 166 | @param ssh_cmd: The ssh command to use in performing the remote wget. |
| 167 | """ |
| 168 | wget_cmd = ("wget -O - %s | %s 'cat >%s'" % |
| 169 | (source_url, ssh_cmd, dest_path)) |
| 170 | base_utils.run(wget_cmd) |
| 171 | |
J. Richard Barnette | 3cbd76b | 2013-11-27 12:11:25 -0800 | [diff] [blame] | 172 | |
J. Richard Barnette | 266da2a | 2013-11-27 15:09:55 -0800 | [diff] [blame] | 173 | _MAX_LAB_STATUS_ATTEMPTS = 5 |
| 174 | def _get_lab_status(status_url): |
J. Richard Barnette | 3cbd76b | 2013-11-27 12:11:25 -0800 | [diff] [blame] | 175 | """Grabs the current lab status and message. |
| 176 | |
J. Richard Barnette | 266da2a | 2013-11-27 15:09:55 -0800 | [diff] [blame] | 177 | @returns The JSON object obtained from the given URL. |
| 178 | |
J. Richard Barnette | 3cbd76b | 2013-11-27 12:11:25 -0800 | [diff] [blame] | 179 | """ |
J. Richard Barnette | 3cbd76b | 2013-11-27 12:11:25 -0800 | [diff] [blame] | 180 | retry_waittime = 1 |
J. Richard Barnette | 266da2a | 2013-11-27 15:09:55 -0800 | [diff] [blame] | 181 | for _ in range(_MAX_LAB_STATUS_ATTEMPTS): |
J. Richard Barnette | 3cbd76b | 2013-11-27 12:11:25 -0800 | [diff] [blame] | 182 | try: |
| 183 | response = urllib2.urlopen(status_url) |
| 184 | except IOError as e: |
J. Richard Barnette | 266da2a | 2013-11-27 15:09:55 -0800 | [diff] [blame] | 185 | logging.debug('Error occurred when grabbing the lab status: %s.', |
J. Richard Barnette | 3cbd76b | 2013-11-27 12:11:25 -0800 | [diff] [blame] | 186 | e) |
| 187 | time.sleep(retry_waittime) |
| 188 | continue |
| 189 | # Check for successful response code. |
| 190 | if response.getcode() == 200: |
J. Richard Barnette | 266da2a | 2013-11-27 15:09:55 -0800 | [diff] [blame] | 191 | return json.load(response) |
J. Richard Barnette | 3cbd76b | 2013-11-27 12:11:25 -0800 | [diff] [blame] | 192 | time.sleep(retry_waittime) |
J. Richard Barnette | 266da2a | 2013-11-27 15:09:55 -0800 | [diff] [blame] | 193 | return None |
J. Richard Barnette | 3cbd76b | 2013-11-27 12:11:25 -0800 | [diff] [blame] | 194 | |
| 195 | |
J. Richard Barnette | abbe096 | 2013-12-10 18:15:44 -0800 | [diff] [blame] | 196 | def _decode_lab_status(lab_status, build): |
J. Richard Barnette | 266da2a | 2013-11-27 15:09:55 -0800 | [diff] [blame] | 197 | """Decode lab status, and report exceptions as needed. |
J. Richard Barnette | 3cbd76b | 2013-11-27 12:11:25 -0800 | [diff] [blame] | 198 | |
J. Richard Barnette | abbe096 | 2013-12-10 18:15:44 -0800 | [diff] [blame] | 199 | Take a deserialized JSON object from the lab status page, and |
| 200 | interpret it to determine the actual lab status. Raise |
J. Richard Barnette | 266da2a | 2013-11-27 15:09:55 -0800 | [diff] [blame] | 201 | exceptions as required to report when the lab is down. |
J. Richard Barnette | 3cbd76b | 2013-11-27 12:11:25 -0800 | [diff] [blame] | 202 | |
J. Richard Barnette | abbe096 | 2013-12-10 18:15:44 -0800 | [diff] [blame] | 203 | @param build: build name that we want to check the status of. |
J. Richard Barnette | 3cbd76b | 2013-11-27 12:11:25 -0800 | [diff] [blame] | 204 | |
J. Richard Barnette | abbe096 | 2013-12-10 18:15:44 -0800 | [diff] [blame] | 205 | @raises TestLabException Raised if a request to test for the given |
| 206 | status and build should be blocked. |
J. Richard Barnette | 3cbd76b | 2013-11-27 12:11:25 -0800 | [diff] [blame] | 207 | """ |
J. Richard Barnette | 3cbd76b | 2013-11-27 12:11:25 -0800 | [diff] [blame] | 208 | # First check if the lab is up. |
J. Richard Barnette | 266da2a | 2013-11-27 15:09:55 -0800 | [diff] [blame] | 209 | if not lab_status['general_state'] in LAB_GOOD_STATES: |
J. Richard Barnette | abbe096 | 2013-12-10 18:15:44 -0800 | [diff] [blame] | 210 | raise TestLabException('Chromium OS Test Lab is closed: ' |
| 211 | '%s.' % lab_status['message']) |
J. Richard Barnette | 3cbd76b | 2013-11-27 12:11:25 -0800 | [diff] [blame] | 212 | |
J. Richard Barnette | abbe096 | 2013-12-10 18:15:44 -0800 | [diff] [blame] | 213 | # Check if the build we wish to use is disabled. |
J. Richard Barnette | 3cbd76b | 2013-11-27 12:11:25 -0800 | [diff] [blame] | 214 | # Lab messages should be in the format of: |
J. Richard Barnette | abbe096 | 2013-12-10 18:15:44 -0800 | [diff] [blame] | 215 | # Lab is 'status' [regex ...] (comment) |
| 216 | # If the build name matches any regex, it will be blocked. |
| 217 | build_exceptions = re.search('\[(.*)\]', lab_status['message']) |
Prashanth Balasubramanian | ae43721 | 2014-10-27 11:17:26 -0700 | [diff] [blame] | 218 | if not build_exceptions or not build: |
J. Richard Barnette | abbe096 | 2013-12-10 18:15:44 -0800 | [diff] [blame] | 219 | return |
| 220 | for build_pattern in build_exceptions.group(1).split(): |
| 221 | if re.search(build_pattern, build): |
| 222 | raise TestLabException('Chromium OS Test Lab is closed: ' |
| 223 | '%s matches %s.' % ( |
| 224 | build, build_pattern)) |
J. Richard Barnette | 3cbd76b | 2013-11-27 12:11:25 -0800 | [diff] [blame] | 225 | return |
J. Richard Barnette | 266da2a | 2013-11-27 15:09:55 -0800 | [diff] [blame] | 226 | |
| 227 | |
Dan Shi | 94234cb | 2014-05-23 20:04:31 -0700 | [diff] [blame] | 228 | def is_in_lab(): |
| 229 | """Check if current Autotest instance is in lab |
| 230 | |
| 231 | @return: True if the Autotest instance is in lab. |
| 232 | """ |
| 233 | test_server_name = global_config.global_config.get_config_value( |
| 234 | 'SERVER', 'hostname') |
| 235 | return test_server_name.startswith('cautotest') |
| 236 | |
| 237 | |
J. Richard Barnette | abbe096 | 2013-12-10 18:15:44 -0800 | [diff] [blame] | 238 | def check_lab_status(build): |
| 239 | """Check if the lab status allows us to schedule for a build. |
J. Richard Barnette | 266da2a | 2013-11-27 15:09:55 -0800 | [diff] [blame] | 240 | |
J. Richard Barnette | abbe096 | 2013-12-10 18:15:44 -0800 | [diff] [blame] | 241 | Checks if the lab is down, or if testing for the requested build |
| 242 | should be blocked. |
J. Richard Barnette | 266da2a | 2013-11-27 15:09:55 -0800 | [diff] [blame] | 243 | |
J. Richard Barnette | abbe096 | 2013-12-10 18:15:44 -0800 | [diff] [blame] | 244 | @param build: Name of the build to be scheduled for testing. |
J. Richard Barnette | 266da2a | 2013-11-27 15:09:55 -0800 | [diff] [blame] | 245 | |
J. Richard Barnette | abbe096 | 2013-12-10 18:15:44 -0800 | [diff] [blame] | 246 | @raises TestLabException Raised if a request to test for the given |
| 247 | status and build should be blocked. |
J. Richard Barnette | 266da2a | 2013-11-27 15:09:55 -0800 | [diff] [blame] | 248 | |
| 249 | """ |
| 250 | # Ensure we are trying to schedule on the actual lab. |
Dan Shi | 94234cb | 2014-05-23 20:04:31 -0700 | [diff] [blame] | 251 | if not is_in_lab(): |
J. Richard Barnette | 266da2a | 2013-11-27 15:09:55 -0800 | [diff] [blame] | 252 | return |
| 253 | |
| 254 | # Download the lab status from its home on the web. |
| 255 | status_url = global_config.global_config.get_config_value( |
| 256 | 'CROS', 'lab_status_url') |
| 257 | json_status = _get_lab_status(status_url) |
| 258 | if json_status is None: |
| 259 | # We go ahead and say the lab is open if we can't get the status. |
Ilja H. Friedel | 04be2bd | 2014-05-07 21:29:59 -0700 | [diff] [blame] | 260 | logging.warning('Could not get a status from %s', status_url) |
J. Richard Barnette | 266da2a | 2013-11-27 15:09:55 -0800 | [diff] [blame] | 261 | return |
J. Richard Barnette | abbe096 | 2013-12-10 18:15:44 -0800 | [diff] [blame] | 262 | _decode_lab_status(json_status, build) |
beeps | 023afc6 | 2014-02-04 16:59:22 -0800 | [diff] [blame] | 263 | |
| 264 | |
| 265 | def lock_host_with_labels(afe, lock_manager, labels): |
| 266 | """Lookup and lock one host that matches the list of input labels. |
| 267 | |
| 268 | @param afe: An instance of the afe class, as defined in server.frontend. |
| 269 | @param lock_manager: A lock manager capable of locking hosts, eg the |
| 270 | one defined in server.cros.host_lock_manager. |
| 271 | @param labels: A list of labels to look for on hosts. |
| 272 | |
| 273 | @return: The hostname of a host matching all labels, and locked through the |
| 274 | lock_manager. The hostname will be as specified in the database the afe |
| 275 | object is associated with, i.e if it exists in afe_hosts with a .cros |
| 276 | suffix, the hostname returned will contain a .cros suffix. |
| 277 | |
| 278 | @raises: error.NoEligibleHostException: If no hosts matching the list of |
| 279 | input labels are available. |
| 280 | @raises: error.TestError: If unable to lock a host matching the labels. |
| 281 | """ |
| 282 | potential_hosts = afe.get_hosts(multiple_labels=labels) |
| 283 | if not potential_hosts: |
| 284 | raise error.NoEligibleHostException( |
| 285 | 'No devices found with labels %s.' % labels) |
| 286 | |
| 287 | # This prevents errors where a fault might seem repeatable |
| 288 | # because we lock, say, the same packet capturer for each test run. |
| 289 | random.shuffle(potential_hosts) |
| 290 | for host in potential_hosts: |
| 291 | if lock_manager.lock([host.hostname]): |
| 292 | logging.info('Locked device %s with labels %s.', |
| 293 | host.hostname, labels) |
| 294 | return host.hostname |
| 295 | else: |
| 296 | logging.info('Unable to lock device %s with labels %s.', |
| 297 | host.hostname, labels) |
| 298 | |
| 299 | raise error.TestError('Could not lock a device with labels %s' % labels) |
Dan Shi | 7e04fa8 | 2013-07-25 15:08:48 -0700 | [diff] [blame] | 300 | |
| 301 | |
| 302 | def get_test_views_from_tko(suite_job_id, tko): |
| 303 | """Get test name and result for given suite job ID. |
| 304 | |
| 305 | @param suite_job_id: ID of suite job. |
| 306 | @param tko: an instance of TKO as defined in server/frontend.py. |
| 307 | @return: A dictionary of test status keyed by test name, e.g., |
| 308 | {'dummy_Fail.Error': 'ERROR', 'dummy_Fail.NAError': 'TEST_NA'} |
| 309 | @raise: Exception when there is no test view found. |
| 310 | |
| 311 | """ |
| 312 | views = tko.run('get_detailed_test_views', afe_job_id=suite_job_id) |
| 313 | relevant_views = filter(job_status.view_is_relevant, views) |
| 314 | if not relevant_views: |
| 315 | raise Exception('Failed to retrieve job results.') |
| 316 | |
| 317 | test_views = {} |
| 318 | for view in relevant_views: |
| 319 | test_views[view['test_name']] = view['status'] |
| 320 | |
| 321 | return test_views |
MK Ryu | 35d661e | 2014-09-25 17:44:10 -0700 | [diff] [blame] | 322 | |
| 323 | |
| 324 | def parse_simple_config(config_file): |
| 325 | """Get paths by parsing a simple config file. |
| 326 | |
| 327 | Each line of the config file is a path for a file or directory. |
| 328 | Ignore an empty line and a line starting with a hash character ('#'). |
| 329 | One example of this kind of simple config file is |
| 330 | client/common_lib/logs_to_collect. |
| 331 | |
| 332 | @param config_file: Config file path |
| 333 | @return: A list of directory strings |
| 334 | """ |
| 335 | dirs = [] |
| 336 | for l in open(config_file): |
| 337 | l = l.strip() |
| 338 | if l and not l.startswith('#'): |
| 339 | dirs.append(l) |
| 340 | return dirs |
| 341 | |
| 342 | |
| 343 | def concat_path_except_last(base, sub): |
| 344 | """Concatenate two paths but exclude last entry. |
| 345 | |
| 346 | Take two paths as parameters and return a path string in which |
| 347 | the second path becomes under the first path. |
| 348 | In addition, remove the last path entry from the concatenated path. |
| 349 | This works even when two paths are absolute paths. |
| 350 | |
| 351 | e.g., /usr/local/autotest/results/ + /var/log/ = |
| 352 | /usr/local/autotest/results/var |
| 353 | |
| 354 | e.g., /usr/local/autotest/results/ + /var/log/syslog = |
| 355 | /usr/local/autotest/results/var/log |
| 356 | |
| 357 | @param base: Beginning path |
| 358 | @param sub: The path that is concatenated to base |
| 359 | @return: Concatenated path string |
| 360 | """ |
| 361 | dirname = os.path.dirname(sub.rstrip('/')) |
| 362 | return os.path.join(base, dirname.strip('/')) |
MK Ryu | c9c0c3f | 2014-10-27 14:36:01 -0700 | [diff] [blame] | 363 | |
| 364 | |
| 365 | def get_data_key(prefix, suite, build, board): |
| 366 | """ |
| 367 | Constructs a key string from parameters. |
| 368 | |
| 369 | @param prefix: Prefix for the generating key. |
| 370 | @param suite: a suite name. e.g., bvt-cq, bvt-inline, dummy |
| 371 | @param build: The build string. This string should have a consistent |
| 372 | format eg: x86-mario-release/R26-3570.0.0. If the format of this |
| 373 | string changes such that we can't determine build_type or branch |
| 374 | we give up and use the parametes we're sure of instead (suite, |
| 375 | board). eg: |
| 376 | 1. build = x86-alex-pgo-release/R26-3570.0.0 |
| 377 | branch = 26 |
| 378 | build_type = pgo-release |
| 379 | 2. build = lumpy-paladin/R28-3993.0.0-rc5 |
| 380 | branch = 28 |
| 381 | build_type = paladin |
| 382 | @param board: The board that this suite ran on. |
| 383 | @return: The key string used for a dictionary. |
| 384 | """ |
| 385 | try: |
| 386 | _board, build_type, branch = ParseBuildName(build)[:3] |
| 387 | except ParseBuildNameException as e: |
| 388 | logging.error(str(e)) |
| 389 | branch = 'Unknown' |
| 390 | build_type = 'Unknown' |
| 391 | else: |
| 392 | embedded_str = re.search(r'x86-\w+-(.*)', _board) |
| 393 | if embedded_str: |
| 394 | build_type = embedded_str.group(1) + '-' + build_type |
| 395 | |
| 396 | data_key_dict = { |
| 397 | 'prefix': prefix, |
| 398 | 'board': board, |
| 399 | 'branch': branch, |
| 400 | 'build_type': build_type, |
| 401 | 'suite': suite, |
| 402 | } |
| 403 | return ('%(prefix)s.%(board)s.%(build_type)s.%(branch)s.%(suite)s' |
| 404 | % data_key_dict) |
MK Ryu | 8318435 | 2014-12-10 14:59:40 -0800 | [diff] [blame] | 405 | |
| 406 | |
MK Ryu | 2d0a364 | 2015-01-07 15:11:19 -0800 | [diff] [blame] | 407 | def setup_logging(logfile=None, prefix=False): |
MK Ryu | 8318435 | 2014-12-10 14:59:40 -0800 | [diff] [blame] | 408 | """Setup basic logging with all logging info stripped. |
| 409 | |
| 410 | Calls to logging will only show the message. No severity is logged. |
| 411 | |
| 412 | @param logfile: If specified dump output to a file as well. |
MK Ryu | 2d0a364 | 2015-01-07 15:11:19 -0800 | [diff] [blame] | 413 | @param prefix: Flag for log prefix. Set to True to add prefix to log |
| 414 | entries to include timestamp and log level. Default is False. |
MK Ryu | 8318435 | 2014-12-10 14:59:40 -0800 | [diff] [blame] | 415 | """ |
| 416 | # Remove all existing handlers. client/common_lib/logging_config adds |
| 417 | # a StreamHandler to logger when modules are imported, e.g., |
| 418 | # autotest_lib.client.bin.utils. A new StreamHandler will be added here to |
| 419 | # log only messages, not severity. |
| 420 | logging.getLogger().handlers = [] |
| 421 | |
MK Ryu | 2d0a364 | 2015-01-07 15:11:19 -0800 | [diff] [blame] | 422 | if prefix: |
| 423 | log_format = '%(asctime)s %(levelname)-5s| %(message)s' |
| 424 | else: |
| 425 | log_format = '%(message)s' |
| 426 | |
MK Ryu | 8318435 | 2014-12-10 14:59:40 -0800 | [diff] [blame] | 427 | screen_handler = logging.StreamHandler() |
MK Ryu | 2d0a364 | 2015-01-07 15:11:19 -0800 | [diff] [blame] | 428 | screen_handler.setFormatter(logging.Formatter(log_format)) |
MK Ryu | 8318435 | 2014-12-10 14:59:40 -0800 | [diff] [blame] | 429 | logging.getLogger().addHandler(screen_handler) |
| 430 | logging.getLogger().setLevel(logging.INFO) |
| 431 | if logfile: |
| 432 | file_handler = logging.FileHandler(logfile) |
MK Ryu | 2d0a364 | 2015-01-07 15:11:19 -0800 | [diff] [blame] | 433 | file_handler.setFormatter(logging.Formatter(log_format)) |
MK Ryu | 8318435 | 2014-12-10 14:59:40 -0800 | [diff] [blame] | 434 | file_handler.setLevel(logging.DEBUG) |
| 435 | logging.getLogger().addHandler(file_handler) |
Prashanth Balasubramanian | 8c98ac1 | 2014-12-23 11:26:44 -0800 | [diff] [blame] | 436 | |
| 437 | |
| 438 | def is_shard(): |
| 439 | """Determines if this instance is running as a shard. |
| 440 | |
| 441 | Reads the global_config value shard_hostname in the section SHARD. |
| 442 | |
| 443 | @return True, if shard_hostname is set, False otherwise. |
| 444 | """ |
| 445 | hostname = global_config.global_config.get_config_value( |
| 446 | 'SHARD', 'shard_hostname', default=None) |
MK Ryu | 0c1a37d | 2015-04-30 12:00:55 -0700 | [diff] [blame] | 447 | return bool(hostname) |
| 448 | |
| 449 | |
| 450 | def get_special_task_status(is_complete, success, is_active): |
| 451 | """Get the status of a special task. |
| 452 | |
| 453 | Emulate a host queue entry status for a special task |
| 454 | Although SpecialTasks are not HostQueueEntries, it is helpful to |
| 455 | the user to present similar statuses. |
| 456 | |
| 457 | @param is_complete Boolean if the task is completed. |
| 458 | @param success Boolean if the task succeeded. |
| 459 | @param is_active Boolean if the task is active. |
| 460 | |
| 461 | @return The status of a special task. |
| 462 | """ |
| 463 | if is_complete: |
| 464 | if success: |
| 465 | return host_queue_entry_states.Status.COMPLETED |
| 466 | return host_queue_entry_states.Status.FAILED |
| 467 | if is_active: |
| 468 | return host_queue_entry_states.Status.RUNNING |
| 469 | return host_queue_entry_states.Status.QUEUED |
| 470 | |
| 471 | |
| 472 | def get_special_task_exec_path(hostname, task_id, task_name, time_requested): |
| 473 | """Get the execution path of the SpecialTask. |
| 474 | |
| 475 | This method returns different paths depending on where a |
| 476 | the task ran: |
| 477 | * Master: hosts/hostname/task_id-task_type |
| 478 | * Shard: Master_path/time_created |
| 479 | This is to work around the fact that a shard can fail independent |
| 480 | of the master, and be replaced by another shard that has the same |
| 481 | hosts. Without the time_created stamp the logs of the tasks running |
| 482 | on the second shard will clobber the logs from the first in google |
| 483 | storage, because task ids are not globally unique. |
| 484 | |
| 485 | @param hostname Hostname |
| 486 | @param task_id Special task id |
| 487 | @param task_name Special task name (e.g., Verify, Repair, etc) |
| 488 | @param time_requested Special task requested time. |
| 489 | |
| 490 | @return An execution path for the task. |
| 491 | """ |
| 492 | results_path = 'hosts/%s/%s-%s' % (hostname, task_id, task_name.lower()) |
| 493 | |
| 494 | # If we do this on the master it will break backward compatibility, |
| 495 | # as there are tasks that currently don't have timestamps. If a host |
| 496 | # or job has been sent to a shard, the rpc for that host/job will |
| 497 | # be redirected to the shard, so this global_config check will happen |
| 498 | # on the shard the logs are on. |
| 499 | if not is_shard(): |
| 500 | return results_path |
| 501 | |
| 502 | # Generate a uid to disambiguate special task result directories |
| 503 | # in case this shard fails. The simplest uid is the job_id, however |
| 504 | # in rare cases tasks do not have jobs associated with them (eg: |
| 505 | # frontend verify), so just use the creation timestamp. The clocks |
| 506 | # between a shard and master should always be in sync. Any discrepancies |
| 507 | # will be brought to our attention in the form of job timeouts. |
| 508 | uid = time_requested.strftime('%Y%d%m%H%M%S') |
| 509 | |
| 510 | # TODO: This is a hack, however it is the easiest way to achieve |
| 511 | # correctness. There is currently some debate over the future of |
| 512 | # tasks in our infrastructure and refactoring everything right |
| 513 | # now isn't worth the time. |
| 514 | return '%s/%s' % (results_path, uid) |
| 515 | |
| 516 | |
| 517 | def get_job_tag(id, owner): |
| 518 | """Returns a string tag for a job. |
| 519 | |
| 520 | @param id Job id |
| 521 | @param owner Job owner |
| 522 | |
| 523 | """ |
| 524 | return '%s-%s' % (id, owner) |
| 525 | |
| 526 | |
| 527 | def get_hqe_exec_path(tag, execution_subdir): |
| 528 | """Returns a execution path to a HQE's results. |
| 529 | |
| 530 | @param tag Tag string for a job associated with a HQE. |
| 531 | @param execution_subdir Execution sub-directory string of a HQE. |
| 532 | |
| 533 | """ |
| 534 | return os.path.join(tag, execution_subdir) |
Dan Shi | 82997b9 | 2015-05-06 12:08:02 -0700 | [diff] [blame^] | 535 | |
| 536 | |
| 537 | def is_inside_chroot(): |
| 538 | """Check if the process is running inside chroot. |
| 539 | |
| 540 | This is a wrapper around chromite.lib.cros_build_lib.IsInsideChroot(). The |
| 541 | method checks if cros_build_lib can be imported first. |
| 542 | |
| 543 | @return: True if the process is running inside chroot or cros_build_lib |
| 544 | cannot be imported. |
| 545 | |
| 546 | """ |
| 547 | return not cros_build_lib or cros_build_lib.IsInsideChroot() |