blob: 2e12f396a434be311e6e5114aee232c0426f9af6 [file] [log] [blame]
Dan Shia1ecd5c2013-06-06 11:21:31 -07001# Copyright (c) 2013 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
Dan Shia1ecd5c2013-06-06 11:21:31 -07005
J. Richard Barnette3cbd76b2013-11-27 12:11:25 -08006import httplib
7import json
Alex Millerdadc2c22013-07-08 15:21:21 -07008import logging
MK Ryu35d661e2014-09-25 17:44:10 -07009import os
beeps023afc62014-02-04 16:59:22 -080010import random
Alex Millerdadc2c22013-07-08 15:21:21 -070011import re
J. Richard Barnette3cbd76b2013-11-27 12:11:25 -080012import time
Paul Drewsbef578d2013-09-24 15:10:36 -070013import urllib2
Alex Millerdadc2c22013-07-08 15:21:21 -070014
J. Richard Barnette3cbd76b2013-11-27 12:11:25 -080015import common
beeps023afc62014-02-04 16:59:22 -080016from autotest_lib.client.common_lib import base_utils
17from autotest_lib.client.common_lib import error
18from autotest_lib.client.common_lib import global_config
Dan Shia1ecd5c2013-06-06 11:21:31 -070019from autotest_lib.server.cros.dynamic_suite import constants
Dan Shi7e04fa82013-07-25 15:08:48 -070020from autotest_lib.server.cros.dynamic_suite import job_status
Dan Shia1ecd5c2013-06-06 11:21:31 -070021
22
Alex Millerdadc2c22013-07-08 15:21:21 -070023_SHERIFF_JS = global_config.global_config.get_config_value(
24 'NOTIFICATIONS', 'sheriffs', default='')
Fang Deng3197b392013-06-26 11:42:02 -070025_LAB_SHERIFF_JS = global_config.global_config.get_config_value(
26 'NOTIFICATIONS', 'lab_sheriffs', default='')
Alex Millerdadc2c22013-07-08 15:21:21 -070027_CHROMIUM_BUILD_URL = global_config.global_config.get_config_value(
28 'NOTIFICATIONS', 'chromium_build_url', default='')
29
J. Richard Barnette3cbd76b2013-11-27 12:11:25 -080030LAB_GOOD_STATES = ('open', 'throttled')
31
32
J. Richard Barnetteabbe0962013-12-10 18:15:44 -080033class TestLabException(Exception):
34 """Exception raised when the Test Lab blocks a test or suite."""
J. Richard Barnette3cbd76b2013-11-27 12:11:25 -080035 pass
36
37
38class ParseBuildNameException(Exception):
39 """Raised when ParseBuildName() cannot parse a build name."""
40 pass
41
42
43def ParseBuildName(name):
44 """Format a build name, given board, type, milestone, and manifest num.
45
Simran Basib7d21162014-05-21 15:26:16 -070046 @param name: a build name, e.g. 'x86-alex-release/R20-2015.0.0' or a
47 relative build name, e.g. 'x86-alex-release/LATEST'
J. Richard Barnette3cbd76b2013-11-27 12:11:25 -080048
49 @return board: board the manifest is for, e.g. x86-alex.
50 @return type: one of 'release', 'factory', or 'firmware'
51 @return milestone: (numeric) milestone the manifest was associated with.
Simran Basib7d21162014-05-21 15:26:16 -070052 Will be None for relative build names.
53 @return manifest: manifest number, e.g. '2015.0.0'.
54 Will be None for relative build names.
J. Richard Barnette3cbd76b2013-11-27 12:11:25 -080055
56 """
Simran Basif8f648e2014-09-09 11:40:03 -070057 match = re.match(r'(trybot-)?(?P<board>[\w-]+)-(?P<type>\w+)/'
58 r'(R(?P<milestone>\d+)-(?P<manifest>[\d.ab-]+)|LATEST)',
59 name)
60 if match and len(match.groups()) >= 5:
Simran Basib7d21162014-05-21 15:26:16 -070061 return (match.group('board'), match.group('type'),
62 match.group('milestone'), match.group('manifest'))
J. Richard Barnette3cbd76b2013-11-27 12:11:25 -080063 raise ParseBuildNameException('%s is a malformed build name.' % name)
64
Alex Millerdadc2c22013-07-08 15:21:21 -070065
Dan Shia1ecd5c2013-06-06 11:21:31 -070066def get_label_from_afe(hostname, label_prefix, afe):
67 """Retrieve a host's specific label from the AFE.
68
69 Looks for a host label that has the form <label_prefix>:<value>
70 and returns the "<value>" part of the label. None is returned
71 if there is not a label matching the pattern
72
73 @param hostname: hostname of given DUT.
74 @param label_prefix: prefix of label to be matched, e.g., |board:|
75 @param afe: afe instance.
76 @returns the label that matches the prefix or 'None'
77
78 """
79 labels = afe.get_labels(name__startswith=label_prefix,
80 host__hostname__in=[hostname])
81 if labels and len(labels) == 1:
82 return labels[0].name.split(label_prefix, 1)[1]
83
84
85def get_board_from_afe(hostname, afe):
86 """Retrieve given host's board from its labels in the AFE.
87
88 Looks for a host label of the form "board:<board>", and
89 returns the "<board>" part of the label. `None` is returned
90 if there is not a single, unique label matching the pattern.
91
92 @param hostname: hostname of given DUT.
93 @param afe: afe instance.
94 @returns board from label, or `None`.
95
96 """
97 return get_label_from_afe(hostname, constants.BOARD_PREFIX, afe)
98
99
100def get_build_from_afe(hostname, afe):
101 """Retrieve the current build for given host from the AFE.
102
103 Looks through the host's labels in the AFE to determine its build.
104
105 @param hostname: hostname of given DUT.
106 @param afe: afe instance.
107 @returns The current build or None if it could not find it or if there
108 were multiple build labels assigned to this host.
109
110 """
111 return get_label_from_afe(hostname, constants.VERSION_PREFIX, afe)
112
113
Fang Deng3197b392013-06-26 11:42:02 -0700114def get_sheriffs(lab_only=False):
Alex Millerdadc2c22013-07-08 15:21:21 -0700115 """
116 Polls the javascript file that holds the identity of the sheriff and
117 parses it's output to return a list of chromium sheriff email addresses.
118 The javascript file can contain the ldap of more than one sheriff, eg:
119 document.write('sheriff_one, sheriff_two').
120
Fang Deng3197b392013-06-26 11:42:02 -0700121 @param lab_only: if True, only pulls lab sheriff.
122 @return: A list of chroium.org sheriff email addresses to cc on the bug.
123 An empty list if failed to parse the javascript.
Alex Millerdadc2c22013-07-08 15:21:21 -0700124 """
125 sheriff_ids = []
Fang Deng3197b392013-06-26 11:42:02 -0700126 sheriff_js_list = _LAB_SHERIFF_JS.split(',')
127 if not lab_only:
128 sheriff_js_list.extend(_SHERIFF_JS.split(','))
129
130 for sheriff_js in sheriff_js_list:
Alex Millerdadc2c22013-07-08 15:21:21 -0700131 try:
132 url_content = base_utils.urlopen('%s%s'% (
133 _CHROMIUM_BUILD_URL, sheriff_js)).read()
134 except (ValueError, IOError) as e:
beeps4efdf032013-09-17 11:27:14 -0700135 logging.warning('could not parse sheriff from url %s%s: %s',
136 _CHROMIUM_BUILD_URL, sheriff_js, str(e))
Paul Drewsbef578d2013-09-24 15:10:36 -0700137 except (urllib2.URLError, httplib.HTTPException) as e:
138 logging.warning('unexpected error reading from url "%s%s": %s',
139 _CHROMIUM_BUILD_URL, sheriff_js, str(e))
Alex Millerdadc2c22013-07-08 15:21:21 -0700140 else:
141 ldaps = re.search(r"document.write\('(.*)'\)", url_content)
142 if not ldaps:
beeps4efdf032013-09-17 11:27:14 -0700143 logging.warning('Could not retrieve sheriff ldaps for: %s',
144 url_content)
Alex Millerdadc2c22013-07-08 15:21:21 -0700145 continue
146 sheriff_ids += ['%s@chromium.org' % alias.replace(' ', '')
147 for alias in ldaps.group(1).split(',')]
148 return sheriff_ids
beeps46dadc92013-11-07 14:07:10 -0800149
150
151def remote_wget(source_url, dest_path, ssh_cmd):
152 """wget source_url from localhost to dest_path on remote host using ssh.
153
154 @param source_url: The complete url of the source of the package to send.
155 @param dest_path: The path on the remote host's file system where we would
156 like to store the package.
157 @param ssh_cmd: The ssh command to use in performing the remote wget.
158 """
159 wget_cmd = ("wget -O - %s | %s 'cat >%s'" %
160 (source_url, ssh_cmd, dest_path))
161 base_utils.run(wget_cmd)
162
J. Richard Barnette3cbd76b2013-11-27 12:11:25 -0800163
J. Richard Barnette266da2a2013-11-27 15:09:55 -0800164_MAX_LAB_STATUS_ATTEMPTS = 5
165def _get_lab_status(status_url):
J. Richard Barnette3cbd76b2013-11-27 12:11:25 -0800166 """Grabs the current lab status and message.
167
J. Richard Barnette266da2a2013-11-27 15:09:55 -0800168 @returns The JSON object obtained from the given URL.
169
J. Richard Barnette3cbd76b2013-11-27 12:11:25 -0800170 """
J. Richard Barnette3cbd76b2013-11-27 12:11:25 -0800171 retry_waittime = 1
J. Richard Barnette266da2a2013-11-27 15:09:55 -0800172 for _ in range(_MAX_LAB_STATUS_ATTEMPTS):
J. Richard Barnette3cbd76b2013-11-27 12:11:25 -0800173 try:
174 response = urllib2.urlopen(status_url)
175 except IOError as e:
J. Richard Barnette266da2a2013-11-27 15:09:55 -0800176 logging.debug('Error occurred when grabbing the lab status: %s.',
J. Richard Barnette3cbd76b2013-11-27 12:11:25 -0800177 e)
178 time.sleep(retry_waittime)
179 continue
180 # Check for successful response code.
181 if response.getcode() == 200:
J. Richard Barnette266da2a2013-11-27 15:09:55 -0800182 return json.load(response)
J. Richard Barnette3cbd76b2013-11-27 12:11:25 -0800183 time.sleep(retry_waittime)
J. Richard Barnette266da2a2013-11-27 15:09:55 -0800184 return None
J. Richard Barnette3cbd76b2013-11-27 12:11:25 -0800185
186
J. Richard Barnetteabbe0962013-12-10 18:15:44 -0800187def _decode_lab_status(lab_status, build):
J. Richard Barnette266da2a2013-11-27 15:09:55 -0800188 """Decode lab status, and report exceptions as needed.
J. Richard Barnette3cbd76b2013-11-27 12:11:25 -0800189
J. Richard Barnetteabbe0962013-12-10 18:15:44 -0800190 Take a deserialized JSON object from the lab status page, and
191 interpret it to determine the actual lab status. Raise
J. Richard Barnette266da2a2013-11-27 15:09:55 -0800192 exceptions as required to report when the lab is down.
J. Richard Barnette3cbd76b2013-11-27 12:11:25 -0800193
J. Richard Barnetteabbe0962013-12-10 18:15:44 -0800194 @param build: build name that we want to check the status of.
J. Richard Barnette3cbd76b2013-11-27 12:11:25 -0800195
J. Richard Barnetteabbe0962013-12-10 18:15:44 -0800196 @raises TestLabException Raised if a request to test for the given
197 status and build should be blocked.
J. Richard Barnette3cbd76b2013-11-27 12:11:25 -0800198 """
J. Richard Barnette3cbd76b2013-11-27 12:11:25 -0800199 # First check if the lab is up.
J. Richard Barnette266da2a2013-11-27 15:09:55 -0800200 if not lab_status['general_state'] in LAB_GOOD_STATES:
J. Richard Barnetteabbe0962013-12-10 18:15:44 -0800201 raise TestLabException('Chromium OS Test Lab is closed: '
202 '%s.' % lab_status['message'])
J. Richard Barnette3cbd76b2013-11-27 12:11:25 -0800203
J. Richard Barnetteabbe0962013-12-10 18:15:44 -0800204 # Check if the build we wish to use is disabled.
J. Richard Barnette3cbd76b2013-11-27 12:11:25 -0800205 # Lab messages should be in the format of:
J. Richard Barnetteabbe0962013-12-10 18:15:44 -0800206 # Lab is 'status' [regex ...] (comment)
207 # If the build name matches any regex, it will be blocked.
208 build_exceptions = re.search('\[(.*)\]', lab_status['message'])
Prashanth Balasubramanianae437212014-10-27 11:17:26 -0700209 if not build_exceptions or not build:
J. Richard Barnetteabbe0962013-12-10 18:15:44 -0800210 return
211 for build_pattern in build_exceptions.group(1).split():
212 if re.search(build_pattern, build):
213 raise TestLabException('Chromium OS Test Lab is closed: '
214 '%s matches %s.' % (
215 build, build_pattern))
J. Richard Barnette3cbd76b2013-11-27 12:11:25 -0800216 return
J. Richard Barnette266da2a2013-11-27 15:09:55 -0800217
218
Dan Shi94234cb2014-05-23 20:04:31 -0700219def is_in_lab():
220 """Check if current Autotest instance is in lab
221
222 @return: True if the Autotest instance is in lab.
223 """
224 test_server_name = global_config.global_config.get_config_value(
225 'SERVER', 'hostname')
226 return test_server_name.startswith('cautotest')
227
228
J. Richard Barnetteabbe0962013-12-10 18:15:44 -0800229def check_lab_status(build):
230 """Check if the lab status allows us to schedule for a build.
J. Richard Barnette266da2a2013-11-27 15:09:55 -0800231
J. Richard Barnetteabbe0962013-12-10 18:15:44 -0800232 Checks if the lab is down, or if testing for the requested build
233 should be blocked.
J. Richard Barnette266da2a2013-11-27 15:09:55 -0800234
J. Richard Barnetteabbe0962013-12-10 18:15:44 -0800235 @param build: Name of the build to be scheduled for testing.
J. Richard Barnette266da2a2013-11-27 15:09:55 -0800236
J. Richard Barnetteabbe0962013-12-10 18:15:44 -0800237 @raises TestLabException Raised if a request to test for the given
238 status and build should be blocked.
J. Richard Barnette266da2a2013-11-27 15:09:55 -0800239
240 """
241 # Ensure we are trying to schedule on the actual lab.
Dan Shi94234cb2014-05-23 20:04:31 -0700242 if not is_in_lab():
J. Richard Barnette266da2a2013-11-27 15:09:55 -0800243 return
244
245 # Download the lab status from its home on the web.
246 status_url = global_config.global_config.get_config_value(
247 'CROS', 'lab_status_url')
248 json_status = _get_lab_status(status_url)
249 if json_status is None:
250 # We go ahead and say the lab is open if we can't get the status.
Ilja H. Friedel04be2bd2014-05-07 21:29:59 -0700251 logging.warning('Could not get a status from %s', status_url)
J. Richard Barnette266da2a2013-11-27 15:09:55 -0800252 return
J. Richard Barnetteabbe0962013-12-10 18:15:44 -0800253 _decode_lab_status(json_status, build)
beeps023afc62014-02-04 16:59:22 -0800254
255
256def lock_host_with_labels(afe, lock_manager, labels):
257 """Lookup and lock one host that matches the list of input labels.
258
259 @param afe: An instance of the afe class, as defined in server.frontend.
260 @param lock_manager: A lock manager capable of locking hosts, eg the
261 one defined in server.cros.host_lock_manager.
262 @param labels: A list of labels to look for on hosts.
263
264 @return: The hostname of a host matching all labels, and locked through the
265 lock_manager. The hostname will be as specified in the database the afe
266 object is associated with, i.e if it exists in afe_hosts with a .cros
267 suffix, the hostname returned will contain a .cros suffix.
268
269 @raises: error.NoEligibleHostException: If no hosts matching the list of
270 input labels are available.
271 @raises: error.TestError: If unable to lock a host matching the labels.
272 """
273 potential_hosts = afe.get_hosts(multiple_labels=labels)
274 if not potential_hosts:
275 raise error.NoEligibleHostException(
276 'No devices found with labels %s.' % labels)
277
278 # This prevents errors where a fault might seem repeatable
279 # because we lock, say, the same packet capturer for each test run.
280 random.shuffle(potential_hosts)
281 for host in potential_hosts:
282 if lock_manager.lock([host.hostname]):
283 logging.info('Locked device %s with labels %s.',
284 host.hostname, labels)
285 return host.hostname
286 else:
287 logging.info('Unable to lock device %s with labels %s.',
288 host.hostname, labels)
289
290 raise error.TestError('Could not lock a device with labels %s' % labels)
Dan Shi7e04fa82013-07-25 15:08:48 -0700291
292
293def get_test_views_from_tko(suite_job_id, tko):
294 """Get test name and result for given suite job ID.
295
296 @param suite_job_id: ID of suite job.
297 @param tko: an instance of TKO as defined in server/frontend.py.
298 @return: A dictionary of test status keyed by test name, e.g.,
299 {'dummy_Fail.Error': 'ERROR', 'dummy_Fail.NAError': 'TEST_NA'}
300 @raise: Exception when there is no test view found.
301
302 """
303 views = tko.run('get_detailed_test_views', afe_job_id=suite_job_id)
304 relevant_views = filter(job_status.view_is_relevant, views)
305 if not relevant_views:
306 raise Exception('Failed to retrieve job results.')
307
308 test_views = {}
309 for view in relevant_views:
310 test_views[view['test_name']] = view['status']
311
312 return test_views
MK Ryu35d661e2014-09-25 17:44:10 -0700313
314
315def parse_simple_config(config_file):
316 """Get paths by parsing a simple config file.
317
318 Each line of the config file is a path for a file or directory.
319 Ignore an empty line and a line starting with a hash character ('#').
320 One example of this kind of simple config file is
321 client/common_lib/logs_to_collect.
322
323 @param config_file: Config file path
324 @return: A list of directory strings
325 """
326 dirs = []
327 for l in open(config_file):
328 l = l.strip()
329 if l and not l.startswith('#'):
330 dirs.append(l)
331 return dirs
332
333
334def concat_path_except_last(base, sub):
335 """Concatenate two paths but exclude last entry.
336
337 Take two paths as parameters and return a path string in which
338 the second path becomes under the first path.
339 In addition, remove the last path entry from the concatenated path.
340 This works even when two paths are absolute paths.
341
342 e.g., /usr/local/autotest/results/ + /var/log/ =
343 /usr/local/autotest/results/var
344
345 e.g., /usr/local/autotest/results/ + /var/log/syslog =
346 /usr/local/autotest/results/var/log
347
348 @param base: Beginning path
349 @param sub: The path that is concatenated to base
350 @return: Concatenated path string
351 """
352 dirname = os.path.dirname(sub.rstrip('/'))
353 return os.path.join(base, dirname.strip('/'))
MK Ryuc9c0c3f2014-10-27 14:36:01 -0700354
355
356def get_data_key(prefix, suite, build, board):
357 """
358 Constructs a key string from parameters.
359
360 @param prefix: Prefix for the generating key.
361 @param suite: a suite name. e.g., bvt-cq, bvt-inline, dummy
362 @param build: The build string. This string should have a consistent
363 format eg: x86-mario-release/R26-3570.0.0. If the format of this
364 string changes such that we can't determine build_type or branch
365 we give up and use the parametes we're sure of instead (suite,
366 board). eg:
367 1. build = x86-alex-pgo-release/R26-3570.0.0
368 branch = 26
369 build_type = pgo-release
370 2. build = lumpy-paladin/R28-3993.0.0-rc5
371 branch = 28
372 build_type = paladin
373 @param board: The board that this suite ran on.
374 @return: The key string used for a dictionary.
375 """
376 try:
377 _board, build_type, branch = ParseBuildName(build)[:3]
378 except ParseBuildNameException as e:
379 logging.error(str(e))
380 branch = 'Unknown'
381 build_type = 'Unknown'
382 else:
383 embedded_str = re.search(r'x86-\w+-(.*)', _board)
384 if embedded_str:
385 build_type = embedded_str.group(1) + '-' + build_type
386
387 data_key_dict = {
388 'prefix': prefix,
389 'board': board,
390 'branch': branch,
391 'build_type': build_type,
392 'suite': suite,
393 }
394 return ('%(prefix)s.%(board)s.%(build_type)s.%(branch)s.%(suite)s'
395 % data_key_dict)
MK Ryu83184352014-12-10 14:59:40 -0800396
397
MK Ryu2d0a3642015-01-07 15:11:19 -0800398def setup_logging(logfile=None, prefix=False):
MK Ryu83184352014-12-10 14:59:40 -0800399 """Setup basic logging with all logging info stripped.
400
401 Calls to logging will only show the message. No severity is logged.
402
403 @param logfile: If specified dump output to a file as well.
MK Ryu2d0a3642015-01-07 15:11:19 -0800404 @param prefix: Flag for log prefix. Set to True to add prefix to log
405 entries to include timestamp and log level. Default is False.
MK Ryu83184352014-12-10 14:59:40 -0800406 """
407 # Remove all existing handlers. client/common_lib/logging_config adds
408 # a StreamHandler to logger when modules are imported, e.g.,
409 # autotest_lib.client.bin.utils. A new StreamHandler will be added here to
410 # log only messages, not severity.
411 logging.getLogger().handlers = []
412
MK Ryu2d0a3642015-01-07 15:11:19 -0800413 if prefix:
414 log_format = '%(asctime)s %(levelname)-5s| %(message)s'
415 else:
416 log_format = '%(message)s'
417
MK Ryu83184352014-12-10 14:59:40 -0800418 screen_handler = logging.StreamHandler()
MK Ryu2d0a3642015-01-07 15:11:19 -0800419 screen_handler.setFormatter(logging.Formatter(log_format))
MK Ryu83184352014-12-10 14:59:40 -0800420 logging.getLogger().addHandler(screen_handler)
421 logging.getLogger().setLevel(logging.INFO)
422 if logfile:
423 file_handler = logging.FileHandler(logfile)
MK Ryu2d0a3642015-01-07 15:11:19 -0800424 file_handler.setFormatter(logging.Formatter(log_format))
MK Ryu83184352014-12-10 14:59:40 -0800425 file_handler.setLevel(logging.DEBUG)
426 logging.getLogger().addHandler(file_handler)
Prashanth Balasubramanian8c98ac12014-12-23 11:26:44 -0800427
428
429def is_shard():
430 """Determines if this instance is running as a shard.
431
432 Reads the global_config value shard_hostname in the section SHARD.
433
434 @return True, if shard_hostname is set, False otherwise.
435 """
436 hostname = global_config.global_config.get_config_value(
437 'SHARD', 'shard_hostname', default=None)
438 return bool(hostname)