Dale Curtis | e5436f3 | 2011-03-31 14:10:37 -0700 | [diff] [blame] | 1 | #!/usr/bin/python -u |
| 2 | # |
Scott Zawalski | facf511 | 2012-10-12 17:51:00 -0400 | [diff] [blame] | 3 | # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. |
Dale Curtis | e5436f3 | 2011-03-31 14:10:37 -0700 | [diff] [blame] | 4 | # Use of this source code is governed by a BSD-style license that can be |
| 5 | # found in the LICENSE file. |
| 6 | # |
Dale Curtis | 4a431e6 | 2011-10-11 17:37:07 -0700 | [diff] [blame] | 7 | # Site extension of the default parser. Generate JSON reports and stack traces. |
Dale Curtis | e5436f3 | 2011-03-31 14:10:37 -0700 | [diff] [blame] | 8 | # |
| 9 | # This site parser is used to generate a JSON report of test failures, crashes, |
Dale Curtis | 4a431e6 | 2011-10-11 17:37:07 -0700 | [diff] [blame] | 10 | # and the associated logs for later consumption by an Email generator. If any |
| 11 | # crashes are found, the debug symbols for the build are retrieved (either from |
| 12 | # Google Storage or local cache) and core dumps are symbolized. |
Dale Curtis | e5436f3 | 2011-03-31 14:10:37 -0700 | [diff] [blame] | 13 | # |
| 14 | # The parser uses the test report generator which comes bundled with the Chrome |
| 15 | # OS source tree in order to maintain consistency. As well as not having to keep |
| 16 | # track of any secondary failure white lists. |
| 17 | # |
Dale Curtis | 4a431e6 | 2011-10-11 17:37:07 -0700 | [diff] [blame] | 18 | # Stack trace generation is done by the minidump_stackwalk utility which is also |
| 19 | # bundled with the Chrome OS source tree. Requires gsutil and cros_sdk utilties |
| 20 | # be present in the path. |
| 21 | # |
Dale Curtis | e5436f3 | 2011-03-31 14:10:37 -0700 | [diff] [blame] | 22 | # The path to the Chrome OS source tree is defined in global_config under the |
| 23 | # CROS section as 'source_tree'. |
| 24 | # |
| 25 | # Existing parse behavior is kept completely intact. If the site parser is not |
| 26 | # configured it will print a debug message and exit after default parser is |
| 27 | # called. |
| 28 | # |
| 29 | |
Prathmesh Prabhu | b9d993b | 2017-02-03 16:22:31 -0800 | [diff] [blame] | 30 | import errno |
| 31 | import json |
| 32 | import os |
| 33 | import sys |
Dale Curtis | e5436f3 | 2011-03-31 14:10:37 -0700 | [diff] [blame] | 34 | |
| 35 | import common |
Prathmesh Prabhu | b9d993b | 2017-02-03 16:22:31 -0800 | [diff] [blame] | 36 | from autotest_lib.client.bin import utils |
Dale Curtis | 4a431e6 | 2011-10-11 17:37:07 -0700 | [diff] [blame] | 37 | from autotest_lib.client.common_lib import global_config |
Prathmesh Prabhu | b9d993b | 2017-02-03 16:22:31 -0800 | [diff] [blame] | 38 | from autotest_lib.tko import models |
| 39 | from autotest_lib.tko import parse |
| 40 | from autotest_lib.tko import utils as tko_utils |
Dale Curtis | 51976cd | 2011-08-11 18:38:52 -0700 | [diff] [blame] | 41 | from autotest_lib.tko.parsers import version_0 |
Dale Curtis | e5436f3 | 2011-03-31 14:10:37 -0700 | [diff] [blame] | 42 | |
| 43 | |
| 44 | # Name of the report file to produce upon completion. |
| 45 | _JSON_REPORT_FILE = 'results.json' |
| 46 | |
Dale Curtis | 51976cd | 2011-08-11 18:38:52 -0700 | [diff] [blame] | 47 | # Number of log lines to include from error log with each test results. |
| 48 | _ERROR_LOG_LIMIT = 10 |
| 49 | |
| 50 | # Status information is generally more useful than error log, so provide a lot. |
| 51 | _STATUS_LOG_LIMIT = 50 |
| 52 | |
| 53 | |
Dale Curtis | 4a431e6 | 2011-10-11 17:37:07 -0700 | [diff] [blame] | 54 | class StackTrace(object): |
| 55 | """Handles all stack trace generation related duties. See generate().""" |
| 56 | |
| 57 | # Cache dir relative to chroot. |
| 58 | _CACHE_DIR = 'tmp/symbol-cache' |
| 59 | |
| 60 | # Flag file indicating symbols have completed processing. One is created in |
| 61 | # each new symbols directory. |
| 62 | _COMPLETE_FILE = '.completed' |
| 63 | |
| 64 | # Maximum cache age in days; all older cache entries will be deleted. |
| 65 | _MAX_CACHE_AGE_DAYS = 1 |
| 66 | |
| 67 | # Directory inside of tarball under which the actual symbols are stored. |
| 68 | _SYMBOL_DIR = 'debug/breakpad' |
| 69 | |
| 70 | # Maximum time to wait for another instance to finish processing symbols. |
| 71 | _SYMBOL_WAIT_TIMEOUT = 10 * 60 |
| 72 | |
| 73 | |
| 74 | def __init__(self, results_dir, cros_src_dir): |
| 75 | """Initializes class variables. |
| 76 | |
| 77 | Args: |
| 78 | results_dir: Full path to the results directory to process. |
| 79 | cros_src_dir: Full path to Chrome OS source tree. Must have a |
| 80 | working chroot. |
| 81 | """ |
| 82 | self._results_dir = results_dir |
| 83 | self._cros_src_dir = cros_src_dir |
| 84 | self._chroot_dir = os.path.join(self._cros_src_dir, 'chroot') |
| 85 | |
| 86 | |
| 87 | def _get_cache_dir(self): |
| 88 | """Returns a path to the local cache dir, creating if nonexistent. |
| 89 | |
| 90 | Symbol cache is kept inside the chroot so we don't have to mount it into |
| 91 | chroot for symbol generation each time. |
| 92 | |
| 93 | Returns: |
| 94 | A path to the local cache dir. |
| 95 | """ |
| 96 | cache_dir = os.path.join(self._chroot_dir, self._CACHE_DIR) |
| 97 | if not os.path.exists(cache_dir): |
| 98 | try: |
| 99 | os.makedirs(cache_dir) |
| 100 | except OSError, e: |
| 101 | if e.errno != errno.EEXIST: |
| 102 | raise |
| 103 | return cache_dir |
| 104 | |
| 105 | |
| 106 | def _get_job_name(self): |
| 107 | """Returns job name read from 'label' keyval in the results dir. |
| 108 | |
| 109 | Returns: |
| 110 | Job name string. |
| 111 | """ |
| 112 | return models.job.read_keyval(self._results_dir).get('label') |
| 113 | |
| 114 | |
| 115 | def _parse_job_name(self, job_name): |
| 116 | """Returns a tuple of (board, rev, version) parsed from the job name. |
| 117 | |
| 118 | Handles job names of the form "<board-rev>-<version>...", |
| 119 | "<board-rev>-<rev>-<version>...", and |
| 120 | "<board-rev>-<rev>-<version_0>_to_<version>..." |
| 121 | |
| 122 | Args: |
| 123 | job_name: A job name of the format detailed above. |
| 124 | |
| 125 | Returns: |
| 126 | A tuple of (board, rev, version) parsed from the job name. |
| 127 | """ |
| 128 | version = job_name.rsplit('-', 3)[1].split('_')[-1] |
| 129 | arch, board, rev = job_name.split('-', 3)[:3] |
| 130 | return '-'.join([arch, board]), rev, version |
| 131 | |
| 132 | |
Dale Curtis | 51976cd | 2011-08-11 18:38:52 -0700 | [diff] [blame] | 133 | def parse_reason(path): |
| 134 | """Process status.log or status and return a test-name: reason dict.""" |
| 135 | status_log = os.path.join(path, 'status.log') |
| 136 | if not os.path.exists(status_log): |
| 137 | status_log = os.path.join(path, 'status') |
| 138 | if not os.path.exists(status_log): |
| 139 | return |
| 140 | |
| 141 | reasons = {} |
| 142 | last_test = None |
| 143 | for line in open(status_log).readlines(): |
| 144 | try: |
| 145 | # Since we just want the status line parser, it's okay to use the |
| 146 | # version_0 parser directly; all other parsers extend it. |
| 147 | status = version_0.status_line.parse_line(line) |
| 148 | except: |
| 149 | status = None |
| 150 | |
| 151 | # Assemble multi-line reasons into a single reason. |
| 152 | if not status and last_test: |
| 153 | reasons[last_test] += line |
| 154 | |
| 155 | # Skip non-lines, empty lines, and successful tests. |
| 156 | if not status or not status.reason.strip() or status.status == 'GOOD': |
| 157 | continue |
| 158 | |
| 159 | # Update last_test name, so we know which reason to append multi-line |
| 160 | # reasons to. |
| 161 | last_test = status.testname |
| 162 | reasons[last_test] = status.reason |
| 163 | |
| 164 | return reasons |
Dale Curtis | e5436f3 | 2011-03-31 14:10:37 -0700 | [diff] [blame] | 165 | |
| 166 | |
| 167 | def main(): |
| 168 | # Call the original parser. |
| 169 | parse.main() |
| 170 | |
| 171 | # Results directory should be the last argument passed in. |
| 172 | results_dir = sys.argv[-1] |
| 173 | |
| 174 | # Load the Chrome OS source tree location. |
| 175 | cros_src_dir = global_config.global_config.get_config_value( |
| 176 | 'CROS', 'source_tree', default='') |
| 177 | |
| 178 | # We want the standard Autotest parser to keep working even if we haven't |
| 179 | # been setup properly. |
| 180 | if not cros_src_dir: |
| 181 | tko_utils.dprint( |
| 182 | 'Unable to load required components for site parser. Falling back' |
| 183 | ' to default parser.') |
| 184 | return |
| 185 | |
| 186 | # Load ResultCollector from the Chrome OS source tree. |
| 187 | sys.path.append(os.path.join( |
| 188 | cros_src_dir, 'src/platform/crostestutils/utils_py')) |
| 189 | from generate_test_report import ResultCollector |
| 190 | |
| 191 | # Collect results using the standard Chrome OS test report generator. Doing |
| 192 | # so allows us to use the same crash white list and reporting standards the |
| 193 | # VM based test instances use. |
Scott Zawalski | facf511 | 2012-10-12 17:51:00 -0400 | [diff] [blame] | 194 | # TODO(scottz): Reevaluate this code usage. crosbug.com/35282 |
| 195 | results = ResultCollector().RecursivelyCollectResults(results_dir) |
Dale Curtis | e5436f3 | 2011-03-31 14:10:37 -0700 | [diff] [blame] | 196 | # We don't care about successful tests. We only want failed or crashing. |
Scott Zawalski | facf511 | 2012-10-12 17:51:00 -0400 | [diff] [blame] | 197 | # Note: list([]) generates a copy of the dictionary, so it's safe to delete. |
| 198 | for test_status in list(results): |
| 199 | if test_status['crashes']: |
| 200 | continue |
| 201 | elif test_status['status'] == 'PASS': |
| 202 | results.remove(test_status) |
Dale Curtis | e5436f3 | 2011-03-31 14:10:37 -0700 | [diff] [blame] | 203 | |
| 204 | # Filter results and collect logs. If we can't find a log for the test, skip |
| 205 | # it. The Emailer will fill in the blanks using Database data later. |
| 206 | filtered_results = {} |
Scott Zawalski | facf511 | 2012-10-12 17:51:00 -0400 | [diff] [blame] | 207 | for test_dict in results: |
Dale Curtis | 51976cd | 2011-08-11 18:38:52 -0700 | [diff] [blame] | 208 | result_log = '' |
Scott Zawalski | facf511 | 2012-10-12 17:51:00 -0400 | [diff] [blame] | 209 | test_name = os.path.basename(test_dict['testdir']) |
| 210 | error = os.path.join( |
| 211 | test_dict['testdir'], 'debug', '%s.ERROR' % test_name) |
Dale Curtis | e5436f3 | 2011-03-31 14:10:37 -0700 | [diff] [blame] | 212 | |
Dale Curtis | 51976cd | 2011-08-11 18:38:52 -0700 | [diff] [blame] | 213 | # If the error log doesn't exist, we don't care about this test. |
| 214 | if not os.path.isfile(error): |
Dale Curtis | e5436f3 | 2011-03-31 14:10:37 -0700 | [diff] [blame] | 215 | continue |
| 216 | |
Dale Curtis | 51976cd | 2011-08-11 18:38:52 -0700 | [diff] [blame] | 217 | # Parse failure reason for this test. |
Scott Zawalski | facf511 | 2012-10-12 17:51:00 -0400 | [diff] [blame] | 218 | for t, r in parse_reason(test_dict['testdir']).iteritems(): |
Dale Curtis | 51976cd | 2011-08-11 18:38:52 -0700 | [diff] [blame] | 219 | # Server tests may have subtests which will each have their own |
| 220 | # reason, so display the test name for the subtest in that case. |
| 221 | if t != test_name: |
| 222 | result_log += '%s: ' % t |
| 223 | result_log += '%s\n\n' % r.strip() |
| 224 | |
| 225 | # Trim results_log to last _STATUS_LOG_LIMIT lines. |
| 226 | short_result_log = '\n'.join( |
| 227 | result_log.splitlines()[-1 * _STATUS_LOG_LIMIT:]).strip() |
Dale Curtis | e5436f3 | 2011-03-31 14:10:37 -0700 | [diff] [blame] | 228 | |
| 229 | # Let the reader know we've trimmed the log. |
Dale Curtis | 51976cd | 2011-08-11 18:38:52 -0700 | [diff] [blame] | 230 | if short_result_log != result_log.strip(): |
| 231 | short_result_log = ( |
| 232 | '[...displaying only the last %d status log lines...]\n%s' % ( |
| 233 | _STATUS_LOG_LIMIT, short_result_log)) |
| 234 | |
| 235 | # Pull out only the last _LOG_LIMIT lines of the file. |
| 236 | short_log = utils.system_output('tail -n %d %s' % ( |
| 237 | _ERROR_LOG_LIMIT, error)) |
| 238 | |
| 239 | # Let the reader know we've trimmed the log. |
| 240 | if len(short_log.splitlines()) == _ERROR_LOG_LIMIT: |
Dale Curtis | e5436f3 | 2011-03-31 14:10:37 -0700 | [diff] [blame] | 241 | short_log = ( |
Dale Curtis | 51976cd | 2011-08-11 18:38:52 -0700 | [diff] [blame] | 242 | '[...displaying only the last %d error log lines...]\n%s' % ( |
| 243 | _ERROR_LOG_LIMIT, short_log)) |
Dale Curtis | e5436f3 | 2011-03-31 14:10:37 -0700 | [diff] [blame] | 244 | |
Scott Zawalski | facf511 | 2012-10-12 17:51:00 -0400 | [diff] [blame] | 245 | filtered_results[test_name] = test_dict |
Dale Curtis | 51976cd | 2011-08-11 18:38:52 -0700 | [diff] [blame] | 246 | filtered_results[test_name]['log'] = '%s\n\n%s' % ( |
| 247 | short_result_log, short_log) |
Dale Curtis | e5436f3 | 2011-03-31 14:10:37 -0700 | [diff] [blame] | 248 | |
| 249 | # Generate JSON dump of results. Store in results dir. |
| 250 | json_file = open(os.path.join(results_dir, _JSON_REPORT_FILE), 'w') |
| 251 | json.dump(filtered_results, json_file) |
| 252 | json_file.close() |
| 253 | |
Dale Curtis | e5436f3 | 2011-03-31 14:10:37 -0700 | [diff] [blame] | 254 | |
| 255 | if __name__ == '__main__': |
| 256 | main() |