Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 1 | #!/usr/bin/python |
| 2 | # Copyright (c) 2010 The Chromium OS Authors. All rights reserved. |
| 3 | # Use of this source code is governed by a BSD-style license that can be |
| 4 | # found in the LICENSE file. |
| 5 | |
| 6 | |
| 7 | """Parses and displays the contents of one or more autoserv result directories. |
| 8 | |
| 9 | This script parses the contents of one or more autoserv results folders and |
| 10 | generates test reports. |
| 11 | """ |
| 12 | |
| 13 | import datetime |
| 14 | import glob |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 15 | import logging |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 16 | import operator |
| 17 | import optparse |
| 18 | import os |
| 19 | import re |
| 20 | import sys |
| 21 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 22 | import common |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 23 | try: |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 24 | # Ensure the chromite site-package is installed. |
| 25 | from chromite.lib import terminal |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 26 | except ImportError: |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 27 | import subprocess |
| 28 | build_externals_path = os.path.join( |
| 29 | os.path.dirname(os.path.dirname(os.path.realpath(__file__))), |
| 30 | 'utils', 'build_externals.py') |
| 31 | subprocess.check_call([build_externals_path, 'chromiterepo']) |
| 32 | # Restart the script so python now finds the autotest site-packages. |
| 33 | sys.exit(os.execv(__file__, sys.argv)) |
| 34 | |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 35 | |
| 36 | _STDOUT_IS_TTY = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty() |
| 37 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 38 | |
| 39 | def Die(message_format, *args, **kwargs): |
| 40 | """Log a message and kill the current process. |
| 41 | |
| 42 | @param message_format: string for logging.error. |
| 43 | |
| 44 | """ |
| 45 | logging.error(message_format, *args, **kwargs) |
| 46 | sys.exit(1) |
| 47 | |
| 48 | |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 49 | class CrashWaiver: |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 50 | """Represents a crash that we want to ignore for now.""" |
| 51 | def __init__(self, signals, deadline, url, person): |
| 52 | self.signals = signals |
| 53 | self.deadline = datetime.datetime.strptime(deadline, '%Y-%b-%d') |
| 54 | self.issue_url = url |
| 55 | self.suppressor = person |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 56 | |
| 57 | # List of crashes which are okay to ignore. This list should almost always be |
| 58 | # empty. If you add an entry, include the bug URL and your name, something like |
| 59 | # 'crashy':CrashWaiver( |
| 60 | # ['sig 11'], '2011-Aug-18', 'http://crosbug/123456', 'developer'), |
| 61 | |
| 62 | _CRASH_WHITELIST = { |
| 63 | } |
| 64 | |
| 65 | |
| 66 | class ResultCollector(object): |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 67 | """Collects status and performance data from an autoserv results dir.""" |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 68 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 69 | def __init__(self, collect_perf=True, collect_attr=False, |
| 70 | collect_info=False, escape_error=False, |
| 71 | whitelist_chrome_crashes=False): |
| 72 | """Initialize ResultsCollector class. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 73 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 74 | @param collect_perf: Should perf keyvals be collected? |
| 75 | @param collect_attr: Should attr keyvals be collected? |
| 76 | @param collect_info: Should info keyvals be collected? |
| 77 | @param escape_error: Escape error message text for tools. |
| 78 | @param whitelist_chrome_crashes: Treat Chrome crashes as non-fatal. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 79 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 80 | """ |
| 81 | self._collect_perf = collect_perf |
| 82 | self._collect_attr = collect_attr |
| 83 | self._collect_info = collect_info |
| 84 | self._escape_error = escape_error |
| 85 | self._whitelist_chrome_crashes = whitelist_chrome_crashes |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 86 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 87 | def _CollectPerf(self, testdir): |
| 88 | """Parses keyval file under testdir and return the perf keyval pairs. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 89 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 90 | @param testdir: autoserv test result directory path. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 91 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 92 | @return dict of perf keyval pairs. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 93 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 94 | """ |
| 95 | if not self._collect_perf: |
| 96 | return {} |
| 97 | return self._CollectKeyval(testdir, 'perf') |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 98 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 99 | def _CollectAttr(self, testdir): |
| 100 | """Parses keyval file under testdir and return the attr keyval pairs. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 101 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 102 | @param testdir: autoserv test result directory path. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 103 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 104 | @return dict of attr keyval pairs. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 105 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 106 | """ |
| 107 | if not self._collect_attr: |
| 108 | return {} |
| 109 | return self._CollectKeyval(testdir, 'attr') |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 110 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 111 | def _CollectKeyval(self, testdir, keyword): |
| 112 | """Parses keyval file under testdir. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 113 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 114 | If testdir contains a result folder, process the keyval file and return |
| 115 | a dictionary of perf keyval pairs. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 116 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 117 | @param testdir: The autoserv test result directory. |
| 118 | @param keyword: The keyword of keyval, either 'perf' or 'attr'. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 119 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 120 | @return If the perf option is disabled or the there's no keyval file |
| 121 | under testdir, returns an empty dictionary. Otherwise, returns |
| 122 | a dictionary of parsed keyvals. Duplicate keys are uniquified |
| 123 | by their instance number. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 124 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 125 | """ |
| 126 | keyval = {} |
| 127 | keyval_file = os.path.join(testdir, 'results', 'keyval') |
| 128 | if not os.path.isfile(keyval_file): |
| 129 | return keyval |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 130 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 131 | instances = {} |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 132 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 133 | for line in open(keyval_file): |
| 134 | match = re.search(r'^(.+){%s}=(.+)$' % keyword, line) |
| 135 | if match: |
| 136 | key = match.group(1) |
| 137 | val = match.group(2) |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 138 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 139 | # If the same key name was generated multiple times, uniquify |
| 140 | # all instances other than the first one by adding the instance |
| 141 | # count to the key name. |
| 142 | key_inst = key |
| 143 | instance = instances.get(key, 0) |
| 144 | if instance: |
| 145 | key_inst = '%s{%d}' % (key, instance) |
| 146 | instances[key] = instance + 1 |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 147 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 148 | keyval[key_inst] = val |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 149 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 150 | return keyval |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 151 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 152 | def _CollectCrashes(self, status_raw): |
| 153 | """Parses status_raw file for crashes. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 154 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 155 | Saves crash details if crashes are discovered. If a whitelist is |
| 156 | present, only records whitelisted crashes. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 157 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 158 | @param status_raw: The contents of the status.log or status file from |
| 159 | the test. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 160 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 161 | @return a list of crash entries to be reported. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 162 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 163 | """ |
| 164 | crashes = [] |
| 165 | regex = re.compile( |
| 166 | 'Received crash notification for ([-\w]+).+ (sig \d+)') |
| 167 | chrome_regex = re.compile(r'^supplied_[cC]hrome|^chrome$') |
| 168 | for match in regex.finditer(status_raw): |
| 169 | w = _CRASH_WHITELIST.get(match.group(1)) |
| 170 | if (self._whitelist_chrome_crashes and |
| 171 | chrome_regex.match(match.group(1))): |
| 172 | print '@@@STEP_WARNINGS@@@' |
| 173 | print '%s crashed with %s' % (match.group(1), match.group(2)) |
| 174 | elif (w is not None and match.group(2) in w.signals and |
| 175 | w.deadline > datetime.datetime.now()): |
| 176 | print 'Ignoring crash in %s for waiver that expires %s' % ( |
| 177 | match.group(1), w.deadline.strftime('%Y-%b-%d')) |
| 178 | else: |
| 179 | crashes.append('%s %s' % match.groups()) |
| 180 | return crashes |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 181 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 182 | def _CollectInfo(self, testdir, custom_info): |
| 183 | """Parses *_info files under testdir/sysinfo/var/log. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 184 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 185 | If the sysinfo/var/log/*info files exist, save information that shows |
| 186 | hw, ec and bios version info. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 187 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 188 | This collection of extra info is disabled by default (this funtion is |
| 189 | a no-op). It is enabled only if the --info command-line option is |
| 190 | explicitly supplied. Normal job parsing does not supply this option. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 191 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 192 | @param testdir: The autoserv test result directory. |
| 193 | @param custom_info: Dictionary to collect detailed ec/bios info. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 194 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 195 | @return a dictionary of info that was discovered. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 196 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 197 | """ |
| 198 | if not self._collect_info: |
| 199 | return {} |
| 200 | info = custom_info |
| 201 | |
| 202 | sysinfo_dir = os.path.join(testdir, 'sysinfo', 'var', 'log') |
| 203 | for info_file, info_keys in {'ec_info.txt': ['fw_version'], |
| 204 | 'bios_info.txt': ['fwid', |
| 205 | 'hwid']}.iteritems(): |
| 206 | info_file_path = os.path.join(sysinfo_dir, info_file) |
| 207 | if not os.path.isfile(info_file_path): |
| 208 | continue |
| 209 | # Some example raw text that might be matched include: |
| 210 | # |
| 211 | # fw_version | snow_v1.1.332-cf20b3e |
| 212 | # fwid = Google_Snow.2711.0.2012_08_06_1139 # Active firmware ID |
| 213 | # hwid = DAISY TEST A-A 9382 # Hardware ID |
| 214 | info_regex = re.compile(r'^(%s)\s*[|=]\s*(.*)' % |
| 215 | '|'.join(info_keys)) |
| 216 | with open(info_file_path, 'r') as f: |
| 217 | for line in f: |
| 218 | line = line.strip() |
| 219 | line = line.split('#')[0] |
| 220 | match = info_regex.match(line) |
| 221 | if match: |
| 222 | info[match.group(1)] = str(match.group(2)).strip() |
| 223 | return info |
| 224 | |
| 225 | def _CollectEndTimes(self, status_raw, status_re='', is_end=True): |
| 226 | """Helper to match and collect timestamp and localtime. |
| 227 | |
| 228 | Preferred to locate timestamp and localtime with an |
| 229 | 'END GOOD test_name...' line. However, aborted tests occasionally fail |
| 230 | to produce this line and then need to scrape timestamps from the 'START |
| 231 | test_name...' line. |
| 232 | |
| 233 | @param status_raw: multi-line text to search. |
| 234 | @param status_re: status regex to seek (e.g. GOOD|FAIL) |
| 235 | @param is_end: if True, search for 'END' otherwise 'START'. |
| 236 | |
| 237 | @return Tuple of timestamp, localtime retrieved from the test status |
| 238 | log. |
| 239 | |
| 240 | """ |
| 241 | timestamp = '' |
| 242 | localtime = '' |
| 243 | |
| 244 | localtime_re = r'\w+\s+\w+\s+[:\w]+' |
| 245 | match_filter = ( |
| 246 | r'^\s*%s\s+(?:%s).*timestamp=(\d*).*localtime=(%s).*$' % ( |
| 247 | 'END' if is_end else 'START', status_re, localtime_re)) |
| 248 | matches = re.findall(match_filter, status_raw, re.MULTILINE) |
| 249 | if matches: |
| 250 | # There may be multiple lines with timestamp/localtime info. |
| 251 | # The last one found is selected because it will reflect the end |
| 252 | # time. |
| 253 | for i in xrange(len(matches)): |
| 254 | timestamp_, localtime_ = matches[-(i+1)] |
| 255 | if not timestamp or timestamp_ > timestamp: |
| 256 | timestamp = timestamp_ |
| 257 | localtime = localtime_ |
| 258 | return timestamp, localtime |
| 259 | |
| 260 | def _CheckExperimental(self, testdir): |
| 261 | """Parses keyval file and return the value of `experimental`. |
| 262 | |
| 263 | @param testdir: The result directory that has the keyval file. |
| 264 | |
| 265 | @return The value of 'experimental', which is a boolean value indicating |
| 266 | whether it is an experimental test or not. |
| 267 | |
| 268 | """ |
| 269 | keyval_file = os.path.join(testdir, 'keyval') |
| 270 | if not os.path.isfile(keyval_file): |
| 271 | return False |
| 272 | |
| 273 | with open(keyval_file) as f: |
| 274 | for line in f: |
| 275 | match = re.match(r'experimental=(.+)', line) |
| 276 | if match: |
| 277 | return match.group(1) == 'True' |
| 278 | else: |
| 279 | return False |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 280 | |
| 281 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 282 | def _CollectResult(self, testdir, results, is_experimental=False): |
| 283 | """Collects results stored under testdir into a dictionary. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 284 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 285 | The presence/location of status files (status.log, status and |
| 286 | job_report.html) varies depending on whether the job is a simple |
| 287 | client test, simple server test, old-style suite or new-style |
| 288 | suite. For example: |
| 289 | -In some cases a single job_report.html may exist but many times |
| 290 | multiple instances are produced in a result tree. |
| 291 | -Most tests will produce a status.log but client tests invoked |
| 292 | by a server test will only emit a status file. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 293 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 294 | The two common criteria that seem to define the presence of a |
| 295 | valid test result are: |
| 296 | 1. Existence of a 'status.log' or 'status' file. Note that if both a |
| 297 | 'status.log' and 'status' file exist for a test, the 'status' file |
| 298 | is always a subset of the 'status.log' fle contents. |
| 299 | 2. Presence of a 'debug' directory. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 300 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 301 | In some cases multiple 'status.log' files will exist where the parent |
| 302 | 'status.log' contains the contents of multiple subdirectory 'status.log' |
| 303 | files. Parent and subdirectory 'status.log' files are always expected |
| 304 | to agree on the outcome of a given test. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 305 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 306 | The test results discovered from the 'status*' files are included |
| 307 | in the result dictionary. The test directory name and a test directory |
| 308 | timestamp/localtime are saved to be used as sort keys for the results. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 309 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 310 | The value of 'is_experimental' is included in the result dictionary. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 311 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 312 | @param testdir: The autoserv test result directory. |
| 313 | @param results: A list to which a populated test-result-dictionary will |
| 314 | be appended if a status file is found. |
| 315 | @param is_experimental: A boolean value indicating whether the result |
| 316 | directory is for an experimental test. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 317 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 318 | """ |
| 319 | status_file = os.path.join(testdir, 'status.log') |
| 320 | if not os.path.isfile(status_file): |
| 321 | status_file = os.path.join(testdir, 'status') |
| 322 | if not os.path.isfile(status_file): |
| 323 | return |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 324 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 325 | # Status is True if GOOD, else False for all others. |
| 326 | status = False |
| 327 | error_msg = None |
| 328 | status_raw = open(status_file, 'r').read() |
| 329 | failure_tags = 'ABORT|ERROR|FAIL' |
| 330 | warning_tag = 'WARN|TEST_NA' |
| 331 | failure = re.search(r'%s' % failure_tags, status_raw) |
| 332 | warning = re.search(r'%s' % warning_tag, status_raw) and not failure |
| 333 | good = (re.search(r'GOOD.+completed successfully', status_raw) and |
| 334 | not (failure or warning)) |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 335 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 336 | # We'd like warnings to allow the tests to pass, but still gather info. |
| 337 | if good or warning: |
| 338 | status = True |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 339 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 340 | if not good: |
| 341 | match = re.search(r'^\t+(%s|%s)\t(.+)' % (failure_tags, |
| 342 | warning_tag), |
| 343 | status_raw, re.MULTILINE) |
| 344 | if match: |
| 345 | failure_type = match.group(1) |
| 346 | reason = match.group(2).split('\t')[4] |
| 347 | if self._escape_error: |
| 348 | reason = re.escape(reason) |
| 349 | error_msg = ': '.join([failure_type, reason]) |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 350 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 351 | # Grab the timestamp - can be used for sorting the test runs. |
| 352 | # Grab the localtime - may be printed to enable line filtering by date. |
| 353 | # Designed to match a line like this: |
| 354 | # END GOOD testname ... timestamp=1347324321 localtime=Sep 10 17:45:21 |
| 355 | status_re = r'GOOD|%s|%s' % (failure_tags, warning_tag) |
| 356 | timestamp, localtime = self._CollectEndTimes(status_raw, status_re) |
| 357 | # Hung tests will occasionally skip printing the END line so grab |
| 358 | # a default timestamp from the START line in those cases. |
| 359 | if not timestamp: |
| 360 | timestamp, localtime = self._CollectEndTimes(status_raw, |
| 361 | is_end=False) |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 362 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 363 | results.append({ |
| 364 | 'testdir': testdir, |
| 365 | 'crashes': self._CollectCrashes(status_raw), |
| 366 | 'status': status, |
| 367 | 'error_msg': error_msg, |
| 368 | 'localtime': localtime, |
| 369 | 'timestamp': timestamp, |
| 370 | 'perf': self._CollectPerf(testdir), |
| 371 | 'attr': self._CollectAttr(testdir), |
| 372 | 'info': self._CollectInfo(testdir, {'localtime': localtime, |
| 373 | 'timestamp': timestamp}), |
| 374 | 'experimental': is_experimental}) |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 375 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 376 | def RecursivelyCollectResults(self, resdir, parent_experimental_tag=False): |
| 377 | """Recursively collect results into a list of dictionaries. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 378 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 379 | Only recurses into directories that possess a 'debug' subdirectory |
| 380 | because anything else is not considered a 'test' directory. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 381 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 382 | The value of 'experimental' in keyval file is used to determine whether |
| 383 | the result is for an experimental test. If it is, all its sub |
| 384 | directories are considered to be experimental tests too. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 385 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 386 | @param resdir: results/test directory to parse results from and recurse |
| 387 | into. |
| 388 | @param parent_experimental_tag: A boolean value, used to keep track of |
| 389 | whether its parent directory is for an experimental test. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 390 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 391 | @return List of dictionaries of results. |
| 392 | |
| 393 | """ |
| 394 | results = [] |
| 395 | is_experimental = (parent_experimental_tag or |
| 396 | self._CheckExperimental(resdir)) |
| 397 | self._CollectResult(resdir, results, is_experimental) |
| 398 | for testdir in glob.glob(os.path.join(resdir, '*')): |
| 399 | # Remove false positives that are missing a debug dir. |
| 400 | if not os.path.exists(os.path.join(testdir, 'debug')): |
| 401 | continue |
| 402 | |
| 403 | results.extend(self.RecursivelyCollectResults( |
| 404 | testdir, is_experimental)) |
| 405 | return results |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 406 | |
| 407 | |
| 408 | class ReportGenerator(object): |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 409 | """Collects and displays data from autoserv results directories. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 410 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 411 | This class collects status and performance data from one or more autoserv |
| 412 | result directories and generates test reports. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 413 | """ |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 414 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 415 | _KEYVAL_INDENT = 2 |
| 416 | _STATUS_STRINGS = {'hr': {'pass': '[ PASSED ]', 'fail': '[ FAILED ]'}, |
| 417 | 'csv': {'pass': 'PASS', 'fail': 'FAIL'}} |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 418 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 419 | def __init__(self, options, args): |
| 420 | self._options = options |
| 421 | self._args = args |
| 422 | self._color = terminal.Color(options.color) |
| 423 | self._results = [] |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 424 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 425 | def _CollectAllResults(self): |
| 426 | """Parses results into the self._results list. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 427 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 428 | Builds a list (self._results) where each entry is a dictionary of |
| 429 | result data from one test (which may contain other tests). Each |
| 430 | dictionary will contain values such as: test folder, status, localtime, |
| 431 | crashes, error_msg, perf keyvals [optional], info [optional]. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 432 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 433 | """ |
| 434 | collector = ResultCollector( |
| 435 | collect_perf=self._options.perf, |
| 436 | collect_attr=self._options.attr, |
| 437 | collect_info=self._options.info, |
| 438 | escape_error=self._options.escape_error, |
| 439 | whitelist_chrome_crashes=self._options.whitelist_chrome_crashes) |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 440 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 441 | for resdir in self._args: |
| 442 | if not os.path.isdir(resdir): |
| 443 | Die('%r does not exist', resdir) |
| 444 | self._results.extend(collector.RecursivelyCollectResults(resdir)) |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 445 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 446 | if not self._results: |
| 447 | Die('no test directories found') |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 448 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 449 | def _GenStatusString(self, status): |
| 450 | """Given a bool indicating success or failure, return the right string. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 451 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 452 | Also takes --csv into account, returns old-style strings if it is set. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 453 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 454 | @param status: True or False, indicating success or failure. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 455 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 456 | @return The appropriate string for printing.. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 457 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 458 | """ |
| 459 | success = 'pass' if status else 'fail' |
| 460 | if self._options.csv: |
| 461 | return self._STATUS_STRINGS['csv'][success] |
| 462 | return self._STATUS_STRINGS['hr'][success] |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 463 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 464 | def _Indent(self, msg): |
| 465 | """Given a message, indents it appropriately. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 466 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 467 | @param msg: string to indent. |
| 468 | @return indented version of msg. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 469 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 470 | """ |
| 471 | return ' ' * self._KEYVAL_INDENT + msg |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 472 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 473 | def _GetTestColumnWidth(self): |
| 474 | """Returns the test column width based on the test data. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 475 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 476 | The test results are aligned by discovering the longest width test |
| 477 | directory name or perf key stored in the list of result dictionaries. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 478 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 479 | @return The width for the test column. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 480 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 481 | """ |
| 482 | width = 0 |
| 483 | for result in self._results: |
| 484 | width = max(width, len(result['testdir'])) |
| 485 | perf = result.get('perf') |
| 486 | if perf: |
| 487 | perf_key_width = len(max(perf, key=len)) |
| 488 | width = max(width, perf_key_width + self._KEYVAL_INDENT) |
| 489 | return width |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 490 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 491 | def _PrintDashLine(self, width): |
| 492 | """Prints a line of dashes as a separator in output. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 493 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 494 | @param width: an integer. |
| 495 | """ |
| 496 | if not self._options.csv: |
| 497 | print ''.ljust(width + len(self._STATUS_STRINGS['hr']['pass']), '-') |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 498 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 499 | def _PrintEntries(self, entries): |
| 500 | """Prints a list of strings, delimited based on --csv flag. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 501 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 502 | @param entries: a list of strings, entities to output. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 503 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 504 | """ |
| 505 | delimiter = ',' if self._options.csv else ' ' |
| 506 | print delimiter.join(entries) |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 507 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 508 | def _PrintErrors(self, test, error_msg): |
| 509 | """Prints an indented error message, unless the --csv flag is set. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 510 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 511 | @param test: the name of a test with which to prefix the line. |
| 512 | @param error_msg: a message to print. None is allowed, but ignored. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 513 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 514 | """ |
| 515 | if not self._options.csv and error_msg: |
| 516 | self._PrintEntries([test, self._Indent(error_msg)]) |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 517 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 518 | def _PrintErrorLogs(self, test, test_string): |
| 519 | """Prints the error log for |test| if --debug is set. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 520 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 521 | @param test: the name of a test suitable for embedding in a path |
| 522 | @param test_string: the name of a test with which to prefix the line. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 523 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 524 | """ |
| 525 | if self._options.print_debug: |
| 526 | debug_file_regex = os.path.join( |
| 527 | 'results.', test, 'debug', |
| 528 | '%s*.ERROR' % os.path.basename(test)) |
| 529 | for path in glob.glob(debug_file_regex): |
| 530 | try: |
| 531 | with open(path) as fh: |
| 532 | for line in fh: |
| 533 | # Ensure line is not just WS. |
| 534 | if len(line.lstrip()) <= 0: |
| 535 | continue |
| 536 | self._PrintEntries( |
| 537 | [test_string, self._Indent(line.rstrip())]) |
| 538 | except IOError: |
| 539 | print 'Could not open %s' % path |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 540 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 541 | def _PrintResultDictKeyVals(self, test_entry, result_dict): |
| 542 | """Formatted print a dict of keyvals like 'perf' or 'info'. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 543 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 544 | This function emits each keyval on a single line for uncompressed |
| 545 | review. The 'perf' dictionary contains performance keyvals while the |
| 546 | 'info' dictionary contains ec info, bios info and some test timestamps. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 547 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 548 | @param test_entry: The unique name of the test (dir) - matches other |
| 549 | test output. |
| 550 | @param result_dict: A dict of keyvals to be presented. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 551 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 552 | """ |
| 553 | if not result_dict: |
| 554 | return |
| 555 | dict_keys = result_dict.keys() |
| 556 | dict_keys.sort() |
| 557 | width = self._GetTestColumnWidth() |
| 558 | for dict_key in dict_keys: |
| 559 | if self._options.csv: |
| 560 | key_entry = dict_key |
| 561 | else: |
| 562 | key_entry = dict_key.ljust(width - self._KEYVAL_INDENT) |
| 563 | key_entry = key_entry.rjust(width) |
| 564 | value_entry = self._color.Color( |
| 565 | self._color.BOLD, result_dict[dict_key]) |
| 566 | self._PrintEntries([test_entry, key_entry, value_entry]) |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 567 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 568 | def _GetSortedTests(self): |
| 569 | """Sort the test result dicts in preparation for results printing. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 570 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 571 | By default sorts the results directionaries by their test names. |
| 572 | However, when running long suites, it is useful to see if an early test |
| 573 | has wedged the system and caused the remaining tests to abort/fail. The |
| 574 | datetime-based chronological sorting allows this view. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 575 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 576 | Uses the --sort-chron command line option to control. |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 577 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 578 | """ |
| 579 | if self._options.sort_chron: |
| 580 | # Need to reverse sort the test dirs to ensure the suite folder |
| 581 | # shows at the bottom. Because the suite folder shares its datetime |
| 582 | # with the last test it shows second-to-last without the reverse |
| 583 | # sort first. |
| 584 | tests = sorted(self._results, key=operator.itemgetter('testdir'), |
| 585 | reverse=True) |
| 586 | tests = sorted(tests, key=operator.itemgetter('timestamp')) |
| 587 | else: |
| 588 | tests = sorted(self._results, key=operator.itemgetter('testdir')) |
| 589 | return tests |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 590 | |
Alex Zamorzaev | 2d9f821 | 2018-09-13 21:40:38 -0700 | [diff] [blame] | 591 | # TODO(zamorzaev): reuse this method in _GetResultsForHTMLReport to avoid |
| 592 | # code copying. |
| 593 | def _GetDedupedResults(self): |
| 594 | """Aggregate results from multiple retries of the same test.""" |
| 595 | deduped_results = {} |
| 596 | for test in self._GetSortedTests(): |
| 597 | test_details_matched = re.search(r'(.*)results-(\d[0-9]*)-(.*)', |
| 598 | test['testdir']) |
| 599 | if not test_details_matched: |
| 600 | continue |
| 601 | |
| 602 | log_dir, test_number, test_name = test_details_matched.groups() |
| 603 | if (test_name in deduped_results and |
| 604 | deduped_results[test_name].get('status')): |
| 605 | # Already have a successfull (re)try. |
| 606 | continue |
| 607 | |
| 608 | deduped_results[test_name] = test |
| 609 | return deduped_results.values() |
| 610 | |
Bogineni Kasaiah | bdd6f18 | 2018-02-01 12:42:47 +0530 | [diff] [blame] | 611 | def _GetResultsForHTMLReport(self): |
| 612 | """Return cleaned results for HTML report.!""" |
| 613 | import copy |
| 614 | tests = copy.deepcopy(self._GetSortedTests()) |
Bogineni Kasaiah | bdd6f18 | 2018-02-01 12:42:47 +0530 | [diff] [blame] | 615 | pass_tag = "Pass" |
| 616 | fail_tag = "Fail" |
| 617 | na_tag = "NA" |
Bogineni Kasaiah | f885456 | 2018-03-14 20:41:15 +0530 | [diff] [blame] | 618 | count = 0 |
| 619 | html_results = {} |
| 620 | for test_status in tests: |
| 621 | individual_tc_results = {} |
| 622 | test_details_matched = re.search(r'(.*)results-(\d[0-9]*)-(.*)', |
| 623 | test_status['testdir']) |
| 624 | if not test_details_matched: |
Bogineni Kasaiah | bdd6f18 | 2018-02-01 12:42:47 +0530 | [diff] [blame] | 625 | continue |
Bogineni Kasaiah | f885456 | 2018-03-14 20:41:15 +0530 | [diff] [blame] | 626 | log_dir = test_details_matched.group(1) |
| 627 | test_number = test_details_matched.group(2) |
| 628 | test_name = test_details_matched.group(3) |
| 629 | if '/' in test_name: |
| 630 | test_name = test_name.split('/')[0] |
Bogineni Kasaiah | bdd6f18 | 2018-02-01 12:42:47 +0530 | [diff] [blame] | 631 | if test_status['error_msg'] is None: |
| 632 | test_status['error_msg'] = '' |
Bogineni Kasaiah | f885456 | 2018-03-14 20:41:15 +0530 | [diff] [blame] | 633 | if not html_results.has_key(test_name): |
| 634 | count = count + 1 |
| 635 | # Arranging the results in an order |
| 636 | individual_tc_results['status'] = test_status['status'] |
| 637 | individual_tc_results['error_msg'] = test_status['error_msg'] |
| 638 | individual_tc_results['s_no'] = count |
| 639 | individual_tc_results['crashes'] = test_status['crashes'] |
Bogineni Kasaiah | bdd6f18 | 2018-02-01 12:42:47 +0530 | [diff] [blame] | 640 | |
Bogineni Kasaiah | f885456 | 2018-03-14 20:41:15 +0530 | [diff] [blame] | 641 | # Add <b> and </b> tag for the good format in the report. |
| 642 | individual_tc_results['attempts'] = \ |
| 643 | '<b>test_result_number: %s - %s</b> : %s' % ( |
| 644 | test_number, log_dir, test_status['error_msg']) |
| 645 | html_results[test_name] = individual_tc_results |
| 646 | else: |
| 647 | |
| 648 | # If test found already then we are using the previous data |
| 649 | # instead of creating two different html rows. If existing |
| 650 | # status is False then needs to be updated |
| 651 | if html_results[test_name]['status'] is False: |
| 652 | html_results[test_name]['status'] = test_status['status'] |
| 653 | html_results[test_name]['error_msg'] = test_status[ |
| 654 | 'error_msg'] |
| 655 | html_results[test_name]['crashes'] = \ |
| 656 | html_results[test_name]['crashes'] + test_status[ |
| 657 | 'crashes'] |
| 658 | html_results[test_name]['attempts'] = \ |
| 659 | html_results[test_name]['attempts'] + \ |
| 660 | '</br><b>test_result_number : %s - %s</b> : %s' % ( |
| 661 | test_number, log_dir, test_status['error_msg']) |
| 662 | |
| 663 | # Re-formating the dictionary as s_no as key. So that we can have |
| 664 | # ordered data at the end |
| 665 | sorted_html_results = {} |
| 666 | for key in html_results.keys(): |
| 667 | sorted_html_results[str(html_results[key]['s_no'])] = \ |
| 668 | html_results[key] |
| 669 | sorted_html_results[str(html_results[key]['s_no'])]['test'] = key |
Bogineni Kasaiah | bdd6f18 | 2018-02-01 12:42:47 +0530 | [diff] [blame] | 670 | |
| 671 | # Mapping the Test case status if True->Pass, False->Fail and if |
| 672 | # True and the error message then NA |
Bogineni Kasaiah | f885456 | 2018-03-14 20:41:15 +0530 | [diff] [blame] | 673 | for key in sorted_html_results.keys(): |
| 674 | if sorted_html_results[key]['status']: |
| 675 | if sorted_html_results[key]['error_msg'] != '': |
| 676 | sorted_html_results[key]['status'] = na_tag |
Bogineni Kasaiah | bdd6f18 | 2018-02-01 12:42:47 +0530 | [diff] [blame] | 677 | else: |
Bogineni Kasaiah | f885456 | 2018-03-14 20:41:15 +0530 | [diff] [blame] | 678 | sorted_html_results[key]['status'] = pass_tag |
Bogineni Kasaiah | bdd6f18 | 2018-02-01 12:42:47 +0530 | [diff] [blame] | 679 | else: |
Bogineni Kasaiah | f885456 | 2018-03-14 20:41:15 +0530 | [diff] [blame] | 680 | sorted_html_results[key]['status'] = fail_tag |
Bogineni Kasaiah | bdd6f18 | 2018-02-01 12:42:47 +0530 | [diff] [blame] | 681 | |
Bogineni Kasaiah | f885456 | 2018-03-14 20:41:15 +0530 | [diff] [blame] | 682 | return sorted_html_results |
Bogineni Kasaiah | bdd6f18 | 2018-02-01 12:42:47 +0530 | [diff] [blame] | 683 | |
| 684 | def GenerateReportHTML(self): |
| 685 | """Generate clean HTMl report for the results.""" |
| 686 | |
| 687 | results = self._GetResultsForHTMLReport() |
Bogineni Kasaiah | f885456 | 2018-03-14 20:41:15 +0530 | [diff] [blame] | 688 | html_table_header = """ <th>S.No</th> |
| 689 | <th>Test</th> |
| 690 | <th>Status</th> |
| 691 | <th>Error Message</th> |
| 692 | <th>Crashes</th> |
| 693 | <th>Attempts</th> |
| 694 | """ |
Bogineni Kasaiah | 567cce8 | 2018-03-22 16:40:10 +0530 | [diff] [blame] | 695 | passed_tests = len([key for key in results.keys() if results[key][ |
| 696 | 'status'].lower() == 'pass']) |
| 697 | failed_tests = len([key for key in results.keys() if results[key][ |
| 698 | 'status'].lower() == 'fail']) |
| 699 | na_tests = len([key for key in results.keys() if results[key][ |
| 700 | 'status'].lower() == 'na']) |
| 701 | total_tests = passed_tests + failed_tests + na_tests |
| 702 | |
Bogineni Kasaiah | f885456 | 2018-03-14 20:41:15 +0530 | [diff] [blame] | 703 | # Sort the keys |
| 704 | ordered_keys = sorted([int(key) for key in results.keys()]) |
Bogineni Kasaiah | bdd6f18 | 2018-02-01 12:42:47 +0530 | [diff] [blame] | 705 | html_table_body = '' |
Bogineni Kasaiah | f885456 | 2018-03-14 20:41:15 +0530 | [diff] [blame] | 706 | for key in ordered_keys: |
| 707 | key = str(key) |
Bogineni Kasaiah | 567cce8 | 2018-03-22 16:40:10 +0530 | [diff] [blame] | 708 | if results[key]['status'].lower() == 'pass': |
Sathees Velayutham | f23305d | 2018-04-10 14:28:30 +0530 | [diff] [blame] | 709 | color = 'LimeGreen' |
Bogineni Kasaiah | 567cce8 | 2018-03-22 16:40:10 +0530 | [diff] [blame] | 710 | elif results[key]['status'].lower() == 'na': |
| 711 | color = 'yellow' |
| 712 | else: |
| 713 | color = 'red' |
Bogineni Kasaiah | f885456 | 2018-03-14 20:41:15 +0530 | [diff] [blame] | 714 | html_table_body = html_table_body + """<tr> |
| 715 | <td>%s</td> |
| 716 | <td>%s</td> |
Bogineni Kasaiah | 567cce8 | 2018-03-22 16:40:10 +0530 | [diff] [blame] | 717 | <td |
| 718 | style="background-color:%s;"> |
| 719 | %s</td> |
Bogineni Kasaiah | f885456 | 2018-03-14 20:41:15 +0530 | [diff] [blame] | 720 | <td>%s</td> |
| 721 | <td>%s</td> |
| 722 | <td>%s</td></tr>""" % \ |
| 723 | (key, results[key]['test'], |
Bogineni Kasaiah | 567cce8 | 2018-03-22 16:40:10 +0530 | [diff] [blame] | 724 | color, |
Bogineni Kasaiah | f885456 | 2018-03-14 20:41:15 +0530 | [diff] [blame] | 725 | results[key]['status'], |
| 726 | results[key]['error_msg'], |
| 727 | results[key]['crashes'], |
| 728 | results[key]['attempts']) |
Bogineni Kasaiah | bdd6f18 | 2018-02-01 12:42:47 +0530 | [diff] [blame] | 729 | html_page = """ |
| 730 | <!DOCTYPE html> |
| 731 | <html lang="en"> |
| 732 | <head> |
| 733 | <title>Automation Results</title> |
| 734 | <meta charset="utf-8"> |
| 735 | <meta name="viewport" content="width=device-width,initial-scale=1"> |
| 736 | <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css"> |
| 737 | <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script> |
| 738 | <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/js/bootstrap.min.js"></script> |
| 739 | </head> |
| 740 | <body> |
| 741 | <div class="container"> |
| 742 | <h2>Automation Report</h2> |
Sathees Velayutham | f23305d | 2018-04-10 14:28:30 +0530 | [diff] [blame] | 743 | <table class="table table-bordered" border="1"> |
Bogineni Kasaiah | bdd6f18 | 2018-02-01 12:42:47 +0530 | [diff] [blame] | 744 | <thead> |
Sathees Velayutham | f23305d | 2018-04-10 14:28:30 +0530 | [diff] [blame] | 745 | <tr style="background-color:LightSkyBlue;"> |
Bogineni Kasaiah | bdd6f18 | 2018-02-01 12:42:47 +0530 | [diff] [blame] | 746 | \n%s |
| 747 | </tr> |
| 748 | </thead> |
| 749 | <tbody> |
| 750 | \n%s |
| 751 | </tbody> |
| 752 | </table> |
Bogineni Kasaiah | 567cce8 | 2018-03-22 16:40:10 +0530 | [diff] [blame] | 753 | <div class="row"> |
| 754 | <div class="col-sm-4">Passed: <b>%d</b></div> |
| 755 | <div class="col-sm-4">Failed: <b>%d</b></div> |
| 756 | <div class="col-sm-4">NA: <b>%d</b></div> |
| 757 | </div> |
| 758 | <div class="row"> |
| 759 | <div class="col-sm-4">Total: <b>%d</b></div> |
| 760 | </div> |
Bogineni Kasaiah | bdd6f18 | 2018-02-01 12:42:47 +0530 | [diff] [blame] | 761 | </div> |
| 762 | </body> |
| 763 | </html> |
| 764 | |
Bogineni Kasaiah | 567cce8 | 2018-03-22 16:40:10 +0530 | [diff] [blame] | 765 | """ % (html_table_header, html_table_body, passed_tests, |
| 766 | failed_tests, na_tests, total_tests) |
Bogineni Kasaiah | bdd6f18 | 2018-02-01 12:42:47 +0530 | [diff] [blame] | 767 | with open(os.path.join(self._options.html_report_dir, |
| 768 | "test_report.html"), 'w') as html_file: |
| 769 | html_file.write(html_page) |
| 770 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 771 | def _GenerateReportText(self): |
| 772 | """Prints a result report to stdout. |
| 773 | |
| 774 | Prints a result table to stdout. Each row of the table contains the |
| 775 | test result directory and the test result (PASS, FAIL). If the perf |
| 776 | option is enabled, each test entry is followed by perf keyval entries |
| 777 | from the test results. |
| 778 | |
| 779 | """ |
| 780 | tests = self._GetSortedTests() |
| 781 | width = self._GetTestColumnWidth() |
| 782 | |
| 783 | crashes = {} |
| 784 | tests_pass = 0 |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 785 | self._PrintDashLine(width) |
| 786 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 787 | for result in tests: |
| 788 | testdir = result['testdir'] |
| 789 | test_entry = testdir if self._options.csv else testdir.ljust(width) |
| 790 | |
| 791 | status_entry = self._GenStatusString(result['status']) |
| 792 | if result['status']: |
| 793 | color = self._color.GREEN |
| 794 | tests_pass += 1 |
| 795 | else: |
| 796 | color = self._color.RED |
| 797 | |
| 798 | test_entries = [test_entry, self._color.Color(color, status_entry)] |
| 799 | |
| 800 | info = result.get('info', {}) |
| 801 | info.update(result.get('attr', {})) |
| 802 | if self._options.csv and (self._options.info or self._options.attr): |
| 803 | if info: |
| 804 | test_entries.extend(['%s=%s' % (k, info[k]) |
| 805 | for k in sorted(info.keys())]) |
| 806 | if not result['status'] and result['error_msg']: |
| 807 | test_entries.append('reason="%s"' % result['error_msg']) |
| 808 | |
| 809 | self._PrintEntries(test_entries) |
| 810 | self._PrintErrors(test_entry, result['error_msg']) |
| 811 | |
| 812 | # Print out error log for failed tests. |
| 813 | if not result['status']: |
| 814 | self._PrintErrorLogs(testdir, test_entry) |
| 815 | |
| 816 | # Emit the perf keyvals entries. There will be no entries if the |
| 817 | # --no-perf option is specified. |
| 818 | self._PrintResultDictKeyVals(test_entry, result['perf']) |
| 819 | |
| 820 | # Determine that there was a crash during this test. |
| 821 | if result['crashes']: |
| 822 | for crash in result['crashes']: |
| 823 | if not crash in crashes: |
| 824 | crashes[crash] = set([]) |
| 825 | crashes[crash].add(testdir) |
| 826 | |
| 827 | # Emit extra test metadata info on separate lines if not --csv. |
| 828 | if not self._options.csv: |
| 829 | self._PrintResultDictKeyVals(test_entry, info) |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 830 | |
| 831 | self._PrintDashLine(width) |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 832 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 833 | if not self._options.csv: |
| 834 | total_tests = len(tests) |
| 835 | percent_pass = 100 * tests_pass / total_tests |
| 836 | pass_str = '%d/%d (%d%%)' % (tests_pass, total_tests, percent_pass) |
| 837 | print 'Total PASS: ' + self._color.Color(self._color.BOLD, pass_str) |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 838 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 839 | if self._options.crash_detection: |
| 840 | print '' |
| 841 | if crashes: |
| 842 | print self._color.Color(self._color.RED, |
| 843 | 'Crashes detected during testing:') |
| 844 | self._PrintDashLine(width) |
| 845 | |
| 846 | for crash_name, crashed_tests in sorted(crashes.iteritems()): |
| 847 | print self._color.Color(self._color.RED, crash_name) |
| 848 | for crashed_test in crashed_tests: |
| 849 | print self._Indent(crashed_test) |
| 850 | |
| 851 | self._PrintDashLine(width) |
| 852 | print ('Total unique crashes: ' + |
| 853 | self._color.Color(self._color.BOLD, str(len(crashes)))) |
| 854 | |
| 855 | # Sometimes the builders exit before these buffers are flushed. |
| 856 | sys.stderr.flush() |
| 857 | sys.stdout.flush() |
| 858 | |
| 859 | def Run(self): |
| 860 | """Runs report generation.""" |
| 861 | self._CollectAllResults() |
Jacob Kopczynski | 2cefa1f | 2018-01-10 17:25:38 -0800 | [diff] [blame] | 862 | if not self._options.just_status_code: |
| 863 | self._GenerateReportText() |
Bogineni Kasaiah | bdd6f18 | 2018-02-01 12:42:47 +0530 | [diff] [blame] | 864 | if self._options.html: |
| 865 | print "\nLogging the data into test_report.html file." |
| 866 | try: |
| 867 | self.GenerateReportHTML() |
| 868 | except Exception as e: |
| 869 | print "Failed to generate HTML report %s" % str(e) |
Alex Zamorzaev | 2d9f821 | 2018-09-13 21:40:38 -0700 | [diff] [blame] | 870 | for d in self._GetDedupedResults(): |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 871 | if d['experimental'] and self._options.ignore_experimental_tests: |
| 872 | continue |
| 873 | if not d['status'] or ( |
| 874 | self._options.crash_detection and d['crashes']): |
| 875 | sys.exit(1) |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 876 | |
| 877 | |
| 878 | def main(): |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 879 | usage = 'Usage: %prog [options] result-directories...' |
| 880 | parser = optparse.OptionParser(usage=usage) |
| 881 | parser.add_option('--color', dest='color', action='store_true', |
| 882 | default=_STDOUT_IS_TTY, |
| 883 | help='Use color for text reports [default if TTY stdout]') |
| 884 | parser.add_option('--no-color', dest='color', action='store_false', |
| 885 | help='Don\'t use color for text reports') |
| 886 | parser.add_option('--no-crash-detection', dest='crash_detection', |
| 887 | action='store_false', default=True, |
| 888 | help='Don\'t report crashes or error out when detected') |
| 889 | parser.add_option('--csv', dest='csv', action='store_true', |
| 890 | help='Output test result in CSV format. ' |
| 891 | 'Implies --no-debug --no-crash-detection.') |
Bogineni Kasaiah | bdd6f18 | 2018-02-01 12:42:47 +0530 | [diff] [blame] | 892 | parser.add_option('--html', dest='html', action='store_true', |
| 893 | help='To generate HTML File. ' |
| 894 | 'Implies --no-debug --no-crash-detection.') |
| 895 | parser.add_option('--html-report-dir', dest='html_report_dir', |
| 896 | action='store', default=None, help='Path to generate ' |
| 897 | 'html report') |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 898 | parser.add_option('--info', dest='info', action='store_true', |
| 899 | default=False, |
| 900 | help='Include info keyvals in the report') |
| 901 | parser.add_option('--escape-error', dest='escape_error', |
| 902 | action='store_true', default=False, |
| 903 | help='Escape error message text for tools.') |
| 904 | parser.add_option('--perf', dest='perf', action='store_true', |
| 905 | default=True, |
| 906 | help='Include perf keyvals in the report [default]') |
| 907 | parser.add_option('--attr', dest='attr', action='store_true', |
| 908 | default=False, |
| 909 | help='Include attr keyvals in the report') |
| 910 | parser.add_option('--no-perf', dest='perf', action='store_false', |
| 911 | help='Don\'t include perf keyvals in the report') |
| 912 | parser.add_option('--sort-chron', dest='sort_chron', action='store_true', |
| 913 | default=False, |
| 914 | help='Sort results by datetime instead of by test name.') |
| 915 | parser.add_option('--no-debug', dest='print_debug', action='store_false', |
| 916 | default=True, |
| 917 | help='Don\'t print out logs when tests fail.') |
| 918 | parser.add_option('--whitelist_chrome_crashes', |
| 919 | dest='whitelist_chrome_crashes', |
| 920 | action='store_true', default=False, |
| 921 | help='Treat Chrome crashes as non-fatal.') |
| 922 | parser.add_option('--ignore_experimental_tests', |
| 923 | dest='ignore_experimental_tests', |
| 924 | action='store_true', default=False, |
| 925 | help='If set, experimental test results will not ' |
| 926 | 'influence the exit code.') |
Jacob Kopczynski | 2cefa1f | 2018-01-10 17:25:38 -0800 | [diff] [blame] | 927 | parser.add_option('--just_status_code', |
| 928 | dest='just_status_code', |
| 929 | action='store_true', default=False, |
| 930 | help='Skip generating a report, just return status code.') |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 931 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 932 | (options, args) = parser.parse_args() |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 933 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 934 | if not args: |
| 935 | parser.print_help() |
| 936 | Die('no result directories provided') |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 937 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 938 | if options.csv and (options.print_debug or options.crash_detection): |
| 939 | Warning('Forcing --no-debug --no-crash-detection') |
| 940 | options.print_debug = False |
| 941 | options.crash_detection = False |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 942 | |
Jacob Kopczynski | 2cefa1f | 2018-01-10 17:25:38 -0800 | [diff] [blame] | 943 | report_options = ['color', 'csv', 'info', 'escape_error', 'perf', 'attr', |
Bogineni Kasaiah | bdd6f18 | 2018-02-01 12:42:47 +0530 | [diff] [blame] | 944 | 'sort_chron', 'print_debug', 'html', 'html_report_dir'] |
Jacob Kopczynski | 2cefa1f | 2018-01-10 17:25:38 -0800 | [diff] [blame] | 945 | if options.just_status_code and any( |
| 946 | getattr(options, opt) for opt in report_options): |
| 947 | Warning('Passed --just_status_code and incompatible options %s' % |
| 948 | ' '.join(opt for opt in report_options if getattr(options,opt))) |
| 949 | |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 950 | generator = ReportGenerator(options, args) |
| 951 | generator.Run() |
Christopher Wiley | 8e8b67f | 2015-12-07 10:13:04 -0800 | [diff] [blame] | 952 | |
| 953 | |
| 954 | if __name__ == '__main__': |
Christopher Wiley | 519624e | 2015-12-07 10:42:05 -0800 | [diff] [blame] | 955 | main() |