Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 1 | # Copyright 2012 the V8 project authors. All rights reserved. |
| 2 | # Redistribution and use in source and binary forms, with or without |
| 3 | # modification, are permitted provided that the following conditions are |
| 4 | # met: |
| 5 | # |
| 6 | # * Redistributions of source code must retain the above copyright |
| 7 | # notice, this list of conditions and the following disclaimer. |
| 8 | # * Redistributions in binary form must reproduce the above |
| 9 | # copyright notice, this list of conditions and the following |
| 10 | # disclaimer in the documentation and/or other materials provided |
| 11 | # with the distribution. |
| 12 | # * Neither the name of Google Inc. nor the names of its |
| 13 | # contributors may be used to endorse or promote products derived |
| 14 | # from this software without specific prior written permission. |
| 15 | # |
| 16 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 17 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 18 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 19 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 20 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 21 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 22 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 23 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 24 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 25 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 26 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 27 | |
| 28 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 29 | from functools import wraps |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 30 | import json |
| 31 | import os |
| 32 | import sys |
| 33 | import time |
| 34 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 35 | from . import execution |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 36 | from . import junit_output |
| 37 | |
| 38 | |
| 39 | ABS_PATH_PREFIX = os.getcwd() + os.sep |
| 40 | |
| 41 | |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 42 | class ProgressIndicator(object): |
| 43 | |
| 44 | def __init__(self): |
| 45 | self.runner = None |
| 46 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 47 | def SetRunner(self, runner): |
| 48 | self.runner = runner |
| 49 | |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 50 | def Starting(self): |
| 51 | pass |
| 52 | |
| 53 | def Done(self): |
| 54 | pass |
| 55 | |
| 56 | def AboutToRun(self, test): |
| 57 | pass |
| 58 | |
| 59 | def HasRun(self, test, has_unexpected_output): |
| 60 | pass |
| 61 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 62 | def Heartbeat(self): |
| 63 | pass |
| 64 | |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 65 | def PrintFailureHeader(self, test): |
| 66 | if test.suite.IsNegativeTest(test): |
| 67 | negative_marker = '[negative] ' |
| 68 | else: |
| 69 | negative_marker = '' |
| 70 | print "=== %(label)s %(negative)s===" % { |
| 71 | 'label': test.GetLabel(), |
| 72 | 'negative': negative_marker |
| 73 | } |
| 74 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 75 | def _EscapeCommand(self, test): |
| 76 | command = execution.GetCommand(test, self.runner.context) |
| 77 | parts = [] |
| 78 | for part in command: |
| 79 | if ' ' in part: |
| 80 | # Escape spaces. We may need to escape more characters for this |
| 81 | # to work properly. |
| 82 | parts.append('"%s"' % part) |
| 83 | else: |
| 84 | parts.append(part) |
| 85 | return " ".join(parts) |
| 86 | |
| 87 | |
| 88 | class IndicatorNotifier(object): |
| 89 | """Holds a list of progress indicators and notifies them all on events.""" |
| 90 | def __init__(self): |
| 91 | self.indicators = [] |
| 92 | |
| 93 | def Register(self, indicator): |
| 94 | self.indicators.append(indicator) |
| 95 | |
| 96 | |
| 97 | # Forge all generic event-dispatching methods in IndicatorNotifier, which are |
| 98 | # part of the ProgressIndicator interface. |
| 99 | for func_name in ProgressIndicator.__dict__: |
| 100 | func = getattr(ProgressIndicator, func_name) |
| 101 | if callable(func) and not func.__name__.startswith('_'): |
| 102 | def wrap_functor(f): |
| 103 | @wraps(f) |
| 104 | def functor(self, *args, **kwargs): |
| 105 | """Generic event dispatcher.""" |
| 106 | for indicator in self.indicators: |
| 107 | getattr(indicator, f.__name__)(*args, **kwargs) |
| 108 | return functor |
| 109 | setattr(IndicatorNotifier, func_name, wrap_functor(func)) |
| 110 | |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 111 | |
| 112 | class SimpleProgressIndicator(ProgressIndicator): |
| 113 | """Abstract base class for {Verbose,Dots}ProgressIndicator""" |
| 114 | |
| 115 | def Starting(self): |
| 116 | print 'Running %i tests' % self.runner.total |
| 117 | |
| 118 | def Done(self): |
| 119 | print |
| 120 | for failed in self.runner.failed: |
| 121 | self.PrintFailureHeader(failed) |
| 122 | if failed.output.stderr: |
| 123 | print "--- stderr ---" |
| 124 | print failed.output.stderr.strip() |
| 125 | if failed.output.stdout: |
| 126 | print "--- stdout ---" |
| 127 | print failed.output.stdout.strip() |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 128 | print "Command: %s" % self._EscapeCommand(failed) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 129 | if failed.output.HasCrashed(): |
| 130 | print "exit code: %d" % failed.output.exit_code |
| 131 | print "--- CRASHED ---" |
| 132 | if failed.output.HasTimedOut(): |
| 133 | print "--- TIMEOUT ---" |
| 134 | if len(self.runner.failed) == 0: |
| 135 | print "===" |
| 136 | print "=== All tests succeeded" |
| 137 | print "===" |
| 138 | else: |
| 139 | print |
| 140 | print "===" |
| 141 | print "=== %i tests failed" % len(self.runner.failed) |
| 142 | if self.runner.crashed > 0: |
| 143 | print "=== %i tests CRASHED" % self.runner.crashed |
| 144 | print "===" |
| 145 | |
| 146 | |
| 147 | class VerboseProgressIndicator(SimpleProgressIndicator): |
| 148 | |
| 149 | def AboutToRun(self, test): |
| 150 | print 'Starting %s...' % test.GetLabel() |
| 151 | sys.stdout.flush() |
| 152 | |
| 153 | def HasRun(self, test, has_unexpected_output): |
| 154 | if has_unexpected_output: |
| 155 | if test.output.HasCrashed(): |
| 156 | outcome = 'CRASH' |
| 157 | else: |
| 158 | outcome = 'FAIL' |
| 159 | else: |
| 160 | outcome = 'pass' |
| 161 | print 'Done running %s: %s' % (test.GetLabel(), outcome) |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 162 | sys.stdout.flush() |
| 163 | |
| 164 | def Heartbeat(self): |
| 165 | print 'Still working...' |
| 166 | sys.stdout.flush() |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 167 | |
| 168 | |
| 169 | class DotsProgressIndicator(SimpleProgressIndicator): |
| 170 | |
| 171 | def HasRun(self, test, has_unexpected_output): |
| 172 | total = self.runner.succeeded + len(self.runner.failed) |
| 173 | if (total > 1) and (total % 50 == 1): |
| 174 | sys.stdout.write('\n') |
| 175 | if has_unexpected_output: |
| 176 | if test.output.HasCrashed(): |
| 177 | sys.stdout.write('C') |
| 178 | sys.stdout.flush() |
| 179 | elif test.output.HasTimedOut(): |
| 180 | sys.stdout.write('T') |
| 181 | sys.stdout.flush() |
| 182 | else: |
| 183 | sys.stdout.write('F') |
| 184 | sys.stdout.flush() |
| 185 | else: |
| 186 | sys.stdout.write('.') |
| 187 | sys.stdout.flush() |
| 188 | |
| 189 | |
| 190 | class CompactProgressIndicator(ProgressIndicator): |
| 191 | """Abstract base class for {Color,Monochrome}ProgressIndicator""" |
| 192 | |
| 193 | def __init__(self, templates): |
| 194 | super(CompactProgressIndicator, self).__init__() |
| 195 | self.templates = templates |
| 196 | self.last_status_length = 0 |
| 197 | self.start_time = time.time() |
| 198 | |
| 199 | def Done(self): |
| 200 | self.PrintProgress('Done') |
| 201 | print "" # Line break. |
| 202 | |
| 203 | def AboutToRun(self, test): |
| 204 | self.PrintProgress(test.GetLabel()) |
| 205 | |
| 206 | def HasRun(self, test, has_unexpected_output): |
| 207 | if has_unexpected_output: |
| 208 | self.ClearLine(self.last_status_length) |
| 209 | self.PrintFailureHeader(test) |
| 210 | stdout = test.output.stdout.strip() |
| 211 | if len(stdout): |
| 212 | print self.templates['stdout'] % stdout |
| 213 | stderr = test.output.stderr.strip() |
| 214 | if len(stderr): |
| 215 | print self.templates['stderr'] % stderr |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 216 | print "Command: %s" % self._EscapeCommand(test) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 217 | if test.output.HasCrashed(): |
| 218 | print "exit code: %d" % test.output.exit_code |
| 219 | print "--- CRASHED ---" |
| 220 | if test.output.HasTimedOut(): |
| 221 | print "--- TIMEOUT ---" |
| 222 | |
| 223 | def Truncate(self, string, length): |
| 224 | if length and (len(string) > (length - 3)): |
| 225 | return string[:(length - 3)] + "..." |
| 226 | else: |
| 227 | return string |
| 228 | |
| 229 | def PrintProgress(self, name): |
| 230 | self.ClearLine(self.last_status_length) |
| 231 | elapsed = time.time() - self.start_time |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 232 | progress = 0 if not self.runner.total else ( |
| 233 | ((self.runner.total - self.runner.remaining) * 100) // |
| 234 | self.runner.total) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 235 | status = self.templates['status_line'] % { |
| 236 | 'passed': self.runner.succeeded, |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 237 | 'progress': progress, |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 238 | 'failed': len(self.runner.failed), |
| 239 | 'test': name, |
| 240 | 'mins': int(elapsed) / 60, |
| 241 | 'secs': int(elapsed) % 60 |
| 242 | } |
| 243 | status = self.Truncate(status, 78) |
| 244 | self.last_status_length = len(status) |
| 245 | print status, |
| 246 | sys.stdout.flush() |
| 247 | |
| 248 | |
| 249 | class ColorProgressIndicator(CompactProgressIndicator): |
| 250 | |
| 251 | def __init__(self): |
| 252 | templates = { |
| 253 | 'status_line': ("[%(mins)02i:%(secs)02i|" |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 254 | "\033[34m%%%(progress) 4d\033[0m|" |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 255 | "\033[32m+%(passed) 4d\033[0m|" |
| 256 | "\033[31m-%(failed) 4d\033[0m]: %(test)s"), |
| 257 | 'stdout': "\033[1m%s\033[0m", |
| 258 | 'stderr': "\033[31m%s\033[0m", |
| 259 | } |
| 260 | super(ColorProgressIndicator, self).__init__(templates) |
| 261 | |
| 262 | def ClearLine(self, last_line_length): |
| 263 | print "\033[1K\r", |
| 264 | |
| 265 | |
| 266 | class MonochromeProgressIndicator(CompactProgressIndicator): |
| 267 | |
| 268 | def __init__(self): |
| 269 | templates = { |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 270 | 'status_line': ("[%(mins)02i:%(secs)02i|%%%(progress) 4d|" |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 271 | "+%(passed) 4d|-%(failed) 4d]: %(test)s"), |
| 272 | 'stdout': '%s', |
| 273 | 'stderr': '%s', |
| 274 | } |
| 275 | super(MonochromeProgressIndicator, self).__init__(templates) |
| 276 | |
| 277 | def ClearLine(self, last_line_length): |
| 278 | print ("\r" + (" " * last_line_length) + "\r"), |
| 279 | |
| 280 | |
| 281 | class JUnitTestProgressIndicator(ProgressIndicator): |
| 282 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 283 | def __init__(self, junitout, junittestsuite): |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 284 | self.outputter = junit_output.JUnitTestOutput(junittestsuite) |
| 285 | if junitout: |
| 286 | self.outfile = open(junitout, "w") |
| 287 | else: |
| 288 | self.outfile = sys.stdout |
| 289 | |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 290 | def Done(self): |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 291 | self.outputter.FinishAndWrite(self.outfile) |
| 292 | if self.outfile != sys.stdout: |
| 293 | self.outfile.close() |
| 294 | |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 295 | def HasRun(self, test, has_unexpected_output): |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 296 | fail_text = "" |
| 297 | if has_unexpected_output: |
| 298 | stdout = test.output.stdout.strip() |
| 299 | if len(stdout): |
| 300 | fail_text += "stdout:\n%s\n" % stdout |
| 301 | stderr = test.output.stderr.strip() |
| 302 | if len(stderr): |
| 303 | fail_text += "stderr:\n%s\n" % stderr |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 304 | fail_text += "Command: %s" % self._EscapeCommand(test) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 305 | if test.output.HasCrashed(): |
| 306 | fail_text += "exit code: %d\n--- CRASHED ---" % test.output.exit_code |
| 307 | if test.output.HasTimedOut(): |
| 308 | fail_text += "--- TIMEOUT ---" |
| 309 | self.outputter.HasRunTest( |
| 310 | [test.GetLabel()] + self.runner.context.mode_flags + test.flags, |
| 311 | test.duration, |
| 312 | fail_text) |
| 313 | |
| 314 | |
| 315 | class JsonTestProgressIndicator(ProgressIndicator): |
| 316 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 317 | def __init__(self, json_test_results, arch, mode, random_seed): |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 318 | self.json_test_results = json_test_results |
| 319 | self.arch = arch |
| 320 | self.mode = mode |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 321 | self.random_seed = random_seed |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 322 | self.results = [] |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 323 | self.tests = [] |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 324 | |
| 325 | def Done(self): |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 326 | complete_results = [] |
| 327 | if os.path.exists(self.json_test_results): |
| 328 | with open(self.json_test_results, "r") as f: |
| 329 | # Buildbot might start out with an empty file. |
| 330 | complete_results = json.loads(f.read() or "[]") |
| 331 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 332 | # Sort tests by duration. |
| 333 | timed_tests = [t for t in self.tests if t.duration is not None] |
| 334 | timed_tests.sort(lambda a, b: cmp(b.duration, a.duration)) |
| 335 | slowest_tests = [ |
| 336 | { |
| 337 | "name": test.GetLabel(), |
| 338 | "flags": test.flags, |
| 339 | "command": self._EscapeCommand(test).replace(ABS_PATH_PREFIX, ""), |
| 340 | "duration": test.duration, |
| 341 | } for test in timed_tests[:20] |
| 342 | ] |
| 343 | |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 344 | complete_results.append({ |
| 345 | "arch": self.arch, |
| 346 | "mode": self.mode, |
| 347 | "results": self.results, |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 348 | "slowest_tests": slowest_tests, |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 349 | }) |
| 350 | |
| 351 | with open(self.json_test_results, "w") as f: |
| 352 | f.write(json.dumps(complete_results)) |
| 353 | |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 354 | def HasRun(self, test, has_unexpected_output): |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 355 | # Buffer all tests for sorting the durations in the end. |
| 356 | self.tests.append(test) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 357 | if not has_unexpected_output: |
| 358 | # Omit tests that run as expected. Passing tests of reruns after failures |
| 359 | # will have unexpected_output to be reported here has well. |
| 360 | return |
| 361 | |
| 362 | self.results.append({ |
| 363 | "name": test.GetLabel(), |
| 364 | "flags": test.flags, |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 365 | "command": self._EscapeCommand(test).replace(ABS_PATH_PREFIX, ""), |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 366 | "run": test.run, |
| 367 | "stdout": test.output.stdout, |
| 368 | "stderr": test.output.stderr, |
| 369 | "exit_code": test.output.exit_code, |
| 370 | "result": test.suite.GetOutcome(test), |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 371 | "expected": list(test.outcomes or ["PASS"]), |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 372 | "duration": test.duration, |
| 373 | |
| 374 | # TODO(machenbach): This stores only the global random seed from the |
| 375 | # context and not possible overrides when using random-seed stress. |
| 376 | "random_seed": self.random_seed, |
| 377 | "target_name": test.suite.shell(), |
| 378 | "variant": test.variant, |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 379 | }) |
| 380 | |
| 381 | |
| 382 | PROGRESS_INDICATORS = { |
| 383 | 'verbose': VerboseProgressIndicator, |
| 384 | 'dots': DotsProgressIndicator, |
| 385 | 'color': ColorProgressIndicator, |
| 386 | 'mono': MonochromeProgressIndicator |
| 387 | } |