Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame^] | 1 | # Copyright 2012 the V8 project authors. All rights reserved. |
| 2 | # Redistribution and use in source and binary forms, with or without |
| 3 | # modification, are permitted provided that the following conditions are |
| 4 | # met: |
| 5 | # |
| 6 | # * Redistributions of source code must retain the above copyright |
| 7 | # notice, this list of conditions and the following disclaimer. |
| 8 | # * Redistributions in binary form must reproduce the above |
| 9 | # copyright notice, this list of conditions and the following |
| 10 | # disclaimer in the documentation and/or other materials provided |
| 11 | # with the distribution. |
| 12 | # * Neither the name of Google Inc. nor the names of its |
| 13 | # contributors may be used to endorse or promote products derived |
| 14 | # from this software without specific prior written permission. |
| 15 | # |
| 16 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 17 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 18 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 19 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 20 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 21 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 22 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 23 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 24 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 25 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 26 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 27 | |
| 28 | |
| 29 | import json |
| 30 | import os |
| 31 | import sys |
| 32 | import time |
| 33 | |
| 34 | from . import junit_output |
| 35 | |
| 36 | |
| 37 | ABS_PATH_PREFIX = os.getcwd() + os.sep |
| 38 | |
| 39 | |
| 40 | def EscapeCommand(command): |
| 41 | parts = [] |
| 42 | for part in command: |
| 43 | if ' ' in part: |
| 44 | # Escape spaces. We may need to escape more characters for this |
| 45 | # to work properly. |
| 46 | parts.append('"%s"' % part) |
| 47 | else: |
| 48 | parts.append(part) |
| 49 | return " ".join(parts) |
| 50 | |
| 51 | |
| 52 | class ProgressIndicator(object): |
| 53 | |
| 54 | def __init__(self): |
| 55 | self.runner = None |
| 56 | |
| 57 | def Starting(self): |
| 58 | pass |
| 59 | |
| 60 | def Done(self): |
| 61 | pass |
| 62 | |
| 63 | def AboutToRun(self, test): |
| 64 | pass |
| 65 | |
| 66 | def HasRun(self, test, has_unexpected_output): |
| 67 | pass |
| 68 | |
| 69 | def PrintFailureHeader(self, test): |
| 70 | if test.suite.IsNegativeTest(test): |
| 71 | negative_marker = '[negative] ' |
| 72 | else: |
| 73 | negative_marker = '' |
| 74 | print "=== %(label)s %(negative)s===" % { |
| 75 | 'label': test.GetLabel(), |
| 76 | 'negative': negative_marker |
| 77 | } |
| 78 | |
| 79 | |
| 80 | class SimpleProgressIndicator(ProgressIndicator): |
| 81 | """Abstract base class for {Verbose,Dots}ProgressIndicator""" |
| 82 | |
| 83 | def Starting(self): |
| 84 | print 'Running %i tests' % self.runner.total |
| 85 | |
| 86 | def Done(self): |
| 87 | print |
| 88 | for failed in self.runner.failed: |
| 89 | self.PrintFailureHeader(failed) |
| 90 | if failed.output.stderr: |
| 91 | print "--- stderr ---" |
| 92 | print failed.output.stderr.strip() |
| 93 | if failed.output.stdout: |
| 94 | print "--- stdout ---" |
| 95 | print failed.output.stdout.strip() |
| 96 | print "Command: %s" % EscapeCommand(self.runner.GetCommand(failed)) |
| 97 | if failed.output.HasCrashed(): |
| 98 | print "exit code: %d" % failed.output.exit_code |
| 99 | print "--- CRASHED ---" |
| 100 | if failed.output.HasTimedOut(): |
| 101 | print "--- TIMEOUT ---" |
| 102 | if len(self.runner.failed) == 0: |
| 103 | print "===" |
| 104 | print "=== All tests succeeded" |
| 105 | print "===" |
| 106 | else: |
| 107 | print |
| 108 | print "===" |
| 109 | print "=== %i tests failed" % len(self.runner.failed) |
| 110 | if self.runner.crashed > 0: |
| 111 | print "=== %i tests CRASHED" % self.runner.crashed |
| 112 | print "===" |
| 113 | |
| 114 | |
| 115 | class VerboseProgressIndicator(SimpleProgressIndicator): |
| 116 | |
| 117 | def AboutToRun(self, test): |
| 118 | print 'Starting %s...' % test.GetLabel() |
| 119 | sys.stdout.flush() |
| 120 | |
| 121 | def HasRun(self, test, has_unexpected_output): |
| 122 | if has_unexpected_output: |
| 123 | if test.output.HasCrashed(): |
| 124 | outcome = 'CRASH' |
| 125 | else: |
| 126 | outcome = 'FAIL' |
| 127 | else: |
| 128 | outcome = 'pass' |
| 129 | print 'Done running %s: %s' % (test.GetLabel(), outcome) |
| 130 | |
| 131 | |
| 132 | class DotsProgressIndicator(SimpleProgressIndicator): |
| 133 | |
| 134 | def HasRun(self, test, has_unexpected_output): |
| 135 | total = self.runner.succeeded + len(self.runner.failed) |
| 136 | if (total > 1) and (total % 50 == 1): |
| 137 | sys.stdout.write('\n') |
| 138 | if has_unexpected_output: |
| 139 | if test.output.HasCrashed(): |
| 140 | sys.stdout.write('C') |
| 141 | sys.stdout.flush() |
| 142 | elif test.output.HasTimedOut(): |
| 143 | sys.stdout.write('T') |
| 144 | sys.stdout.flush() |
| 145 | else: |
| 146 | sys.stdout.write('F') |
| 147 | sys.stdout.flush() |
| 148 | else: |
| 149 | sys.stdout.write('.') |
| 150 | sys.stdout.flush() |
| 151 | |
| 152 | |
| 153 | class CompactProgressIndicator(ProgressIndicator): |
| 154 | """Abstract base class for {Color,Monochrome}ProgressIndicator""" |
| 155 | |
| 156 | def __init__(self, templates): |
| 157 | super(CompactProgressIndicator, self).__init__() |
| 158 | self.templates = templates |
| 159 | self.last_status_length = 0 |
| 160 | self.start_time = time.time() |
| 161 | |
| 162 | def Done(self): |
| 163 | self.PrintProgress('Done') |
| 164 | print "" # Line break. |
| 165 | |
| 166 | def AboutToRun(self, test): |
| 167 | self.PrintProgress(test.GetLabel()) |
| 168 | |
| 169 | def HasRun(self, test, has_unexpected_output): |
| 170 | if has_unexpected_output: |
| 171 | self.ClearLine(self.last_status_length) |
| 172 | self.PrintFailureHeader(test) |
| 173 | stdout = test.output.stdout.strip() |
| 174 | if len(stdout): |
| 175 | print self.templates['stdout'] % stdout |
| 176 | stderr = test.output.stderr.strip() |
| 177 | if len(stderr): |
| 178 | print self.templates['stderr'] % stderr |
| 179 | print "Command: %s" % EscapeCommand(self.runner.GetCommand(test)) |
| 180 | if test.output.HasCrashed(): |
| 181 | print "exit code: %d" % test.output.exit_code |
| 182 | print "--- CRASHED ---" |
| 183 | if test.output.HasTimedOut(): |
| 184 | print "--- TIMEOUT ---" |
| 185 | |
| 186 | def Truncate(self, string, length): |
| 187 | if length and (len(string) > (length - 3)): |
| 188 | return string[:(length - 3)] + "..." |
| 189 | else: |
| 190 | return string |
| 191 | |
| 192 | def PrintProgress(self, name): |
| 193 | self.ClearLine(self.last_status_length) |
| 194 | elapsed = time.time() - self.start_time |
| 195 | status = self.templates['status_line'] % { |
| 196 | 'passed': self.runner.succeeded, |
| 197 | 'remaining': (((self.runner.total - self.runner.remaining) * 100) // |
| 198 | self.runner.total), |
| 199 | 'failed': len(self.runner.failed), |
| 200 | 'test': name, |
| 201 | 'mins': int(elapsed) / 60, |
| 202 | 'secs': int(elapsed) % 60 |
| 203 | } |
| 204 | status = self.Truncate(status, 78) |
| 205 | self.last_status_length = len(status) |
| 206 | print status, |
| 207 | sys.stdout.flush() |
| 208 | |
| 209 | |
| 210 | class ColorProgressIndicator(CompactProgressIndicator): |
| 211 | |
| 212 | def __init__(self): |
| 213 | templates = { |
| 214 | 'status_line': ("[%(mins)02i:%(secs)02i|" |
| 215 | "\033[34m%%%(remaining) 4d\033[0m|" |
| 216 | "\033[32m+%(passed) 4d\033[0m|" |
| 217 | "\033[31m-%(failed) 4d\033[0m]: %(test)s"), |
| 218 | 'stdout': "\033[1m%s\033[0m", |
| 219 | 'stderr': "\033[31m%s\033[0m", |
| 220 | } |
| 221 | super(ColorProgressIndicator, self).__init__(templates) |
| 222 | |
| 223 | def ClearLine(self, last_line_length): |
| 224 | print "\033[1K\r", |
| 225 | |
| 226 | |
| 227 | class MonochromeProgressIndicator(CompactProgressIndicator): |
| 228 | |
| 229 | def __init__(self): |
| 230 | templates = { |
| 231 | 'status_line': ("[%(mins)02i:%(secs)02i|%%%(remaining) 4d|" |
| 232 | "+%(passed) 4d|-%(failed) 4d]: %(test)s"), |
| 233 | 'stdout': '%s', |
| 234 | 'stderr': '%s', |
| 235 | } |
| 236 | super(MonochromeProgressIndicator, self).__init__(templates) |
| 237 | |
| 238 | def ClearLine(self, last_line_length): |
| 239 | print ("\r" + (" " * last_line_length) + "\r"), |
| 240 | |
| 241 | |
| 242 | class JUnitTestProgressIndicator(ProgressIndicator): |
| 243 | |
| 244 | def __init__(self, progress_indicator, junitout, junittestsuite): |
| 245 | self.progress_indicator = progress_indicator |
| 246 | self.outputter = junit_output.JUnitTestOutput(junittestsuite) |
| 247 | if junitout: |
| 248 | self.outfile = open(junitout, "w") |
| 249 | else: |
| 250 | self.outfile = sys.stdout |
| 251 | |
| 252 | def Starting(self): |
| 253 | self.progress_indicator.runner = self.runner |
| 254 | self.progress_indicator.Starting() |
| 255 | |
| 256 | def Done(self): |
| 257 | self.progress_indicator.Done() |
| 258 | self.outputter.FinishAndWrite(self.outfile) |
| 259 | if self.outfile != sys.stdout: |
| 260 | self.outfile.close() |
| 261 | |
| 262 | def AboutToRun(self, test): |
| 263 | self.progress_indicator.AboutToRun(test) |
| 264 | |
| 265 | def HasRun(self, test, has_unexpected_output): |
| 266 | self.progress_indicator.HasRun(test, has_unexpected_output) |
| 267 | fail_text = "" |
| 268 | if has_unexpected_output: |
| 269 | stdout = test.output.stdout.strip() |
| 270 | if len(stdout): |
| 271 | fail_text += "stdout:\n%s\n" % stdout |
| 272 | stderr = test.output.stderr.strip() |
| 273 | if len(stderr): |
| 274 | fail_text += "stderr:\n%s\n" % stderr |
| 275 | fail_text += "Command: %s" % EscapeCommand(self.runner.GetCommand(test)) |
| 276 | if test.output.HasCrashed(): |
| 277 | fail_text += "exit code: %d\n--- CRASHED ---" % test.output.exit_code |
| 278 | if test.output.HasTimedOut(): |
| 279 | fail_text += "--- TIMEOUT ---" |
| 280 | self.outputter.HasRunTest( |
| 281 | [test.GetLabel()] + self.runner.context.mode_flags + test.flags, |
| 282 | test.duration, |
| 283 | fail_text) |
| 284 | |
| 285 | |
| 286 | class JsonTestProgressIndicator(ProgressIndicator): |
| 287 | |
| 288 | def __init__(self, progress_indicator, json_test_results, arch, mode): |
| 289 | self.progress_indicator = progress_indicator |
| 290 | self.json_test_results = json_test_results |
| 291 | self.arch = arch |
| 292 | self.mode = mode |
| 293 | self.results = [] |
| 294 | |
| 295 | def Starting(self): |
| 296 | self.progress_indicator.runner = self.runner |
| 297 | self.progress_indicator.Starting() |
| 298 | |
| 299 | def Done(self): |
| 300 | self.progress_indicator.Done() |
| 301 | complete_results = [] |
| 302 | if os.path.exists(self.json_test_results): |
| 303 | with open(self.json_test_results, "r") as f: |
| 304 | # Buildbot might start out with an empty file. |
| 305 | complete_results = json.loads(f.read() or "[]") |
| 306 | |
| 307 | complete_results.append({ |
| 308 | "arch": self.arch, |
| 309 | "mode": self.mode, |
| 310 | "results": self.results, |
| 311 | }) |
| 312 | |
| 313 | with open(self.json_test_results, "w") as f: |
| 314 | f.write(json.dumps(complete_results)) |
| 315 | |
| 316 | def AboutToRun(self, test): |
| 317 | self.progress_indicator.AboutToRun(test) |
| 318 | |
| 319 | def HasRun(self, test, has_unexpected_output): |
| 320 | self.progress_indicator.HasRun(test, has_unexpected_output) |
| 321 | if not has_unexpected_output: |
| 322 | # Omit tests that run as expected. Passing tests of reruns after failures |
| 323 | # will have unexpected_output to be reported here has well. |
| 324 | return |
| 325 | |
| 326 | self.results.append({ |
| 327 | "name": test.GetLabel(), |
| 328 | "flags": test.flags, |
| 329 | "command": EscapeCommand(self.runner.GetCommand(test)).replace( |
| 330 | ABS_PATH_PREFIX, ""), |
| 331 | "run": test.run, |
| 332 | "stdout": test.output.stdout, |
| 333 | "stderr": test.output.stderr, |
| 334 | "exit_code": test.output.exit_code, |
| 335 | "result": test.suite.GetOutcome(test), |
| 336 | }) |
| 337 | |
| 338 | |
| 339 | PROGRESS_INDICATORS = { |
| 340 | 'verbose': VerboseProgressIndicator, |
| 341 | 'dots': DotsProgressIndicator, |
| 342 | 'color': ColorProgressIndicator, |
| 343 | 'mono': MonochromeProgressIndicator |
| 344 | } |