Steve Block | a7e24c1 | 2009-10-30 11:49:00 +0000 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | # |
| 3 | # Copyright 2008 the V8 project authors. All rights reserved. |
| 4 | # Redistribution and use in source and binary forms, with or without |
| 5 | # modification, are permitted provided that the following conditions are |
| 6 | # met: |
| 7 | # |
| 8 | # * Redistributions of source code must retain the above copyright |
| 9 | # notice, this list of conditions and the following disclaimer. |
| 10 | # * Redistributions in binary form must reproduce the above |
| 11 | # copyright notice, this list of conditions and the following |
| 12 | # disclaimer in the documentation and/or other materials provided |
| 13 | # with the distribution. |
| 14 | # * Neither the name of Google Inc. nor the names of its |
| 15 | # contributors may be used to endorse or promote products derived |
| 16 | # from this software without specific prior written permission. |
| 17 | # |
| 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 19 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 20 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 21 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 22 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 23 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 24 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 25 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 26 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 27 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 28 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 29 | |
| 30 | |
| 31 | import imp |
| 32 | import optparse |
| 33 | import os |
| 34 | from os.path import join, dirname, abspath, basename, isdir, exists |
| 35 | import platform |
| 36 | import re |
| 37 | import signal |
| 38 | import subprocess |
| 39 | import sys |
| 40 | import tempfile |
| 41 | import time |
| 42 | import threading |
| 43 | import utils |
| 44 | from Queue import Queue, Empty |
| 45 | |
| 46 | |
| 47 | VERBOSE = False |
| 48 | |
| 49 | |
| 50 | # --------------------------------------------- |
| 51 | # --- P r o g r e s s I n d i c a t o r s --- |
| 52 | # --------------------------------------------- |
| 53 | |
| 54 | |
| 55 | class ProgressIndicator(object): |
| 56 | |
| 57 | def __init__(self, cases): |
| 58 | self.cases = cases |
| 59 | self.queue = Queue(len(cases)) |
| 60 | for case in cases: |
| 61 | self.queue.put_nowait(case) |
| 62 | self.succeeded = 0 |
| 63 | self.remaining = len(cases) |
| 64 | self.total = len(cases) |
| 65 | self.failed = [ ] |
| 66 | self.crashed = 0 |
| 67 | self.terminate = False |
| 68 | self.lock = threading.Lock() |
| 69 | |
| 70 | def PrintFailureHeader(self, test): |
| 71 | if test.IsNegative(): |
| 72 | negative_marker = '[negative] ' |
| 73 | else: |
| 74 | negative_marker = '' |
| 75 | print "=== %(label)s %(negative)s===" % { |
| 76 | 'label': test.GetLabel(), |
| 77 | 'negative': negative_marker |
| 78 | } |
| 79 | print "Path: %s" % "/".join(test.path) |
| 80 | |
| 81 | def Run(self, tasks): |
| 82 | self.Starting() |
| 83 | threads = [] |
| 84 | # Spawn N-1 threads and then use this thread as the last one. |
| 85 | # That way -j1 avoids threading altogether which is a nice fallback |
| 86 | # in case of threading problems. |
| 87 | for i in xrange(tasks - 1): |
| 88 | thread = threading.Thread(target=self.RunSingle, args=[]) |
| 89 | threads.append(thread) |
| 90 | thread.start() |
| 91 | try: |
| 92 | self.RunSingle() |
| 93 | # Wait for the remaining threads |
| 94 | for thread in threads: |
| 95 | # Use a timeout so that signals (ctrl-c) will be processed. |
| 96 | thread.join(timeout=10000000) |
| 97 | except Exception, e: |
| 98 | # If there's an exception we schedule an interruption for any |
| 99 | # remaining threads. |
| 100 | self.terminate = True |
| 101 | # ...and then reraise the exception to bail out |
| 102 | raise |
| 103 | self.Done() |
| 104 | return not self.failed |
| 105 | |
| 106 | def RunSingle(self): |
| 107 | while not self.terminate: |
| 108 | try: |
| 109 | test = self.queue.get_nowait() |
| 110 | except Empty: |
| 111 | return |
| 112 | case = test.case |
| 113 | self.lock.acquire() |
| 114 | self.AboutToRun(case) |
| 115 | self.lock.release() |
| 116 | try: |
| 117 | start = time.time() |
| 118 | output = case.Run() |
| 119 | case.duration = (time.time() - start) |
| 120 | except IOError, e: |
| 121 | assert self.terminate |
| 122 | return |
| 123 | if self.terminate: |
| 124 | return |
| 125 | self.lock.acquire() |
| 126 | if output.UnexpectedOutput(): |
| 127 | self.failed.append(output) |
| 128 | if output.HasCrashed(): |
| 129 | self.crashed += 1 |
| 130 | else: |
| 131 | self.succeeded += 1 |
| 132 | self.remaining -= 1 |
| 133 | self.HasRun(output) |
| 134 | self.lock.release() |
| 135 | |
| 136 | |
| 137 | def EscapeCommand(command): |
| 138 | parts = [] |
| 139 | for part in command: |
| 140 | if ' ' in part: |
| 141 | # Escape spaces. We may need to escape more characters for this |
| 142 | # to work properly. |
| 143 | parts.append('"%s"' % part) |
| 144 | else: |
| 145 | parts.append(part) |
| 146 | return " ".join(parts) |
| 147 | |
| 148 | |
| 149 | class SimpleProgressIndicator(ProgressIndicator): |
| 150 | |
| 151 | def Starting(self): |
| 152 | print 'Running %i tests' % len(self.cases) |
| 153 | |
| 154 | def Done(self): |
| 155 | print |
| 156 | for failed in self.failed: |
| 157 | self.PrintFailureHeader(failed.test) |
| 158 | if failed.output.stderr: |
| 159 | print "--- stderr ---" |
| 160 | print failed.output.stderr.strip() |
| 161 | if failed.output.stdout: |
| 162 | print "--- stdout ---" |
| 163 | print failed.output.stdout.strip() |
| 164 | print "Command: %s" % EscapeCommand(failed.command) |
| 165 | if failed.HasCrashed(): |
| 166 | print "--- CRASHED ---" |
| 167 | if failed.HasTimedOut(): |
| 168 | print "--- TIMEOUT ---" |
| 169 | if len(self.failed) == 0: |
| 170 | print "===" |
| 171 | print "=== All tests succeeded" |
| 172 | print "===" |
| 173 | else: |
| 174 | print |
| 175 | print "===" |
| 176 | print "=== %i tests failed" % len(self.failed) |
| 177 | if self.crashed > 0: |
| 178 | print "=== %i tests CRASHED" % self.crashed |
| 179 | print "===" |
| 180 | |
| 181 | |
| 182 | class VerboseProgressIndicator(SimpleProgressIndicator): |
| 183 | |
| 184 | def AboutToRun(self, case): |
| 185 | print 'Starting %s...' % case.GetLabel() |
| 186 | sys.stdout.flush() |
| 187 | |
| 188 | def HasRun(self, output): |
| 189 | if output.UnexpectedOutput(): |
| 190 | if output.HasCrashed(): |
| 191 | outcome = 'CRASH' |
| 192 | else: |
| 193 | outcome = 'FAIL' |
| 194 | else: |
| 195 | outcome = 'pass' |
| 196 | print 'Done running %s: %s' % (output.test.GetLabel(), outcome) |
| 197 | |
| 198 | |
| 199 | class DotsProgressIndicator(SimpleProgressIndicator): |
| 200 | |
| 201 | def AboutToRun(self, case): |
| 202 | pass |
| 203 | |
| 204 | def HasRun(self, output): |
| 205 | total = self.succeeded + len(self.failed) |
| 206 | if (total > 1) and (total % 50 == 1): |
| 207 | sys.stdout.write('\n') |
| 208 | if output.UnexpectedOutput(): |
| 209 | if output.HasCrashed(): |
| 210 | sys.stdout.write('C') |
| 211 | sys.stdout.flush() |
| 212 | elif output.HasTimedOut(): |
| 213 | sys.stdout.write('T') |
| 214 | sys.stdout.flush() |
| 215 | else: |
| 216 | sys.stdout.write('F') |
| 217 | sys.stdout.flush() |
| 218 | else: |
| 219 | sys.stdout.write('.') |
| 220 | sys.stdout.flush() |
| 221 | |
| 222 | |
| 223 | class CompactProgressIndicator(ProgressIndicator): |
| 224 | |
| 225 | def __init__(self, cases, templates): |
| 226 | super(CompactProgressIndicator, self).__init__(cases) |
| 227 | self.templates = templates |
| 228 | self.last_status_length = 0 |
| 229 | self.start_time = time.time() |
| 230 | |
| 231 | def Starting(self): |
| 232 | pass |
| 233 | |
| 234 | def Done(self): |
| 235 | self.PrintProgress('Done') |
| 236 | |
| 237 | def AboutToRun(self, case): |
| 238 | self.PrintProgress(case.GetLabel()) |
| 239 | |
| 240 | def HasRun(self, output): |
| 241 | if output.UnexpectedOutput(): |
| 242 | self.ClearLine(self.last_status_length) |
| 243 | self.PrintFailureHeader(output.test) |
| 244 | stdout = output.output.stdout.strip() |
| 245 | if len(stdout): |
| 246 | print self.templates['stdout'] % stdout |
| 247 | stderr = output.output.stderr.strip() |
| 248 | if len(stderr): |
| 249 | print self.templates['stderr'] % stderr |
| 250 | print "Command: %s" % EscapeCommand(output.command) |
| 251 | if output.HasCrashed(): |
| 252 | print "--- CRASHED ---" |
| 253 | if output.HasTimedOut(): |
| 254 | print "--- TIMEOUT ---" |
| 255 | |
| 256 | def Truncate(self, str, length): |
| 257 | if length and (len(str) > (length - 3)): |
| 258 | return str[:(length-3)] + "..." |
| 259 | else: |
| 260 | return str |
| 261 | |
| 262 | def PrintProgress(self, name): |
| 263 | self.ClearLine(self.last_status_length) |
| 264 | elapsed = time.time() - self.start_time |
| 265 | status = self.templates['status_line'] % { |
| 266 | 'passed': self.succeeded, |
| 267 | 'remaining': (((self.total - self.remaining) * 100) // self.total), |
| 268 | 'failed': len(self.failed), |
| 269 | 'test': name, |
| 270 | 'mins': int(elapsed) / 60, |
| 271 | 'secs': int(elapsed) % 60 |
| 272 | } |
| 273 | status = self.Truncate(status, 78) |
| 274 | self.last_status_length = len(status) |
| 275 | print status, |
| 276 | sys.stdout.flush() |
| 277 | |
| 278 | |
| 279 | class ColorProgressIndicator(CompactProgressIndicator): |
| 280 | |
| 281 | def __init__(self, cases): |
| 282 | templates = { |
| 283 | 'status_line': "[%(mins)02i:%(secs)02i|\033[34m%%%(remaining) 4d\033[0m|\033[32m+%(passed) 4d\033[0m|\033[31m-%(failed) 4d\033[0m]: %(test)s", |
| 284 | 'stdout': "\033[1m%s\033[0m", |
| 285 | 'stderr': "\033[31m%s\033[0m", |
| 286 | } |
| 287 | super(ColorProgressIndicator, self).__init__(cases, templates) |
| 288 | |
| 289 | def ClearLine(self, last_line_length): |
| 290 | print "\033[1K\r", |
| 291 | |
| 292 | |
| 293 | class MonochromeProgressIndicator(CompactProgressIndicator): |
| 294 | |
| 295 | def __init__(self, cases): |
| 296 | templates = { |
| 297 | 'status_line': "[%(mins)02i:%(secs)02i|%%%(remaining) 4d|+%(passed) 4d|-%(failed) 4d]: %(test)s", |
| 298 | 'stdout': '%s', |
| 299 | 'stderr': '%s', |
| 300 | 'clear': lambda last_line_length: ("\r" + (" " * last_line_length) + "\r"), |
| 301 | 'max_length': 78 |
| 302 | } |
| 303 | super(MonochromeProgressIndicator, self).__init__(cases, templates) |
| 304 | |
| 305 | def ClearLine(self, last_line_length): |
| 306 | print ("\r" + (" " * last_line_length) + "\r"), |
| 307 | |
| 308 | |
| 309 | PROGRESS_INDICATORS = { |
| 310 | 'verbose': VerboseProgressIndicator, |
| 311 | 'dots': DotsProgressIndicator, |
| 312 | 'color': ColorProgressIndicator, |
| 313 | 'mono': MonochromeProgressIndicator |
| 314 | } |
| 315 | |
| 316 | |
| 317 | # ------------------------- |
| 318 | # --- F r a m e w o r k --- |
| 319 | # ------------------------- |
| 320 | |
| 321 | |
| 322 | class CommandOutput(object): |
| 323 | |
| 324 | def __init__(self, exit_code, timed_out, stdout, stderr): |
| 325 | self.exit_code = exit_code |
| 326 | self.timed_out = timed_out |
| 327 | self.stdout = stdout |
| 328 | self.stderr = stderr |
Steve Block | 3ce2e20 | 2009-11-05 08:53:23 +0000 | [diff] [blame] | 329 | self.failed = None |
Steve Block | a7e24c1 | 2009-10-30 11:49:00 +0000 | [diff] [blame] | 330 | |
| 331 | |
| 332 | class TestCase(object): |
| 333 | |
| 334 | def __init__(self, context, path): |
| 335 | self.path = path |
| 336 | self.context = context |
Steve Block | a7e24c1 | 2009-10-30 11:49:00 +0000 | [diff] [blame] | 337 | self.duration = None |
| 338 | |
| 339 | def IsNegative(self): |
| 340 | return False |
| 341 | |
| 342 | def CompareTime(self, other): |
| 343 | return cmp(other.duration, self.duration) |
| 344 | |
| 345 | def DidFail(self, output): |
Steve Block | 3ce2e20 | 2009-11-05 08:53:23 +0000 | [diff] [blame] | 346 | if output.failed is None: |
| 347 | output.failed = self.IsFailureOutput(output) |
| 348 | return output.failed |
Steve Block | a7e24c1 | 2009-10-30 11:49:00 +0000 | [diff] [blame] | 349 | |
| 350 | def IsFailureOutput(self, output): |
| 351 | return output.exit_code != 0 |
| 352 | |
| 353 | def GetSource(self): |
| 354 | return "(no source available)" |
| 355 | |
| 356 | def RunCommand(self, command): |
| 357 | full_command = self.context.processor(command) |
| 358 | output = Execute(full_command, self.context, self.context.timeout) |
| 359 | self.Cleanup() |
| 360 | return TestOutput(self, full_command, output) |
| 361 | |
Steve Block | d0582a6 | 2009-12-15 09:54:21 +0000 | [diff] [blame] | 362 | def BeforeRun(self): |
| 363 | pass |
| 364 | |
| 365 | def AfterRun(self): |
| 366 | pass |
| 367 | |
Steve Block | a7e24c1 | 2009-10-30 11:49:00 +0000 | [diff] [blame] | 368 | def Run(self): |
Steve Block | d0582a6 | 2009-12-15 09:54:21 +0000 | [diff] [blame] | 369 | self.BeforeRun() |
| 370 | try: |
| 371 | result = self.RunCommand(self.GetCommand()) |
| 372 | finally: |
| 373 | self.AfterRun() |
| 374 | return result |
Steve Block | a7e24c1 | 2009-10-30 11:49:00 +0000 | [diff] [blame] | 375 | |
| 376 | def Cleanup(self): |
| 377 | return |
| 378 | |
| 379 | |
| 380 | class TestOutput(object): |
| 381 | |
| 382 | def __init__(self, test, command, output): |
| 383 | self.test = test |
| 384 | self.command = command |
| 385 | self.output = output |
| 386 | |
| 387 | def UnexpectedOutput(self): |
| 388 | if self.HasCrashed(): |
| 389 | outcome = CRASH |
| 390 | elif self.HasTimedOut(): |
| 391 | outcome = TIMEOUT |
| 392 | elif self.HasFailed(): |
| 393 | outcome = FAIL |
| 394 | else: |
| 395 | outcome = PASS |
| 396 | return not outcome in self.test.outcomes |
| 397 | |
| 398 | def HasCrashed(self): |
| 399 | if utils.IsWindows(): |
| 400 | return 0x80000000 & self.output.exit_code and not (0x3FFFFF00 & self.output.exit_code) |
| 401 | else: |
| 402 | # Timed out tests will have exit_code -signal.SIGTERM. |
| 403 | if self.output.timed_out: |
| 404 | return False |
| 405 | return self.output.exit_code < 0 and \ |
| 406 | self.output.exit_code != -signal.SIGABRT |
| 407 | |
| 408 | def HasTimedOut(self): |
| 409 | return self.output.timed_out; |
| 410 | |
| 411 | def HasFailed(self): |
| 412 | execution_failed = self.test.DidFail(self.output) |
| 413 | if self.test.IsNegative(): |
| 414 | return not execution_failed |
| 415 | else: |
| 416 | return execution_failed |
| 417 | |
| 418 | |
| 419 | def KillProcessWithID(pid): |
| 420 | if utils.IsWindows(): |
| 421 | os.popen('taskkill /T /F /PID %d' % pid) |
| 422 | else: |
| 423 | os.kill(pid, signal.SIGTERM) |
| 424 | |
| 425 | |
| 426 | MAX_SLEEP_TIME = 0.1 |
| 427 | INITIAL_SLEEP_TIME = 0.0001 |
| 428 | SLEEP_TIME_FACTOR = 1.25 |
| 429 | |
| 430 | SEM_INVALID_VALUE = -1 |
| 431 | SEM_NOGPFAULTERRORBOX = 0x0002 # Microsoft Platform SDK WinBase.h |
| 432 | |
| 433 | def Win32SetErrorMode(mode): |
| 434 | prev_error_mode = SEM_INVALID_VALUE |
| 435 | try: |
| 436 | import ctypes |
| 437 | prev_error_mode = ctypes.windll.kernel32.SetErrorMode(mode); |
| 438 | except ImportError: |
| 439 | pass |
| 440 | return prev_error_mode |
| 441 | |
| 442 | def RunProcess(context, timeout, args, **rest): |
| 443 | if context.verbose: print "#", " ".join(args) |
| 444 | popen_args = args |
| 445 | prev_error_mode = SEM_INVALID_VALUE; |
| 446 | if utils.IsWindows(): |
| 447 | popen_args = '"' + subprocess.list2cmdline(args) + '"' |
| 448 | if context.suppress_dialogs: |
| 449 | # Try to change the error mode to avoid dialogs on fatal errors. Don't |
| 450 | # touch any existing error mode flags by merging the existing error mode. |
| 451 | # See http://blogs.msdn.com/oldnewthing/archive/2004/07/27/198410.aspx. |
| 452 | error_mode = SEM_NOGPFAULTERRORBOX; |
| 453 | prev_error_mode = Win32SetErrorMode(error_mode); |
| 454 | Win32SetErrorMode(error_mode | prev_error_mode); |
| 455 | process = subprocess.Popen( |
| 456 | shell = utils.IsWindows(), |
| 457 | args = popen_args, |
| 458 | **rest |
| 459 | ) |
| 460 | if utils.IsWindows() and context.suppress_dialogs and prev_error_mode != SEM_INVALID_VALUE: |
| 461 | Win32SetErrorMode(prev_error_mode) |
| 462 | # Compute the end time - if the process crosses this limit we |
| 463 | # consider it timed out. |
| 464 | if timeout is None: end_time = None |
| 465 | else: end_time = time.time() + timeout |
| 466 | timed_out = False |
| 467 | # Repeatedly check the exit code from the process in a |
| 468 | # loop and keep track of whether or not it times out. |
| 469 | exit_code = None |
| 470 | sleep_time = INITIAL_SLEEP_TIME |
| 471 | while exit_code is None: |
| 472 | if (not end_time is None) and (time.time() >= end_time): |
| 473 | # Kill the process and wait for it to exit. |
| 474 | KillProcessWithID(process.pid) |
| 475 | exit_code = process.wait() |
| 476 | timed_out = True |
| 477 | else: |
| 478 | exit_code = process.poll() |
| 479 | time.sleep(sleep_time) |
| 480 | sleep_time = sleep_time * SLEEP_TIME_FACTOR |
| 481 | if sleep_time > MAX_SLEEP_TIME: |
| 482 | sleep_time = MAX_SLEEP_TIME |
| 483 | return (process, exit_code, timed_out) |
| 484 | |
| 485 | |
| 486 | def PrintError(str): |
| 487 | sys.stderr.write(str) |
| 488 | sys.stderr.write('\n') |
| 489 | |
| 490 | |
| 491 | def CheckedUnlink(name): |
| 492 | try: |
| 493 | os.unlink(name) |
| 494 | except OSError, e: |
| 495 | PrintError("os.unlink() " + str(e)) |
| 496 | |
| 497 | |
| 498 | def Execute(args, context, timeout=None): |
| 499 | (fd_out, outname) = tempfile.mkstemp() |
| 500 | (fd_err, errname) = tempfile.mkstemp() |
| 501 | (process, exit_code, timed_out) = RunProcess( |
| 502 | context, |
| 503 | timeout, |
| 504 | args = args, |
| 505 | stdout = fd_out, |
| 506 | stderr = fd_err, |
| 507 | ) |
| 508 | os.close(fd_out) |
| 509 | os.close(fd_err) |
| 510 | output = file(outname).read() |
| 511 | errors = file(errname).read() |
| 512 | CheckedUnlink(outname) |
| 513 | CheckedUnlink(errname) |
| 514 | return CommandOutput(exit_code, timed_out, output, errors) |
| 515 | |
| 516 | |
| 517 | def ExecuteNoCapture(args, context, timeout=None): |
| 518 | (process, exit_code, timed_out) = RunProcess( |
| 519 | context, |
| 520 | timeout, |
| 521 | args = args, |
| 522 | ) |
| 523 | return CommandOutput(exit_code, False, "", "") |
| 524 | |
| 525 | |
| 526 | def CarCdr(path): |
| 527 | if len(path) == 0: |
| 528 | return (None, [ ]) |
| 529 | else: |
| 530 | return (path[0], path[1:]) |
| 531 | |
| 532 | |
| 533 | class TestConfiguration(object): |
| 534 | |
| 535 | def __init__(self, context, root): |
| 536 | self.context = context |
| 537 | self.root = root |
| 538 | |
| 539 | def Contains(self, path, file): |
| 540 | if len(path) > len(file): |
| 541 | return False |
| 542 | for i in xrange(len(path)): |
| 543 | if not path[i].match(file[i]): |
| 544 | return False |
| 545 | return True |
| 546 | |
| 547 | def GetTestStatus(self, sections, defs): |
| 548 | pass |
| 549 | |
| 550 | |
| 551 | class TestSuite(object): |
| 552 | |
| 553 | def __init__(self, name): |
| 554 | self.name = name |
| 555 | |
| 556 | def GetName(self): |
| 557 | return self.name |
| 558 | |
| 559 | |
| 560 | class TestRepository(TestSuite): |
| 561 | |
| 562 | def __init__(self, path): |
| 563 | normalized_path = abspath(path) |
| 564 | super(TestRepository, self).__init__(basename(normalized_path)) |
| 565 | self.path = normalized_path |
| 566 | self.is_loaded = False |
| 567 | self.config = None |
| 568 | |
| 569 | def GetConfiguration(self, context): |
| 570 | if self.is_loaded: |
| 571 | return self.config |
| 572 | self.is_loaded = True |
| 573 | file = None |
| 574 | try: |
| 575 | (file, pathname, description) = imp.find_module('testcfg', [ self.path ]) |
| 576 | module = imp.load_module('testcfg', file, pathname, description) |
| 577 | self.config = module.GetConfiguration(context, self.path) |
| 578 | finally: |
| 579 | if file: |
| 580 | file.close() |
| 581 | return self.config |
| 582 | |
| 583 | def GetBuildRequirements(self, path, context): |
| 584 | return self.GetConfiguration(context).GetBuildRequirements() |
| 585 | |
| 586 | def ListTests(self, current_path, path, context, mode): |
| 587 | return self.GetConfiguration(context).ListTests(current_path, path, mode) |
| 588 | |
| 589 | def GetTestStatus(self, context, sections, defs): |
| 590 | self.GetConfiguration(context).GetTestStatus(sections, defs) |
| 591 | |
| 592 | |
| 593 | class LiteralTestSuite(TestSuite): |
| 594 | |
| 595 | def __init__(self, tests): |
| 596 | super(LiteralTestSuite, self).__init__('root') |
| 597 | self.tests = tests |
| 598 | |
| 599 | def GetBuildRequirements(self, path, context): |
| 600 | (name, rest) = CarCdr(path) |
| 601 | result = [ ] |
| 602 | for test in self.tests: |
| 603 | if not name or name.match(test.GetName()): |
| 604 | result += test.GetBuildRequirements(rest, context) |
| 605 | return result |
| 606 | |
| 607 | def ListTests(self, current_path, path, context, mode): |
| 608 | (name, rest) = CarCdr(path) |
| 609 | result = [ ] |
| 610 | for test in self.tests: |
| 611 | test_name = test.GetName() |
| 612 | if not name or name.match(test_name): |
| 613 | full_path = current_path + [test_name] |
| 614 | result += test.ListTests(full_path, path, context, mode) |
| 615 | return result |
| 616 | |
| 617 | def GetTestStatus(self, context, sections, defs): |
| 618 | for test in self.tests: |
| 619 | test.GetTestStatus(context, sections, defs) |
| 620 | |
| 621 | |
| 622 | SUFFIX = {'debug': '_g', 'release': ''} |
| 623 | |
| 624 | |
| 625 | class Context(object): |
| 626 | |
| 627 | def __init__(self, workspace, buildspace, verbose, vm, timeout, processor, suppress_dialogs): |
| 628 | self.workspace = workspace |
| 629 | self.buildspace = buildspace |
| 630 | self.verbose = verbose |
| 631 | self.vm_root = vm |
| 632 | self.timeout = timeout |
| 633 | self.processor = processor |
| 634 | self.suppress_dialogs = suppress_dialogs |
| 635 | |
| 636 | def GetVm(self, mode): |
| 637 | name = self.vm_root + SUFFIX[mode] |
| 638 | if utils.IsWindows() and not name.endswith('.exe'): |
| 639 | name = name + '.exe' |
| 640 | return name |
| 641 | |
Leon Clarke | e46be81 | 2010-01-19 14:06:41 +0000 | [diff] [blame] | 642 | def RunTestCases(cases_to_run, progress, tasks): |
Steve Block | a7e24c1 | 2009-10-30 11:49:00 +0000 | [diff] [blame] | 643 | progress = PROGRESS_INDICATORS[progress](cases_to_run) |
| 644 | return progress.Run(tasks) |
| 645 | |
| 646 | |
| 647 | def BuildRequirements(context, requirements, mode, scons_flags): |
| 648 | command_line = (['scons', '-Y', context.workspace, 'mode=' + ",".join(mode)] |
| 649 | + requirements |
| 650 | + scons_flags) |
| 651 | output = ExecuteNoCapture(command_line, context) |
| 652 | return output.exit_code == 0 |
| 653 | |
| 654 | |
| 655 | # ------------------------------------------- |
| 656 | # --- T e s t C o n f i g u r a t i o n --- |
| 657 | # ------------------------------------------- |
| 658 | |
| 659 | |
| 660 | SKIP = 'skip' |
| 661 | FAIL = 'fail' |
| 662 | PASS = 'pass' |
| 663 | OKAY = 'okay' |
| 664 | TIMEOUT = 'timeout' |
| 665 | CRASH = 'crash' |
| 666 | SLOW = 'slow' |
| 667 | |
| 668 | |
| 669 | class Expression(object): |
| 670 | pass |
| 671 | |
| 672 | |
| 673 | class Constant(Expression): |
| 674 | |
| 675 | def __init__(self, value): |
| 676 | self.value = value |
| 677 | |
| 678 | def Evaluate(self, env, defs): |
| 679 | return self.value |
| 680 | |
| 681 | |
| 682 | class Variable(Expression): |
| 683 | |
| 684 | def __init__(self, name): |
| 685 | self.name = name |
| 686 | |
| 687 | def GetOutcomes(self, env, defs): |
| 688 | if self.name in env: return ListSet([env[self.name]]) |
| 689 | else: return Nothing() |
| 690 | |
| 691 | |
| 692 | class Outcome(Expression): |
| 693 | |
| 694 | def __init__(self, name): |
| 695 | self.name = name |
| 696 | |
| 697 | def GetOutcomes(self, env, defs): |
| 698 | if self.name in defs: |
| 699 | return defs[self.name].GetOutcomes(env, defs) |
| 700 | else: |
| 701 | return ListSet([self.name]) |
| 702 | |
| 703 | |
| 704 | class Set(object): |
| 705 | pass |
| 706 | |
| 707 | |
| 708 | class ListSet(Set): |
| 709 | |
| 710 | def __init__(self, elms): |
| 711 | self.elms = elms |
| 712 | |
| 713 | def __str__(self): |
| 714 | return "ListSet%s" % str(self.elms) |
| 715 | |
| 716 | def Intersect(self, that): |
| 717 | if not isinstance(that, ListSet): |
| 718 | return that.Intersect(self) |
| 719 | return ListSet([ x for x in self.elms if x in that.elms ]) |
| 720 | |
| 721 | def Union(self, that): |
| 722 | if not isinstance(that, ListSet): |
| 723 | return that.Union(self) |
| 724 | return ListSet(self.elms + [ x for x in that.elms if x not in self.elms ]) |
| 725 | |
| 726 | def IsEmpty(self): |
| 727 | return len(self.elms) == 0 |
| 728 | |
| 729 | |
| 730 | class Everything(Set): |
| 731 | |
| 732 | def Intersect(self, that): |
| 733 | return that |
| 734 | |
| 735 | def Union(self, that): |
| 736 | return self |
| 737 | |
| 738 | def IsEmpty(self): |
| 739 | return False |
| 740 | |
| 741 | |
| 742 | class Nothing(Set): |
| 743 | |
| 744 | def Intersect(self, that): |
| 745 | return self |
| 746 | |
| 747 | def Union(self, that): |
| 748 | return that |
| 749 | |
| 750 | def IsEmpty(self): |
| 751 | return True |
| 752 | |
| 753 | |
| 754 | class Operation(Expression): |
| 755 | |
| 756 | def __init__(self, left, op, right): |
| 757 | self.left = left |
| 758 | self.op = op |
| 759 | self.right = right |
| 760 | |
| 761 | def Evaluate(self, env, defs): |
| 762 | if self.op == '||' or self.op == ',': |
| 763 | return self.left.Evaluate(env, defs) or self.right.Evaluate(env, defs) |
| 764 | elif self.op == 'if': |
| 765 | return False |
| 766 | elif self.op == '==': |
| 767 | inter = self.left.GetOutcomes(env, defs).Intersect(self.right.GetOutcomes(env, defs)) |
| 768 | return not inter.IsEmpty() |
| 769 | else: |
| 770 | assert self.op == '&&' |
| 771 | return self.left.Evaluate(env, defs) and self.right.Evaluate(env, defs) |
| 772 | |
| 773 | def GetOutcomes(self, env, defs): |
| 774 | if self.op == '||' or self.op == ',': |
| 775 | return self.left.GetOutcomes(env, defs).Union(self.right.GetOutcomes(env, defs)) |
| 776 | elif self.op == 'if': |
| 777 | if self.right.Evaluate(env, defs): return self.left.GetOutcomes(env, defs) |
| 778 | else: return Nothing() |
| 779 | else: |
| 780 | assert self.op == '&&' |
| 781 | return self.left.GetOutcomes(env, defs).Intersect(self.right.GetOutcomes(env, defs)) |
| 782 | |
| 783 | |
| 784 | def IsAlpha(str): |
| 785 | for char in str: |
| 786 | if not (char.isalpha() or char.isdigit() or char == '_'): |
| 787 | return False |
| 788 | return True |
| 789 | |
| 790 | |
| 791 | class Tokenizer(object): |
| 792 | """A simple string tokenizer that chops expressions into variables, |
| 793 | parens and operators""" |
| 794 | |
| 795 | def __init__(self, expr): |
| 796 | self.index = 0 |
| 797 | self.expr = expr |
| 798 | self.length = len(expr) |
| 799 | self.tokens = None |
| 800 | |
| 801 | def Current(self, length = 1): |
| 802 | if not self.HasMore(length): return "" |
| 803 | return self.expr[self.index:self.index+length] |
| 804 | |
| 805 | def HasMore(self, length = 1): |
| 806 | return self.index < self.length + (length - 1) |
| 807 | |
| 808 | def Advance(self, count = 1): |
| 809 | self.index = self.index + count |
| 810 | |
| 811 | def AddToken(self, token): |
| 812 | self.tokens.append(token) |
| 813 | |
| 814 | def SkipSpaces(self): |
| 815 | while self.HasMore() and self.Current().isspace(): |
| 816 | self.Advance() |
| 817 | |
| 818 | def Tokenize(self): |
| 819 | self.tokens = [ ] |
| 820 | while self.HasMore(): |
| 821 | self.SkipSpaces() |
| 822 | if not self.HasMore(): |
| 823 | return None |
| 824 | if self.Current() == '(': |
| 825 | self.AddToken('(') |
| 826 | self.Advance() |
| 827 | elif self.Current() == ')': |
| 828 | self.AddToken(')') |
| 829 | self.Advance() |
| 830 | elif self.Current() == '$': |
| 831 | self.AddToken('$') |
| 832 | self.Advance() |
| 833 | elif self.Current() == ',': |
| 834 | self.AddToken(',') |
| 835 | self.Advance() |
| 836 | elif IsAlpha(self.Current()): |
| 837 | buf = "" |
| 838 | while self.HasMore() and IsAlpha(self.Current()): |
| 839 | buf += self.Current() |
| 840 | self.Advance() |
| 841 | self.AddToken(buf) |
| 842 | elif self.Current(2) == '&&': |
| 843 | self.AddToken('&&') |
| 844 | self.Advance(2) |
| 845 | elif self.Current(2) == '||': |
| 846 | self.AddToken('||') |
| 847 | self.Advance(2) |
| 848 | elif self.Current(2) == '==': |
| 849 | self.AddToken('==') |
| 850 | self.Advance(2) |
| 851 | else: |
| 852 | return None |
| 853 | return self.tokens |
| 854 | |
| 855 | |
| 856 | class Scanner(object): |
| 857 | """A simple scanner that can serve out tokens from a given list""" |
| 858 | |
| 859 | def __init__(self, tokens): |
| 860 | self.tokens = tokens |
| 861 | self.length = len(tokens) |
| 862 | self.index = 0 |
| 863 | |
| 864 | def HasMore(self): |
| 865 | return self.index < self.length |
| 866 | |
| 867 | def Current(self): |
| 868 | return self.tokens[self.index] |
| 869 | |
| 870 | def Advance(self): |
| 871 | self.index = self.index + 1 |
| 872 | |
| 873 | |
| 874 | def ParseAtomicExpression(scan): |
| 875 | if scan.Current() == "true": |
| 876 | scan.Advance() |
| 877 | return Constant(True) |
| 878 | elif scan.Current() == "false": |
| 879 | scan.Advance() |
| 880 | return Constant(False) |
| 881 | elif IsAlpha(scan.Current()): |
| 882 | name = scan.Current() |
| 883 | scan.Advance() |
| 884 | return Outcome(name.lower()) |
| 885 | elif scan.Current() == '$': |
| 886 | scan.Advance() |
| 887 | if not IsAlpha(scan.Current()): |
| 888 | return None |
| 889 | name = scan.Current() |
| 890 | scan.Advance() |
| 891 | return Variable(name.lower()) |
| 892 | elif scan.Current() == '(': |
| 893 | scan.Advance() |
| 894 | result = ParseLogicalExpression(scan) |
| 895 | if (not result) or (scan.Current() != ')'): |
| 896 | return None |
| 897 | scan.Advance() |
| 898 | return result |
| 899 | else: |
| 900 | return None |
| 901 | |
| 902 | |
| 903 | BINARIES = ['=='] |
| 904 | def ParseOperatorExpression(scan): |
| 905 | left = ParseAtomicExpression(scan) |
| 906 | if not left: return None |
| 907 | while scan.HasMore() and (scan.Current() in BINARIES): |
| 908 | op = scan.Current() |
| 909 | scan.Advance() |
| 910 | right = ParseOperatorExpression(scan) |
| 911 | if not right: |
| 912 | return None |
| 913 | left = Operation(left, op, right) |
| 914 | return left |
| 915 | |
| 916 | |
| 917 | def ParseConditionalExpression(scan): |
| 918 | left = ParseOperatorExpression(scan) |
| 919 | if not left: return None |
| 920 | while scan.HasMore() and (scan.Current() == 'if'): |
| 921 | scan.Advance() |
| 922 | right = ParseOperatorExpression(scan) |
| 923 | if not right: |
| 924 | return None |
| 925 | left= Operation(left, 'if', right) |
| 926 | return left |
| 927 | |
| 928 | |
| 929 | LOGICALS = ["&&", "||", ","] |
| 930 | def ParseLogicalExpression(scan): |
| 931 | left = ParseConditionalExpression(scan) |
| 932 | if not left: return None |
| 933 | while scan.HasMore() and (scan.Current() in LOGICALS): |
| 934 | op = scan.Current() |
| 935 | scan.Advance() |
| 936 | right = ParseConditionalExpression(scan) |
| 937 | if not right: |
| 938 | return None |
| 939 | left = Operation(left, op, right) |
| 940 | return left |
| 941 | |
| 942 | |
| 943 | def ParseCondition(expr): |
| 944 | """Parses a logical expression into an Expression object""" |
| 945 | tokens = Tokenizer(expr).Tokenize() |
| 946 | if not tokens: |
| 947 | print "Malformed expression: '%s'" % expr |
| 948 | return None |
| 949 | scan = Scanner(tokens) |
| 950 | ast = ParseLogicalExpression(scan) |
| 951 | if not ast: |
| 952 | print "Malformed expression: '%s'" % expr |
| 953 | return None |
| 954 | if scan.HasMore(): |
| 955 | print "Malformed expression: '%s'" % expr |
| 956 | return None |
| 957 | return ast |
| 958 | |
| 959 | |
| 960 | class ClassifiedTest(object): |
| 961 | |
| 962 | def __init__(self, case, outcomes): |
| 963 | self.case = case |
| 964 | self.outcomes = outcomes |
| 965 | |
| 966 | |
| 967 | class Configuration(object): |
| 968 | """The parsed contents of a configuration file""" |
| 969 | |
| 970 | def __init__(self, sections, defs): |
| 971 | self.sections = sections |
| 972 | self.defs = defs |
| 973 | |
| 974 | def ClassifyTests(self, cases, env): |
| 975 | sections = [s for s in self.sections if s.condition.Evaluate(env, self.defs)] |
| 976 | all_rules = reduce(list.__add__, [s.rules for s in sections], []) |
| 977 | unused_rules = set(all_rules) |
| 978 | result = [ ] |
| 979 | all_outcomes = set([]) |
| 980 | for case in cases: |
| 981 | matches = [ r for r in all_rules if r.Contains(case.path) ] |
| 982 | outcomes = set([]) |
| 983 | for rule in matches: |
| 984 | outcomes = outcomes.union(rule.GetOutcomes(env, self.defs)) |
| 985 | unused_rules.discard(rule) |
| 986 | if not outcomes: |
| 987 | outcomes = [PASS] |
| 988 | case.outcomes = outcomes |
| 989 | all_outcomes = all_outcomes.union(outcomes) |
| 990 | result.append(ClassifiedTest(case, outcomes)) |
| 991 | return (result, list(unused_rules), all_outcomes) |
| 992 | |
| 993 | |
| 994 | class Section(object): |
| 995 | """A section of the configuration file. Sections are enabled or |
| 996 | disabled prior to running the tests, based on their conditions""" |
| 997 | |
| 998 | def __init__(self, condition): |
| 999 | self.condition = condition |
| 1000 | self.rules = [ ] |
| 1001 | |
| 1002 | def AddRule(self, rule): |
| 1003 | self.rules.append(rule) |
| 1004 | |
| 1005 | |
| 1006 | class Rule(object): |
| 1007 | """A single rule that specifies the expected outcome for a single |
| 1008 | test.""" |
| 1009 | |
| 1010 | def __init__(self, raw_path, path, value): |
| 1011 | self.raw_path = raw_path |
| 1012 | self.path = path |
| 1013 | self.value = value |
| 1014 | |
| 1015 | def GetOutcomes(self, env, defs): |
| 1016 | set = self.value.GetOutcomes(env, defs) |
| 1017 | assert isinstance(set, ListSet) |
| 1018 | return set.elms |
| 1019 | |
| 1020 | def Contains(self, path): |
| 1021 | if len(self.path) > len(path): |
| 1022 | return False |
| 1023 | for i in xrange(len(self.path)): |
| 1024 | if not self.path[i].match(path[i]): |
| 1025 | return False |
| 1026 | return True |
| 1027 | |
| 1028 | |
| 1029 | HEADER_PATTERN = re.compile(r'\[([^]]+)\]') |
| 1030 | RULE_PATTERN = re.compile(r'\s*([^: ]*)\s*:(.*)') |
| 1031 | DEF_PATTERN = re.compile(r'^def\s*(\w+)\s*=(.*)$') |
| 1032 | PREFIX_PATTERN = re.compile(r'^\s*prefix\s+([\w\_\.\-\/]+)$') |
| 1033 | |
| 1034 | |
| 1035 | def ReadConfigurationInto(path, sections, defs): |
| 1036 | current_section = Section(Constant(True)) |
| 1037 | sections.append(current_section) |
| 1038 | prefix = [] |
| 1039 | for line in utils.ReadLinesFrom(path): |
| 1040 | header_match = HEADER_PATTERN.match(line) |
| 1041 | if header_match: |
| 1042 | condition_str = header_match.group(1).strip() |
| 1043 | condition = ParseCondition(condition_str) |
| 1044 | new_section = Section(condition) |
| 1045 | sections.append(new_section) |
| 1046 | current_section = new_section |
| 1047 | continue |
| 1048 | rule_match = RULE_PATTERN.match(line) |
| 1049 | if rule_match: |
| 1050 | path = prefix + SplitPath(rule_match.group(1).strip()) |
| 1051 | value_str = rule_match.group(2).strip() |
| 1052 | value = ParseCondition(value_str) |
| 1053 | if not value: |
| 1054 | return False |
| 1055 | current_section.AddRule(Rule(rule_match.group(1), path, value)) |
| 1056 | continue |
| 1057 | def_match = DEF_PATTERN.match(line) |
| 1058 | if def_match: |
| 1059 | name = def_match.group(1).lower() |
| 1060 | value = ParseCondition(def_match.group(2).strip()) |
| 1061 | if not value: |
| 1062 | return False |
| 1063 | defs[name] = value |
| 1064 | continue |
| 1065 | prefix_match = PREFIX_PATTERN.match(line) |
| 1066 | if prefix_match: |
| 1067 | prefix = SplitPath(prefix_match.group(1).strip()) |
| 1068 | continue |
| 1069 | print "Malformed line: '%s'." % line |
| 1070 | return False |
| 1071 | return True |
| 1072 | |
| 1073 | |
| 1074 | # --------------- |
| 1075 | # --- M a i n --- |
| 1076 | # --------------- |
| 1077 | |
| 1078 | |
| 1079 | ARCH_GUESS = utils.GuessArchitecture() |
| 1080 | |
| 1081 | |
| 1082 | def BuildOptions(): |
| 1083 | result = optparse.OptionParser() |
| 1084 | result.add_option("-m", "--mode", help="The test modes in which to run (comma-separated)", |
| 1085 | default='release') |
| 1086 | result.add_option("-v", "--verbose", help="Verbose output", |
| 1087 | default=False, action="store_true") |
| 1088 | result.add_option("-S", dest="scons_flags", help="Flag to pass through to scons", |
| 1089 | default=[], action="append") |
| 1090 | result.add_option("-p", "--progress", |
| 1091 | help="The style of progress indicator (verbose, dots, color, mono)", |
| 1092 | choices=PROGRESS_INDICATORS.keys(), default="mono") |
| 1093 | result.add_option("--no-build", help="Don't build requirements", |
| 1094 | default=False, action="store_true") |
| 1095 | result.add_option("--build-only", help="Only build requirements, don't run the tests", |
| 1096 | default=False, action="store_true") |
| 1097 | result.add_option("--report", help="Print a summary of the tests to be run", |
| 1098 | default=False, action="store_true") |
| 1099 | result.add_option("-s", "--suite", help="A test suite", |
| 1100 | default=[], action="append") |
| 1101 | result.add_option("-t", "--timeout", help="Timeout in seconds", |
| 1102 | default=60, type="int") |
| 1103 | result.add_option("--arch", help='The architecture to run tests for', |
| 1104 | default='none') |
Steve Block | d0582a6 | 2009-12-15 09:54:21 +0000 | [diff] [blame] | 1105 | result.add_option("--snapshot", help="Run the tests with snapshot turned on", |
| 1106 | default=False, action="store_true") |
Steve Block | a7e24c1 | 2009-10-30 11:49:00 +0000 | [diff] [blame] | 1107 | result.add_option("--simulator", help="Run tests with architecture simulator", |
| 1108 | default='none') |
| 1109 | result.add_option("--special-command", default=None) |
| 1110 | result.add_option("--valgrind", help="Run tests through valgrind", |
| 1111 | default=False, action="store_true") |
| 1112 | result.add_option("--cat", help="Print the source of the tests", |
| 1113 | default=False, action="store_true") |
| 1114 | result.add_option("--warn-unused", help="Report unused rules", |
| 1115 | default=False, action="store_true") |
| 1116 | result.add_option("-j", help="The number of parallel tasks to run", |
| 1117 | default=1, type="int") |
| 1118 | result.add_option("--time", help="Print timing information after running", |
| 1119 | default=False, action="store_true") |
| 1120 | result.add_option("--suppress-dialogs", help="Suppress Windows dialogs for crashing tests", |
| 1121 | dest="suppress_dialogs", default=True, action="store_true") |
| 1122 | result.add_option("--no-suppress-dialogs", help="Display Windows dialogs for crashing tests", |
| 1123 | dest="suppress_dialogs", action="store_false") |
| 1124 | result.add_option("--shell", help="Path to V8 shell", default="shell"); |
| 1125 | return result |
| 1126 | |
| 1127 | |
| 1128 | def ProcessOptions(options): |
| 1129 | global VERBOSE |
| 1130 | VERBOSE = options.verbose |
| 1131 | options.mode = options.mode.split(',') |
| 1132 | for mode in options.mode: |
| 1133 | if not mode in ['debug', 'release']: |
| 1134 | print "Unknown mode %s" % mode |
| 1135 | return False |
| 1136 | if options.simulator != 'none': |
| 1137 | # Simulator argument was set. Make sure arch and simulator agree. |
| 1138 | if options.simulator != options.arch: |
| 1139 | if options.arch == 'none': |
| 1140 | options.arch = options.simulator |
| 1141 | else: |
| 1142 | print "Architecture %s does not match sim %s" %(options.arch, options.simulator) |
| 1143 | return False |
| 1144 | # Ensure that the simulator argument is handed down to scons. |
| 1145 | options.scons_flags.append("simulator=" + options.simulator) |
| 1146 | else: |
| 1147 | # If options.arch is not set by the command line and no simulator setting |
| 1148 | # was found, set the arch to the guess. |
| 1149 | if options.arch == 'none': |
| 1150 | options.arch = ARCH_GUESS |
| 1151 | options.scons_flags.append("arch=" + options.arch) |
Steve Block | d0582a6 | 2009-12-15 09:54:21 +0000 | [diff] [blame] | 1152 | if options.snapshot: |
| 1153 | options.scons_flags.append("snapshot=on") |
Steve Block | a7e24c1 | 2009-10-30 11:49:00 +0000 | [diff] [blame] | 1154 | return True |
| 1155 | |
| 1156 | |
| 1157 | REPORT_TEMPLATE = """\ |
| 1158 | Total: %(total)i tests |
| 1159 | * %(skipped)4d tests will be skipped |
| 1160 | * %(nocrash)4d tests are expected to be flaky but not crash |
| 1161 | * %(pass)4d tests are expected to pass |
| 1162 | * %(fail_ok)4d tests are expected to fail that we won't fix |
| 1163 | * %(fail)4d tests are expected to fail that we should fix\ |
| 1164 | """ |
| 1165 | |
| 1166 | def PrintReport(cases): |
| 1167 | def IsFlaky(o): |
| 1168 | return (PASS in o) and (FAIL in o) and (not CRASH in o) and (not OKAY in o) |
| 1169 | def IsFailOk(o): |
| 1170 | return (len(o) == 2) and (FAIL in o) and (OKAY in o) |
| 1171 | unskipped = [c for c in cases if not SKIP in c.outcomes] |
| 1172 | print REPORT_TEMPLATE % { |
| 1173 | 'total': len(cases), |
| 1174 | 'skipped': len(cases) - len(unskipped), |
| 1175 | 'nocrash': len([t for t in unskipped if IsFlaky(t.outcomes)]), |
| 1176 | 'pass': len([t for t in unskipped if list(t.outcomes) == [PASS]]), |
| 1177 | 'fail_ok': len([t for t in unskipped if IsFailOk(t.outcomes)]), |
| 1178 | 'fail': len([t for t in unskipped if list(t.outcomes) == [FAIL]]) |
| 1179 | } |
| 1180 | |
| 1181 | |
| 1182 | class Pattern(object): |
| 1183 | |
| 1184 | def __init__(self, pattern): |
| 1185 | self.pattern = pattern |
| 1186 | self.compiled = None |
| 1187 | |
| 1188 | def match(self, str): |
| 1189 | if not self.compiled: |
| 1190 | pattern = "^" + self.pattern.replace('*', '.*') + "$" |
| 1191 | self.compiled = re.compile(pattern) |
| 1192 | return self.compiled.match(str) |
| 1193 | |
| 1194 | def __str__(self): |
| 1195 | return self.pattern |
| 1196 | |
| 1197 | |
| 1198 | def SplitPath(s): |
| 1199 | stripped = [ c.strip() for c in s.split('/') ] |
| 1200 | return [ Pattern(s) for s in stripped if len(s) > 0 ] |
| 1201 | |
| 1202 | |
| 1203 | def GetSpecialCommandProcessor(value): |
| 1204 | if (not value) or (value.find('@') == -1): |
| 1205 | def ExpandCommand(args): |
| 1206 | return args |
| 1207 | return ExpandCommand |
| 1208 | else: |
| 1209 | pos = value.find('@') |
| 1210 | import urllib |
| 1211 | prefix = urllib.unquote(value[:pos]).split() |
| 1212 | suffix = urllib.unquote(value[pos+1:]).split() |
| 1213 | def ExpandCommand(args): |
| 1214 | return prefix + args + suffix |
| 1215 | return ExpandCommand |
| 1216 | |
| 1217 | |
| 1218 | BUILT_IN_TESTS = ['mjsunit', 'cctest', 'message'] |
| 1219 | |
| 1220 | |
| 1221 | def GetSuites(test_root): |
| 1222 | def IsSuite(path): |
| 1223 | return isdir(path) and exists(join(path, 'testcfg.py')) |
| 1224 | return [ f for f in os.listdir(test_root) if IsSuite(join(test_root, f)) ] |
| 1225 | |
| 1226 | |
| 1227 | def FormatTime(d): |
| 1228 | millis = round(d * 1000) % 1000 |
| 1229 | return time.strftime("%M:%S.", time.gmtime(d)) + ("%03i" % millis) |
| 1230 | |
| 1231 | |
| 1232 | def Main(): |
| 1233 | parser = BuildOptions() |
| 1234 | (options, args) = parser.parse_args() |
| 1235 | if not ProcessOptions(options): |
| 1236 | parser.print_help() |
| 1237 | return 1 |
| 1238 | |
| 1239 | workspace = abspath(join(dirname(sys.argv[0]), '..')) |
| 1240 | suites = GetSuites(join(workspace, 'test')) |
| 1241 | repositories = [TestRepository(join(workspace, 'test', name)) for name in suites] |
| 1242 | repositories += [TestRepository(a) for a in options.suite] |
| 1243 | |
| 1244 | root = LiteralTestSuite(repositories) |
| 1245 | if len(args) == 0: |
| 1246 | paths = [SplitPath(t) for t in BUILT_IN_TESTS] |
| 1247 | else: |
| 1248 | paths = [ ] |
| 1249 | for arg in args: |
| 1250 | path = SplitPath(arg) |
| 1251 | paths.append(path) |
| 1252 | |
| 1253 | # Check for --valgrind option. If enabled, we overwrite the special |
| 1254 | # command flag with a command that uses the run-valgrind.py script. |
| 1255 | if options.valgrind: |
| 1256 | run_valgrind = join(workspace, "tools", "run-valgrind.py") |
| 1257 | options.special_command = "python -u " + run_valgrind + " @" |
| 1258 | |
| 1259 | shell = abspath(options.shell) |
| 1260 | buildspace = dirname(shell) |
| 1261 | context = Context(workspace, buildspace, VERBOSE, |
| 1262 | shell, |
| 1263 | options.timeout, |
| 1264 | GetSpecialCommandProcessor(options.special_command), |
| 1265 | options.suppress_dialogs) |
| 1266 | # First build the required targets |
| 1267 | if not options.no_build: |
| 1268 | reqs = [ ] |
| 1269 | for path in paths: |
| 1270 | reqs += root.GetBuildRequirements(path, context) |
| 1271 | reqs = list(set(reqs)) |
| 1272 | if len(reqs) > 0: |
| 1273 | if options.j != 1: |
| 1274 | options.scons_flags += ['-j', str(options.j)] |
| 1275 | if not BuildRequirements(context, reqs, options.mode, options.scons_flags): |
| 1276 | return 1 |
| 1277 | |
| 1278 | # Just return if we are only building the targets for running the tests. |
| 1279 | if options.build_only: |
| 1280 | return 0 |
| 1281 | |
| 1282 | # Get status for tests |
| 1283 | sections = [ ] |
| 1284 | defs = { } |
| 1285 | root.GetTestStatus(context, sections, defs) |
| 1286 | config = Configuration(sections, defs) |
| 1287 | |
| 1288 | # List the tests |
| 1289 | all_cases = [ ] |
| 1290 | all_unused = [ ] |
| 1291 | unclassified_tests = [ ] |
| 1292 | globally_unused_rules = None |
| 1293 | for path in paths: |
| 1294 | for mode in options.mode: |
| 1295 | if not exists(context.GetVm(mode)): |
| 1296 | print "Can't find shell executable: '%s'" % context.GetVm(mode) |
| 1297 | continue |
| 1298 | env = { |
| 1299 | 'mode': mode, |
| 1300 | 'system': utils.GuessOS(), |
| 1301 | 'arch': options.arch, |
| 1302 | 'simulator': options.simulator |
| 1303 | } |
| 1304 | test_list = root.ListTests([], path, context, mode) |
| 1305 | unclassified_tests += test_list |
| 1306 | (cases, unused_rules, all_outcomes) = config.ClassifyTests(test_list, env) |
| 1307 | if globally_unused_rules is None: |
| 1308 | globally_unused_rules = set(unused_rules) |
| 1309 | else: |
| 1310 | globally_unused_rules = globally_unused_rules.intersection(unused_rules) |
| 1311 | all_cases += cases |
| 1312 | all_unused.append(unused_rules) |
| 1313 | |
| 1314 | if options.cat: |
| 1315 | visited = set() |
| 1316 | for test in unclassified_tests: |
| 1317 | key = tuple(test.path) |
| 1318 | if key in visited: |
| 1319 | continue |
| 1320 | visited.add(key) |
| 1321 | print "--- begin source: %s ---" % test.GetLabel() |
| 1322 | source = test.GetSource().strip() |
| 1323 | print source |
| 1324 | print "--- end source: %s ---" % test.GetLabel() |
| 1325 | return 0 |
| 1326 | |
| 1327 | if options.warn_unused: |
| 1328 | for rule in globally_unused_rules: |
| 1329 | print "Rule for '%s' was not used." % '/'.join([str(s) for s in rule.path]) |
| 1330 | |
| 1331 | if options.report: |
| 1332 | PrintReport(all_cases) |
| 1333 | |
| 1334 | result = None |
Leon Clarke | e46be81 | 2010-01-19 14:06:41 +0000 | [diff] [blame] | 1335 | def DoSkip(case): |
| 1336 | return SKIP in case.outcomes or SLOW in case.outcomes |
| 1337 | cases_to_run = [ c for c in all_cases if not DoSkip(c) ] |
| 1338 | if len(cases_to_run) == 0: |
Steve Block | a7e24c1 | 2009-10-30 11:49:00 +0000 | [diff] [blame] | 1339 | print "No tests to run." |
| 1340 | return 0 |
| 1341 | else: |
| 1342 | try: |
| 1343 | start = time.time() |
Leon Clarke | e46be81 | 2010-01-19 14:06:41 +0000 | [diff] [blame] | 1344 | if RunTestCases(cases_to_run, options.progress, options.j): |
Steve Block | a7e24c1 | 2009-10-30 11:49:00 +0000 | [diff] [blame] | 1345 | result = 0 |
| 1346 | else: |
| 1347 | result = 1 |
| 1348 | duration = time.time() - start |
| 1349 | except KeyboardInterrupt: |
| 1350 | print "Interrupted" |
| 1351 | return 1 |
| 1352 | |
| 1353 | if options.time: |
| 1354 | # Write the times to stderr to make it easy to separate from the |
| 1355 | # test output. |
| 1356 | print |
| 1357 | sys.stderr.write("--- Total time: %s ---\n" % FormatTime(duration)) |
Leon Clarke | e46be81 | 2010-01-19 14:06:41 +0000 | [diff] [blame] | 1358 | timed_tests = [ t.case for t in cases_to_run if not t.case.duration is None ] |
Steve Block | a7e24c1 | 2009-10-30 11:49:00 +0000 | [diff] [blame] | 1359 | timed_tests.sort(lambda a, b: a.CompareTime(b)) |
| 1360 | index = 1 |
| 1361 | for entry in timed_tests[:20]: |
| 1362 | t = FormatTime(entry.duration) |
| 1363 | sys.stderr.write("%4i (%s) %s\n" % (index, t, entry.GetLabel())) |
| 1364 | index += 1 |
| 1365 | |
| 1366 | return result |
| 1367 | |
| 1368 | |
| 1369 | if __name__ == '__main__': |
| 1370 | sys.exit(Main()) |