Upgrade to 3.29
Update V8 to 3.29.88.17 and update makefiles to support building on
all the relevant platforms.
Bug: 17370214
Change-Id: Ia3407c157fd8d72a93e23d8318ccaf6ecf77fa4e
diff --git a/tools/testrunner/local/__init__.py b/tools/testrunner/local/__init__.py
new file mode 100644
index 0000000..202a262
--- /dev/null
+++ b/tools/testrunner/local/__init__.py
@@ -0,0 +1,26 @@
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/tools/testrunner/local/commands.py b/tools/testrunner/local/commands.py
new file mode 100644
index 0000000..d6445d0
--- /dev/null
+++ b/tools/testrunner/local/commands.py
@@ -0,0 +1,151 @@
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import os
+import signal
+import subprocess
+import sys
+import tempfile
+import time
+
+from ..local import utils
+from ..objects import output
+
+
+def KillProcessWithID(pid):
+ if utils.IsWindows():
+ os.popen('taskkill /T /F /PID %d' % pid)
+ else:
+ os.kill(pid, signal.SIGTERM)
+
+
+MAX_SLEEP_TIME = 0.1
+INITIAL_SLEEP_TIME = 0.0001
+SLEEP_TIME_FACTOR = 1.25
+
+SEM_INVALID_VALUE = -1
+SEM_NOGPFAULTERRORBOX = 0x0002 # Microsoft Platform SDK WinBase.h
+
+
+def Win32SetErrorMode(mode):
+ prev_error_mode = SEM_INVALID_VALUE
+ try:
+ import ctypes
+ prev_error_mode = \
+ ctypes.windll.kernel32.SetErrorMode(mode) #@UndefinedVariable
+ except ImportError:
+ pass
+ return prev_error_mode
+
+
+def RunProcess(verbose, timeout, args, **rest):
+ if verbose: print "#", " ".join(args)
+ popen_args = args
+ prev_error_mode = SEM_INVALID_VALUE
+ if utils.IsWindows():
+ popen_args = subprocess.list2cmdline(args)
+ # Try to change the error mode to avoid dialogs on fatal errors. Don't
+ # touch any existing error mode flags by merging the existing error mode.
+ # See http://blogs.msdn.com/oldnewthing/archive/2004/07/27/198410.aspx.
+ error_mode = SEM_NOGPFAULTERRORBOX
+ prev_error_mode = Win32SetErrorMode(error_mode)
+ Win32SetErrorMode(error_mode | prev_error_mode)
+ process = subprocess.Popen(
+ shell=utils.IsWindows(),
+ args=popen_args,
+ **rest
+ )
+ if (utils.IsWindows() and prev_error_mode != SEM_INVALID_VALUE):
+ Win32SetErrorMode(prev_error_mode)
+ # Compute the end time - if the process crosses this limit we
+ # consider it timed out.
+ if timeout is None: end_time = None
+ else: end_time = time.time() + timeout
+ timed_out = False
+ # Repeatedly check the exit code from the process in a
+ # loop and keep track of whether or not it times out.
+ exit_code = None
+ sleep_time = INITIAL_SLEEP_TIME
+ while exit_code is None:
+ if (not end_time is None) and (time.time() >= end_time):
+ # Kill the process and wait for it to exit.
+ KillProcessWithID(process.pid)
+ exit_code = process.wait()
+ timed_out = True
+ else:
+ exit_code = process.poll()
+ time.sleep(sleep_time)
+ sleep_time = sleep_time * SLEEP_TIME_FACTOR
+ if sleep_time > MAX_SLEEP_TIME:
+ sleep_time = MAX_SLEEP_TIME
+ return (exit_code, timed_out)
+
+
+def PrintError(string):
+ sys.stderr.write(string)
+ sys.stderr.write("\n")
+
+
+def CheckedUnlink(name):
+ # On Windows, when run with -jN in parallel processes,
+ # OS often fails to unlink the temp file. Not sure why.
+ # Need to retry.
+ # Idea from https://bugs.webkit.org/attachment.cgi?id=75982&action=prettypatch
+ retry_count = 0
+ while retry_count < 30:
+ try:
+ os.unlink(name)
+ return
+ except OSError, e:
+ retry_count += 1
+ time.sleep(retry_count * 0.1)
+ PrintError("os.unlink() " + str(e))
+
+
+def Execute(args, verbose=False, timeout=None):
+ try:
+ args = [ c for c in args if c != "" ]
+ (fd_out, outname) = tempfile.mkstemp()
+ (fd_err, errname) = tempfile.mkstemp()
+ (exit_code, timed_out) = RunProcess(
+ verbose,
+ timeout,
+ args=args,
+ stdout=fd_out,
+ stderr=fd_err
+ )
+ finally:
+ # TODO(machenbach): A keyboard interrupt before the assignment to
+ # fd_out|err can lead to reference errors here.
+ os.close(fd_out)
+ os.close(fd_err)
+ out = file(outname).read()
+ errors = file(errname).read()
+ CheckedUnlink(outname)
+ CheckedUnlink(errname)
+ return output.Output(exit_code, timed_out, out, errors)
diff --git a/tools/testrunner/local/execution.py b/tools/testrunner/local/execution.py
new file mode 100644
index 0000000..36ce7be
--- /dev/null
+++ b/tools/testrunner/local/execution.py
@@ -0,0 +1,269 @@
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import os
+import shutil
+import time
+
+from pool import Pool
+from . import commands
+from . import perfdata
+from . import utils
+
+
+class Job(object):
+ def __init__(self, command, dep_command, test_id, timeout, verbose):
+ self.command = command
+ self.dep_command = dep_command
+ self.id = test_id
+ self.timeout = timeout
+ self.verbose = verbose
+
+
+def RunTest(job):
+ start_time = time.time()
+ if job.dep_command is not None:
+ dep_output = commands.Execute(job.dep_command, job.verbose, job.timeout)
+ # TODO(jkummerow): We approximate the test suite specific function
+ # IsFailureOutput() by just checking the exit code here. Currently
+ # only cctests define dependencies, for which this simplification is
+ # correct.
+ if dep_output.exit_code != 0:
+ return (job.id, dep_output, time.time() - start_time)
+ output = commands.Execute(job.command, job.verbose, job.timeout)
+ return (job.id, output, time.time() - start_time)
+
+class Runner(object):
+
+ def __init__(self, suites, progress_indicator, context):
+ self.datapath = os.path.join("out", "testrunner_data")
+ self.perf_data_manager = perfdata.PerfDataManager(self.datapath)
+ self.perfdata = self.perf_data_manager.GetStore(context.arch, context.mode)
+ self.perf_failures = False
+ self.printed_allocations = False
+ self.tests = [ t for s in suites for t in s.tests ]
+ if not context.no_sorting:
+ for t in self.tests:
+ t.duration = self.perfdata.FetchPerfData(t) or 1.0
+ self.tests.sort(key=lambda t: t.duration, reverse=True)
+ self._CommonInit(len(self.tests), progress_indicator, context)
+
+ def _CommonInit(self, num_tests, progress_indicator, context):
+ self.indicator = progress_indicator
+ progress_indicator.runner = self
+ self.context = context
+ self.succeeded = 0
+ self.total = num_tests
+ self.remaining = num_tests
+ self.failed = []
+ self.crashed = 0
+ self.reran_tests = 0
+
+ def _RunPerfSafe(self, fun):
+ try:
+ fun()
+ except Exception, e:
+ print("PerfData exception: %s" % e)
+ self.perf_failures = True
+
+ def _GetJob(self, test):
+ command = self.GetCommand(test)
+ timeout = self.context.timeout
+ if ("--stress-opt" in test.flags or
+ "--stress-opt" in self.context.mode_flags or
+ "--stress-opt" in self.context.extra_flags):
+ timeout *= 4
+ if test.dependency is not None:
+ dep_command = [ c.replace(test.path, test.dependency) for c in command ]
+ else:
+ dep_command = None
+ return Job(command, dep_command, test.id, timeout, self.context.verbose)
+
+ def _MaybeRerun(self, pool, test):
+ if test.run <= self.context.rerun_failures_count:
+ # Possibly rerun this test if its run count is below the maximum per
+ # test. <= as the flag controls reruns not including the first run.
+ if test.run == 1:
+ # Count the overall number of reran tests on the first rerun.
+ if self.reran_tests < self.context.rerun_failures_max:
+ self.reran_tests += 1
+ else:
+ # Don't rerun this if the overall number of rerun tests has been
+ # reached.
+ return
+ if test.run >= 2 and test.duration > self.context.timeout / 20.0:
+ # Rerun slow tests at most once.
+ return
+
+ # Rerun this test.
+ test.duration = None
+ test.output = None
+ test.run += 1
+ pool.add([self._GetJob(test)])
+ self.remaining += 1
+
+ def _ProcessTestNormal(self, test, result, pool):
+ self.indicator.AboutToRun(test)
+ test.output = result[1]
+ test.duration = result[2]
+ has_unexpected_output = test.suite.HasUnexpectedOutput(test)
+ if has_unexpected_output:
+ self.failed.append(test)
+ if test.output.HasCrashed():
+ self.crashed += 1
+ else:
+ self.succeeded += 1
+ self.remaining -= 1
+ # For the indicator, everything that happens after the first run is treated
+ # as unexpected even if it flakily passes in order to include it in the
+ # output.
+ self.indicator.HasRun(test, has_unexpected_output or test.run > 1)
+ if has_unexpected_output:
+ # Rerun test failures after the indicator has processed the results.
+ self._MaybeRerun(pool, test)
+ # Update the perf database if the test succeeded.
+ return not has_unexpected_output
+
+ def _ProcessTestPredictable(self, test, result, pool):
+ def HasDifferentAllocations(output1, output2):
+ def AllocationStr(stdout):
+ for line in reversed((stdout or "").splitlines()):
+ if line.startswith("### Allocations = "):
+ self.printed_allocations = True
+ return line
+ return ""
+ return (AllocationStr(output1.stdout) != AllocationStr(output2.stdout))
+
+ # Always pass the test duration for the database update.
+ test.duration = result[2]
+ if test.run == 1 and result[1].HasTimedOut():
+ # If we get a timeout in the first run, we are already in an
+ # unpredictable state. Just report it as a failure and don't rerun.
+ self.indicator.AboutToRun(test)
+ test.output = result[1]
+ self.remaining -= 1
+ self.failed.append(test)
+ self.indicator.HasRun(test, True)
+ if test.run > 1 and HasDifferentAllocations(test.output, result[1]):
+ # From the second run on, check for different allocations. If a
+ # difference is found, call the indicator twice to report both tests.
+ # All runs of each test are counted as one for the statistic.
+ self.indicator.AboutToRun(test)
+ self.remaining -= 1
+ self.failed.append(test)
+ self.indicator.HasRun(test, True)
+ self.indicator.AboutToRun(test)
+ test.output = result[1]
+ self.indicator.HasRun(test, True)
+ elif test.run >= 3:
+ # No difference on the third run -> report a success.
+ self.indicator.AboutToRun(test)
+ self.remaining -= 1
+ self.succeeded += 1
+ test.output = result[1]
+ self.indicator.HasRun(test, False)
+ else:
+ # No difference yet and less than three runs -> add another run and
+ # remember the output for comparison.
+ test.run += 1
+ test.output = result[1]
+ pool.add([self._GetJob(test)])
+ # Always update the perf database.
+ return True
+
+ def Run(self, jobs):
+ self.indicator.Starting()
+ self._RunInternal(jobs)
+ self.indicator.Done()
+ if self.failed or self.remaining:
+ return 1
+ return 0
+
+ def _RunInternal(self, jobs):
+ pool = Pool(jobs)
+ test_map = {}
+ # TODO(machenbach): Instead of filling the queue completely before
+ # pool.imap_unordered, make this a generator that already starts testing
+ # while the queue is filled.
+ queue = []
+ queued_exception = None
+ for test in self.tests:
+ assert test.id >= 0
+ test_map[test.id] = test
+ try:
+ queue.append([self._GetJob(test)])
+ except Exception, e:
+ # If this failed, save the exception and re-raise it later (after
+ # all other tests have had a chance to run).
+ queued_exception = e
+ continue
+ try:
+ it = pool.imap_unordered(RunTest, queue)
+ for result in it:
+ test = test_map[result[0]]
+ if self.context.predictable:
+ update_perf = self._ProcessTestPredictable(test, result, pool)
+ else:
+ update_perf = self._ProcessTestNormal(test, result, pool)
+ if update_perf:
+ self._RunPerfSafe(lambda: self.perfdata.UpdatePerfData(test))
+ finally:
+ pool.terminate()
+ self._RunPerfSafe(lambda: self.perf_data_manager.close())
+ if self.perf_failures:
+ # Nuke perf data in case of failures. This might not work on windows as
+ # some files might still be open.
+ print "Deleting perf test data due to db corruption."
+ shutil.rmtree(self.datapath)
+ if queued_exception:
+ raise queued_exception
+
+ # Make sure that any allocations were printed in predictable mode.
+ assert not self.context.predictable or self.printed_allocations
+
+ def GetCommand(self, test):
+ d8testflag = []
+ shell = test.suite.shell()
+ if shell == "d8":
+ d8testflag = ["--test"]
+ if utils.IsWindows():
+ shell += ".exe"
+ cmd = (self.context.command_prefix +
+ [os.path.abspath(os.path.join(self.context.shell_dir, shell))] +
+ d8testflag +
+ ["--random-seed=%s" % self.context.random_seed] +
+ test.suite.GetFlagsForTestCase(test, self.context) +
+ self.context.extra_flags)
+ return cmd
+
+
+class BreakNowException(Exception):
+ def __init__(self, value):
+ self.value = value
+ def __str__(self):
+ return repr(self.value)
diff --git a/tools/testrunner/local/junit_output.py b/tools/testrunner/local/junit_output.py
new file mode 100644
index 0000000..d2748fe
--- /dev/null
+++ b/tools/testrunner/local/junit_output.py
@@ -0,0 +1,48 @@
+# Copyright 2013 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import xml.etree.ElementTree as xml
+
+
+class JUnitTestOutput:
+ def __init__(self, test_suite_name):
+ self.root = xml.Element("testsuite")
+ self.root.attrib["name"] = test_suite_name
+
+ def HasRunTest(self, test_name, test_duration, test_failure):
+ testCaseElement = xml.Element("testcase")
+ testCaseElement.attrib["name"] = " ".join(test_name)
+ testCaseElement.attrib["time"] = str(round(test_duration, 3))
+ if len(test_failure):
+ failureElement = xml.Element("failure")
+ failureElement.text = test_failure
+ testCaseElement.append(failureElement)
+ self.root.append(testCaseElement)
+
+ def FinishAndWrite(self, file):
+ xml.ElementTree(self.root).write(file, "UTF-8")
diff --git a/tools/testrunner/local/perfdata.py b/tools/testrunner/local/perfdata.py
new file mode 100644
index 0000000..2979dc4
--- /dev/null
+++ b/tools/testrunner/local/perfdata.py
@@ -0,0 +1,120 @@
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import os
+import shelve
+import threading
+
+
+class PerfDataEntry(object):
+ def __init__(self):
+ self.avg = 0.0
+ self.count = 0
+
+ def AddResult(self, result):
+ kLearnRateLimiter = 99 # Greater value means slower learning.
+ # We use an approximation of the average of the last 100 results here:
+ # The existing average is weighted with kLearnRateLimiter (or less
+ # if there are fewer data points).
+ effective_count = min(self.count, kLearnRateLimiter)
+ self.avg = self.avg * effective_count + result
+ self.count = effective_count + 1
+ self.avg /= self.count
+
+
+class PerfDataStore(object):
+ def __init__(self, datadir, arch, mode):
+ filename = os.path.join(datadir, "%s.%s.perfdata" % (arch, mode))
+ self.database = shelve.open(filename, protocol=2)
+ self.closed = False
+ self.lock = threading.Lock()
+
+ def __del__(self):
+ self.close()
+
+ def close(self):
+ if self.closed: return
+ self.database.close()
+ self.closed = True
+
+ def GetKey(self, test):
+ """Computes the key used to access data for the given testcase."""
+ flags = "".join(test.flags)
+ return str("%s.%s.%s" % (test.suitename(), test.path, flags))
+
+ def FetchPerfData(self, test):
+ """Returns the observed duration for |test| as read from the store."""
+ key = self.GetKey(test)
+ if key in self.database:
+ return self.database[key].avg
+ return None
+
+ def UpdatePerfData(self, test):
+ """Updates the persisted value in the store with test.duration."""
+ testkey = self.GetKey(test)
+ self.RawUpdatePerfData(testkey, test.duration)
+
+ def RawUpdatePerfData(self, testkey, duration):
+ with self.lock:
+ if testkey in self.database:
+ entry = self.database[testkey]
+ else:
+ entry = PerfDataEntry()
+ entry.AddResult(duration)
+ self.database[testkey] = entry
+
+
+class PerfDataManager(object):
+ def __init__(self, datadir):
+ self.datadir = os.path.abspath(datadir)
+ if not os.path.exists(self.datadir):
+ os.makedirs(self.datadir)
+ self.stores = {} # Keyed by arch, then mode.
+ self.closed = False
+ self.lock = threading.Lock()
+
+ def __del__(self):
+ self.close()
+
+ def close(self):
+ if self.closed: return
+ for arch in self.stores:
+ modes = self.stores[arch]
+ for mode in modes:
+ store = modes[mode]
+ store.close()
+ self.closed = True
+
+ def GetStore(self, arch, mode):
+ with self.lock:
+ if not arch in self.stores:
+ self.stores[arch] = {}
+ modes = self.stores[arch]
+ if not mode in modes:
+ modes[mode] = PerfDataStore(self.datadir, arch, mode)
+ return modes[mode]
diff --git a/tools/testrunner/local/pool.py b/tools/testrunner/local/pool.py
new file mode 100644
index 0000000..602a2d4
--- /dev/null
+++ b/tools/testrunner/local/pool.py
@@ -0,0 +1,146 @@
+#!/usr/bin/env python
+# Copyright 2014 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from multiprocessing import Event, Process, Queue
+
+class NormalResult():
+ def __init__(self, result):
+ self.result = result
+ self.exception = False
+ self.break_now = False
+
+
+class ExceptionResult():
+ def __init__(self):
+ self.exception = True
+ self.break_now = False
+
+
+class BreakResult():
+ def __init__(self):
+ self.exception = False
+ self.break_now = True
+
+
+def Worker(fn, work_queue, done_queue, done):
+ """Worker to be run in a child process.
+ The worker stops on two conditions. 1. When the poison pill "STOP" is
+ reached or 2. when the event "done" is set."""
+ try:
+ for args in iter(work_queue.get, "STOP"):
+ if done.is_set():
+ break
+ try:
+ done_queue.put(NormalResult(fn(*args)))
+ except Exception, e:
+ print(">>> EXCEPTION: %s" % e)
+ done_queue.put(ExceptionResult())
+ except KeyboardInterrupt:
+ done_queue.put(BreakResult())
+
+
+class Pool():
+ """Distributes tasks to a number of worker processes.
+ New tasks can be added dynamically even after the workers have been started.
+ Requirement: Tasks can only be added from the parent process, e.g. while
+ consuming the results generator."""
+
+ # Factor to calculate the maximum number of items in the work/done queue.
+ # Necessary to not overflow the queue's pipe if a keyboard interrupt happens.
+ BUFFER_FACTOR = 4
+
+ def __init__(self, num_workers):
+ self.num_workers = num_workers
+ self.processes = []
+ self.terminated = False
+
+ # Invariant: count >= #work_queue + #done_queue. It is greater when a
+ # worker takes an item from the work_queue and before the result is
+ # submitted to the done_queue. It is equal when no worker is working,
+ # e.g. when all workers have finished, and when no results are processed.
+ # Count is only accessed by the parent process. Only the parent process is
+ # allowed to remove items from the done_queue and to add items to the
+ # work_queue.
+ self.count = 0
+ self.work_queue = Queue()
+ self.done_queue = Queue()
+ self.done = Event()
+
+ def imap_unordered(self, fn, gen):
+ """Maps function "fn" to items in generator "gen" on the worker processes
+ in an arbitrary order. The items are expected to be lists of arguments to
+ the function. Returns a results iterator."""
+ try:
+ gen = iter(gen)
+ self.advance = self._advance_more
+
+ for w in xrange(self.num_workers):
+ p = Process(target=Worker, args=(fn,
+ self.work_queue,
+ self.done_queue,
+ self.done))
+ self.processes.append(p)
+ p.start()
+
+ self.advance(gen)
+ while self.count > 0:
+ result = self.done_queue.get()
+ self.count -= 1
+ if result.exception:
+ # Ignore items with unexpected exceptions.
+ continue
+ elif result.break_now:
+ # A keyboard interrupt happened in one of the worker processes.
+ raise KeyboardInterrupt
+ else:
+ yield result.result
+ self.advance(gen)
+ finally:
+ self.terminate()
+
+ def _advance_more(self, gen):
+ while self.count < self.num_workers * self.BUFFER_FACTOR:
+ try:
+ self.work_queue.put(gen.next())
+ self.count += 1
+ except StopIteration:
+ self.advance = self._advance_empty
+ break
+
+ def _advance_empty(self, gen):
+ pass
+
+ def add(self, args):
+ """Adds an item to the work queue. Can be called dynamically while
+ processing the results from imap_unordered."""
+ self.work_queue.put(args)
+ self.count += 1
+
+ def terminate(self):
+ if self.terminated:
+ return
+ self.terminated = True
+
+ # For exceptional tear down set the "done" event to stop the workers before
+ # they empty the queue buffer.
+ self.done.set()
+
+ for p in self.processes:
+ # During normal tear down the workers block on get(). Feed a poison pill
+ # per worker to make them stop.
+ self.work_queue.put("STOP")
+
+ for p in self.processes:
+ p.join()
+
+ # Drain the queues to prevent failures when queues are garbage collected.
+ try:
+ while True: self.work_queue.get(False)
+ except:
+ pass
+ try:
+ while True: self.done_queue.get(False)
+ except:
+ pass
diff --git a/tools/testrunner/local/pool_unittest.py b/tools/testrunner/local/pool_unittest.py
new file mode 100644
index 0000000..bf2b3f8
--- /dev/null
+++ b/tools/testrunner/local/pool_unittest.py
@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+# Copyright 2014 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import unittest
+
+from pool import Pool
+
+def Run(x):
+ if x == 10:
+ raise Exception("Expected exception triggered by test.")
+ return x
+
+class PoolTest(unittest.TestCase):
+ def testNormal(self):
+ results = set()
+ pool = Pool(3)
+ for result in pool.imap_unordered(Run, [[x] for x in range(0, 10)]):
+ results.add(result)
+ self.assertEquals(set(range(0, 10)), results)
+
+ def testException(self):
+ results = set()
+ pool = Pool(3)
+ for result in pool.imap_unordered(Run, [[x] for x in range(0, 12)]):
+ # Item 10 will not appear in results due to an internal exception.
+ results.add(result)
+ expect = set(range(0, 12))
+ expect.remove(10)
+ self.assertEquals(expect, results)
+
+ def testAdd(self):
+ results = set()
+ pool = Pool(3)
+ for result in pool.imap_unordered(Run, [[x] for x in range(0, 10)]):
+ results.add(result)
+ if result < 30:
+ pool.add([result + 20])
+ self.assertEquals(set(range(0, 10) + range(20, 30) + range(40, 50)),
+ results)
diff --git a/tools/testrunner/local/progress.py b/tools/testrunner/local/progress.py
new file mode 100644
index 0000000..8caa58c
--- /dev/null
+++ b/tools/testrunner/local/progress.py
@@ -0,0 +1,344 @@
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import json
+import os
+import sys
+import time
+
+from . import junit_output
+
+
+ABS_PATH_PREFIX = os.getcwd() + os.sep
+
+
+def EscapeCommand(command):
+ parts = []
+ for part in command:
+ if ' ' in part:
+ # Escape spaces. We may need to escape more characters for this
+ # to work properly.
+ parts.append('"%s"' % part)
+ else:
+ parts.append(part)
+ return " ".join(parts)
+
+
+class ProgressIndicator(object):
+
+ def __init__(self):
+ self.runner = None
+
+ def Starting(self):
+ pass
+
+ def Done(self):
+ pass
+
+ def AboutToRun(self, test):
+ pass
+
+ def HasRun(self, test, has_unexpected_output):
+ pass
+
+ def PrintFailureHeader(self, test):
+ if test.suite.IsNegativeTest(test):
+ negative_marker = '[negative] '
+ else:
+ negative_marker = ''
+ print "=== %(label)s %(negative)s===" % {
+ 'label': test.GetLabel(),
+ 'negative': negative_marker
+ }
+
+
+class SimpleProgressIndicator(ProgressIndicator):
+ """Abstract base class for {Verbose,Dots}ProgressIndicator"""
+
+ def Starting(self):
+ print 'Running %i tests' % self.runner.total
+
+ def Done(self):
+ print
+ for failed in self.runner.failed:
+ self.PrintFailureHeader(failed)
+ if failed.output.stderr:
+ print "--- stderr ---"
+ print failed.output.stderr.strip()
+ if failed.output.stdout:
+ print "--- stdout ---"
+ print failed.output.stdout.strip()
+ print "Command: %s" % EscapeCommand(self.runner.GetCommand(failed))
+ if failed.output.HasCrashed():
+ print "exit code: %d" % failed.output.exit_code
+ print "--- CRASHED ---"
+ if failed.output.HasTimedOut():
+ print "--- TIMEOUT ---"
+ if len(self.runner.failed) == 0:
+ print "==="
+ print "=== All tests succeeded"
+ print "==="
+ else:
+ print
+ print "==="
+ print "=== %i tests failed" % len(self.runner.failed)
+ if self.runner.crashed > 0:
+ print "=== %i tests CRASHED" % self.runner.crashed
+ print "==="
+
+
+class VerboseProgressIndicator(SimpleProgressIndicator):
+
+ def AboutToRun(self, test):
+ print 'Starting %s...' % test.GetLabel()
+ sys.stdout.flush()
+
+ def HasRun(self, test, has_unexpected_output):
+ if has_unexpected_output:
+ if test.output.HasCrashed():
+ outcome = 'CRASH'
+ else:
+ outcome = 'FAIL'
+ else:
+ outcome = 'pass'
+ print 'Done running %s: %s' % (test.GetLabel(), outcome)
+
+
+class DotsProgressIndicator(SimpleProgressIndicator):
+
+ def HasRun(self, test, has_unexpected_output):
+ total = self.runner.succeeded + len(self.runner.failed)
+ if (total > 1) and (total % 50 == 1):
+ sys.stdout.write('\n')
+ if has_unexpected_output:
+ if test.output.HasCrashed():
+ sys.stdout.write('C')
+ sys.stdout.flush()
+ elif test.output.HasTimedOut():
+ sys.stdout.write('T')
+ sys.stdout.flush()
+ else:
+ sys.stdout.write('F')
+ sys.stdout.flush()
+ else:
+ sys.stdout.write('.')
+ sys.stdout.flush()
+
+
+class CompactProgressIndicator(ProgressIndicator):
+ """Abstract base class for {Color,Monochrome}ProgressIndicator"""
+
+ def __init__(self, templates):
+ super(CompactProgressIndicator, self).__init__()
+ self.templates = templates
+ self.last_status_length = 0
+ self.start_time = time.time()
+
+ def Done(self):
+ self.PrintProgress('Done')
+ print "" # Line break.
+
+ def AboutToRun(self, test):
+ self.PrintProgress(test.GetLabel())
+
+ def HasRun(self, test, has_unexpected_output):
+ if has_unexpected_output:
+ self.ClearLine(self.last_status_length)
+ self.PrintFailureHeader(test)
+ stdout = test.output.stdout.strip()
+ if len(stdout):
+ print self.templates['stdout'] % stdout
+ stderr = test.output.stderr.strip()
+ if len(stderr):
+ print self.templates['stderr'] % stderr
+ print "Command: %s" % EscapeCommand(self.runner.GetCommand(test))
+ if test.output.HasCrashed():
+ print "exit code: %d" % test.output.exit_code
+ print "--- CRASHED ---"
+ if test.output.HasTimedOut():
+ print "--- TIMEOUT ---"
+
+ def Truncate(self, string, length):
+ if length and (len(string) > (length - 3)):
+ return string[:(length - 3)] + "..."
+ else:
+ return string
+
+ def PrintProgress(self, name):
+ self.ClearLine(self.last_status_length)
+ elapsed = time.time() - self.start_time
+ status = self.templates['status_line'] % {
+ 'passed': self.runner.succeeded,
+ 'remaining': (((self.runner.total - self.runner.remaining) * 100) //
+ self.runner.total),
+ 'failed': len(self.runner.failed),
+ 'test': name,
+ 'mins': int(elapsed) / 60,
+ 'secs': int(elapsed) % 60
+ }
+ status = self.Truncate(status, 78)
+ self.last_status_length = len(status)
+ print status,
+ sys.stdout.flush()
+
+
+class ColorProgressIndicator(CompactProgressIndicator):
+
+ def __init__(self):
+ templates = {
+ 'status_line': ("[%(mins)02i:%(secs)02i|"
+ "\033[34m%%%(remaining) 4d\033[0m|"
+ "\033[32m+%(passed) 4d\033[0m|"
+ "\033[31m-%(failed) 4d\033[0m]: %(test)s"),
+ 'stdout': "\033[1m%s\033[0m",
+ 'stderr': "\033[31m%s\033[0m",
+ }
+ super(ColorProgressIndicator, self).__init__(templates)
+
+ def ClearLine(self, last_line_length):
+ print "\033[1K\r",
+
+
+class MonochromeProgressIndicator(CompactProgressIndicator):
+
+ def __init__(self):
+ templates = {
+ 'status_line': ("[%(mins)02i:%(secs)02i|%%%(remaining) 4d|"
+ "+%(passed) 4d|-%(failed) 4d]: %(test)s"),
+ 'stdout': '%s',
+ 'stderr': '%s',
+ }
+ super(MonochromeProgressIndicator, self).__init__(templates)
+
+ def ClearLine(self, last_line_length):
+ print ("\r" + (" " * last_line_length) + "\r"),
+
+
+class JUnitTestProgressIndicator(ProgressIndicator):
+
+ def __init__(self, progress_indicator, junitout, junittestsuite):
+ self.progress_indicator = progress_indicator
+ self.outputter = junit_output.JUnitTestOutput(junittestsuite)
+ if junitout:
+ self.outfile = open(junitout, "w")
+ else:
+ self.outfile = sys.stdout
+
+ def Starting(self):
+ self.progress_indicator.runner = self.runner
+ self.progress_indicator.Starting()
+
+ def Done(self):
+ self.progress_indicator.Done()
+ self.outputter.FinishAndWrite(self.outfile)
+ if self.outfile != sys.stdout:
+ self.outfile.close()
+
+ def AboutToRun(self, test):
+ self.progress_indicator.AboutToRun(test)
+
+ def HasRun(self, test, has_unexpected_output):
+ self.progress_indicator.HasRun(test, has_unexpected_output)
+ fail_text = ""
+ if has_unexpected_output:
+ stdout = test.output.stdout.strip()
+ if len(stdout):
+ fail_text += "stdout:\n%s\n" % stdout
+ stderr = test.output.stderr.strip()
+ if len(stderr):
+ fail_text += "stderr:\n%s\n" % stderr
+ fail_text += "Command: %s" % EscapeCommand(self.runner.GetCommand(test))
+ if test.output.HasCrashed():
+ fail_text += "exit code: %d\n--- CRASHED ---" % test.output.exit_code
+ if test.output.HasTimedOut():
+ fail_text += "--- TIMEOUT ---"
+ self.outputter.HasRunTest(
+ [test.GetLabel()] + self.runner.context.mode_flags + test.flags,
+ test.duration,
+ fail_text)
+
+
+class JsonTestProgressIndicator(ProgressIndicator):
+
+ def __init__(self, progress_indicator, json_test_results, arch, mode):
+ self.progress_indicator = progress_indicator
+ self.json_test_results = json_test_results
+ self.arch = arch
+ self.mode = mode
+ self.results = []
+
+ def Starting(self):
+ self.progress_indicator.runner = self.runner
+ self.progress_indicator.Starting()
+
+ def Done(self):
+ self.progress_indicator.Done()
+ complete_results = []
+ if os.path.exists(self.json_test_results):
+ with open(self.json_test_results, "r") as f:
+ # Buildbot might start out with an empty file.
+ complete_results = json.loads(f.read() or "[]")
+
+ complete_results.append({
+ "arch": self.arch,
+ "mode": self.mode,
+ "results": self.results,
+ })
+
+ with open(self.json_test_results, "w") as f:
+ f.write(json.dumps(complete_results))
+
+ def AboutToRun(self, test):
+ self.progress_indicator.AboutToRun(test)
+
+ def HasRun(self, test, has_unexpected_output):
+ self.progress_indicator.HasRun(test, has_unexpected_output)
+ if not has_unexpected_output:
+ # Omit tests that run as expected. Passing tests of reruns after failures
+ # will have unexpected_output to be reported here has well.
+ return
+
+ self.results.append({
+ "name": test.GetLabel(),
+ "flags": test.flags,
+ "command": EscapeCommand(self.runner.GetCommand(test)).replace(
+ ABS_PATH_PREFIX, ""),
+ "run": test.run,
+ "stdout": test.output.stdout,
+ "stderr": test.output.stderr,
+ "exit_code": test.output.exit_code,
+ "result": test.suite.GetOutcome(test),
+ })
+
+
+PROGRESS_INDICATORS = {
+ 'verbose': VerboseProgressIndicator,
+ 'dots': DotsProgressIndicator,
+ 'color': ColorProgressIndicator,
+ 'mono': MonochromeProgressIndicator
+}
diff --git a/tools/testrunner/local/statusfile.py b/tools/testrunner/local/statusfile.py
new file mode 100644
index 0000000..7c3ca7f
--- /dev/null
+++ b/tools/testrunner/local/statusfile.py
@@ -0,0 +1,140 @@
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+# These outcomes can occur in a TestCase's outcomes list:
+SKIP = "SKIP"
+FAIL = "FAIL"
+PASS = "PASS"
+OKAY = "OKAY"
+TIMEOUT = "TIMEOUT"
+CRASH = "CRASH"
+SLOW = "SLOW"
+FLAKY = "FLAKY"
+NO_VARIANTS = "NO_VARIANTS"
+# These are just for the status files and are mapped below in DEFS:
+FAIL_OK = "FAIL_OK"
+PASS_OR_FAIL = "PASS_OR_FAIL"
+
+ALWAYS = "ALWAYS"
+
+KEYWORDS = {}
+for key in [SKIP, FAIL, PASS, OKAY, TIMEOUT, CRASH, SLOW, FLAKY, FAIL_OK,
+ NO_VARIANTS, PASS_OR_FAIL, ALWAYS]:
+ KEYWORDS[key] = key
+
+DEFS = {FAIL_OK: [FAIL, OKAY],
+ PASS_OR_FAIL: [PASS, FAIL]}
+
+# Support arches, modes to be written as keywords instead of strings.
+VARIABLES = {ALWAYS: True}
+for var in ["debug", "release", "android_arm", "android_arm64", "android_ia32", "android_x87",
+ "arm", "arm64", "ia32", "mips", "mipsel", "mips64el", "x64", "x87", "nacl_ia32",
+ "nacl_x64", "macos", "windows", "linux"]:
+ VARIABLES[var] = var
+
+
+def DoSkip(outcomes):
+ return SKIP in outcomes
+
+
+def IsSlow(outcomes):
+ return SLOW in outcomes
+
+
+def OnlyStandardVariant(outcomes):
+ return NO_VARIANTS in outcomes
+
+
+def IsFlaky(outcomes):
+ return FLAKY in outcomes
+
+
+def IsPassOrFail(outcomes):
+ return ((PASS in outcomes) and (FAIL in outcomes) and
+ (not CRASH in outcomes) and (not OKAY in outcomes))
+
+
+def IsFailOk(outcomes):
+ return (FAIL in outcomes) and (OKAY in outcomes)
+
+
+def _AddOutcome(result, new):
+ global DEFS
+ if new in DEFS:
+ mapped = DEFS[new]
+ if type(mapped) == list:
+ for m in mapped:
+ _AddOutcome(result, m)
+ elif type(mapped) == str:
+ _AddOutcome(result, mapped)
+ else:
+ result.add(new)
+
+
+def _ParseOutcomeList(rule, outcomes, target_dict, variables):
+ result = set([])
+ if type(outcomes) == str:
+ outcomes = [outcomes]
+ for item in outcomes:
+ if type(item) == str:
+ _AddOutcome(result, item)
+ elif type(item) == list:
+ if not eval(item[0], variables): continue
+ for outcome in item[1:]:
+ assert type(outcome) == str
+ _AddOutcome(result, outcome)
+ else:
+ assert False
+ if len(result) == 0: return
+ if rule in target_dict:
+ target_dict[rule] |= result
+ else:
+ target_dict[rule] = result
+
+
+def ReadStatusFile(path, variables):
+ with open(path) as f:
+ global KEYWORDS
+ contents = eval(f.read(), KEYWORDS)
+
+ rules = {}
+ wildcards = {}
+ variables.update(VARIABLES)
+ for section in contents:
+ assert type(section) == list
+ assert len(section) == 2
+ if not eval(section[0], variables): continue
+ section = section[1]
+ assert type(section) == dict
+ for rule in section:
+ assert type(rule) == str
+ if rule[-1] == '*':
+ _ParseOutcomeList(rule, section[rule], wildcards, variables)
+ else:
+ _ParseOutcomeList(rule, section[rule], rules, variables)
+ return rules, wildcards
diff --git a/tools/testrunner/local/testsuite.py b/tools/testrunner/local/testsuite.py
new file mode 100644
index 0000000..47bc08f
--- /dev/null
+++ b/tools/testrunner/local/testsuite.py
@@ -0,0 +1,257 @@
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import imp
+import os
+
+from . import commands
+from . import statusfile
+from . import utils
+from ..objects import testcase
+
+class TestSuite(object):
+
+ @staticmethod
+ def LoadTestSuite(root):
+ name = root.split(os.path.sep)[-1]
+ f = None
+ try:
+ (f, pathname, description) = imp.find_module("testcfg", [root])
+ module = imp.load_module("testcfg", f, pathname, description)
+ return module.GetSuite(name, root)
+ except:
+ # Use default if no testcfg is present.
+ return GoogleTestSuite(name, root)
+ finally:
+ if f:
+ f.close()
+
+ def __init__(self, name, root):
+ self.name = name # string
+ self.root = root # string containing path
+ self.tests = None # list of TestCase objects
+ self.rules = None # dictionary mapping test path to list of outcomes
+ self.wildcards = None # dictionary mapping test paths to list of outcomes
+ self.total_duration = None # float, assigned on demand
+
+ def shell(self):
+ return "d8"
+
+ def suffix(self):
+ return ".js"
+
+ def status_file(self):
+ return "%s/%s.status" % (self.root, self.name)
+
+ # Used in the status file and for stdout printing.
+ def CommonTestName(self, testcase):
+ if utils.IsWindows():
+ return testcase.path.replace("\\", "/")
+ else:
+ return testcase.path
+
+ def ListTests(self, context):
+ raise NotImplementedError
+
+ def VariantFlags(self, testcase, default_flags):
+ if testcase.outcomes and statusfile.OnlyStandardVariant(testcase.outcomes):
+ return [[]]
+ return default_flags
+
+ def DownloadData(self):
+ pass
+
+ def ReadStatusFile(self, variables):
+ (self.rules, self.wildcards) = \
+ statusfile.ReadStatusFile(self.status_file(), variables)
+
+ def ReadTestCases(self, context):
+ self.tests = self.ListTests(context)
+
+ @staticmethod
+ def _FilterFlaky(flaky, mode):
+ return (mode == "run" and not flaky) or (mode == "skip" and flaky)
+
+ @staticmethod
+ def _FilterSlow(slow, mode):
+ return (mode == "run" and not slow) or (mode == "skip" and slow)
+
+ @staticmethod
+ def _FilterPassFail(pass_fail, mode):
+ return (mode == "run" and not pass_fail) or (mode == "skip" and pass_fail)
+
+ def FilterTestCasesByStatus(self, warn_unused_rules,
+ flaky_tests="dontcare",
+ slow_tests="dontcare",
+ pass_fail_tests="dontcare"):
+ filtered = []
+ used_rules = set()
+ for t in self.tests:
+ flaky = False
+ slow = False
+ pass_fail = False
+ testname = self.CommonTestName(t)
+ if testname in self.rules:
+ used_rules.add(testname)
+ # Even for skipped tests, as the TestCase object stays around and
+ # PrintReport() uses it.
+ t.outcomes = self.rules[testname]
+ if statusfile.DoSkip(t.outcomes):
+ continue # Don't add skipped tests to |filtered|.
+ flaky = statusfile.IsFlaky(t.outcomes)
+ slow = statusfile.IsSlow(t.outcomes)
+ pass_fail = statusfile.IsPassOrFail(t.outcomes)
+ skip = False
+ for rule in self.wildcards:
+ assert rule[-1] == '*'
+ if testname.startswith(rule[:-1]):
+ used_rules.add(rule)
+ t.outcomes = self.wildcards[rule]
+ if statusfile.DoSkip(t.outcomes):
+ skip = True
+ break # "for rule in self.wildcards"
+ flaky = flaky or statusfile.IsFlaky(t.outcomes)
+ slow = slow or statusfile.IsSlow(t.outcomes)
+ pass_fail = pass_fail or statusfile.IsPassOrFail(t.outcomes)
+ if (skip or self._FilterFlaky(flaky, flaky_tests)
+ or self._FilterSlow(slow, slow_tests)
+ or self._FilterPassFail(pass_fail, pass_fail_tests)):
+ continue # "for t in self.tests"
+ filtered.append(t)
+ self.tests = filtered
+
+ if not warn_unused_rules:
+ return
+
+ for rule in self.rules:
+ if rule not in used_rules:
+ print("Unused rule: %s -> %s" % (rule, self.rules[rule]))
+ for rule in self.wildcards:
+ if rule not in used_rules:
+ print("Unused rule: %s -> %s" % (rule, self.wildcards[rule]))
+
+ def FilterTestCasesByArgs(self, args):
+ filtered = []
+ filtered_args = []
+ for a in args:
+ argpath = a.split(os.path.sep)
+ if argpath[0] != self.name:
+ continue
+ if len(argpath) == 1 or (len(argpath) == 2 and argpath[1] == '*'):
+ return # Don't filter, run all tests in this suite.
+ path = os.path.sep.join(argpath[1:])
+ if path[-1] == '*':
+ path = path[:-1]
+ filtered_args.append(path)
+ for t in self.tests:
+ for a in filtered_args:
+ if t.path.startswith(a):
+ filtered.append(t)
+ break
+ self.tests = filtered
+
+ def GetFlagsForTestCase(self, testcase, context):
+ raise NotImplementedError
+
+ def GetSourceForTest(self, testcase):
+ return "(no source available)"
+
+ def IsFailureOutput(self, output, testpath):
+ return output.exit_code != 0
+
+ def IsNegativeTest(self, testcase):
+ return False
+
+ def HasFailed(self, testcase):
+ execution_failed = self.IsFailureOutput(testcase.output, testcase.path)
+ if self.IsNegativeTest(testcase):
+ return not execution_failed
+ else:
+ return execution_failed
+
+ def GetOutcome(self, testcase):
+ if testcase.output.HasCrashed():
+ return statusfile.CRASH
+ elif testcase.output.HasTimedOut():
+ return statusfile.TIMEOUT
+ elif self.HasFailed(testcase):
+ return statusfile.FAIL
+ else:
+ return statusfile.PASS
+
+ def HasUnexpectedOutput(self, testcase):
+ outcome = self.GetOutcome(testcase)
+ return not outcome in (testcase.outcomes or [statusfile.PASS])
+
+ def StripOutputForTransmit(self, testcase):
+ if not self.HasUnexpectedOutput(testcase):
+ testcase.output.stdout = ""
+ testcase.output.stderr = ""
+
+ def CalculateTotalDuration(self):
+ self.total_duration = 0.0
+ for t in self.tests:
+ self.total_duration += t.duration
+ return self.total_duration
+
+
+class GoogleTestSuite(TestSuite):
+ def __init__(self, name, root):
+ super(GoogleTestSuite, self).__init__(name, root)
+
+ def ListTests(self, context):
+ shell = os.path.abspath(os.path.join(context.shell_dir, self.shell()))
+ if utils.IsWindows():
+ shell += ".exe"
+ output = commands.Execute(context.command_prefix +
+ [shell, "--gtest_list_tests"] +
+ context.extra_flags)
+ if output.exit_code != 0:
+ print output.stdout
+ print output.stderr
+ return []
+ tests = []
+ test_case = ''
+ for line in output.stdout.splitlines():
+ test_desc = line.strip().split()[0]
+ if test_desc.endswith('.'):
+ test_case = test_desc
+ elif test_case and test_desc:
+ test = testcase.TestCase(self, test_case + test_desc, dependency=None)
+ tests.append(test)
+ tests.sort()
+ return tests
+
+ def GetFlagsForTestCase(self, testcase, context):
+ return (testcase.flags + ["--gtest_filter=" + testcase.path] +
+ ["--gtest_random_seed=%s" % context.random_seed] +
+ ["--gtest_print_time=0"] +
+ context.mode_flags)
+
+ def shell(self):
+ return self.name
diff --git a/tools/testrunner/local/utils.py b/tools/testrunner/local/utils.py
new file mode 100644
index 0000000..7bc21b1
--- /dev/null
+++ b/tools/testrunner/local/utils.py
@@ -0,0 +1,121 @@
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import os
+from os.path import exists
+from os.path import isdir
+from os.path import join
+import platform
+import re
+import urllib2
+
+
+def GetSuitePaths(test_root):
+ return [ f for f in os.listdir(test_root) if isdir(join(test_root, f)) ]
+
+
+# Reads a file into an array of strings
+def ReadLinesFrom(name):
+ lines = []
+ with open(name) as f:
+ for line in f:
+ if line.startswith('#'): continue
+ if '#' in line:
+ line = line[:line.find('#')]
+ line = line.strip()
+ if not line: continue
+ lines.append(line)
+ return lines
+
+
+def GuessOS():
+ system = platform.system()
+ if system == 'Linux':
+ return 'linux'
+ elif system == 'Darwin':
+ return 'macos'
+ elif system.find('CYGWIN') >= 0:
+ return 'cygwin'
+ elif system == 'Windows' or system == 'Microsoft':
+ # On Windows Vista platform.system() can return 'Microsoft' with some
+ # versions of Python, see http://bugs.python.org/issue1082
+ return 'windows'
+ elif system == 'FreeBSD':
+ return 'freebsd'
+ elif system == 'OpenBSD':
+ return 'openbsd'
+ elif system == 'SunOS':
+ return 'solaris'
+ elif system == 'NetBSD':
+ return 'netbsd'
+ else:
+ return None
+
+
+def UseSimulator(arch):
+ machine = platform.machine()
+ return (machine and
+ (arch == "mipsel" or arch == "arm" or arch == "arm64") and
+ not arch.startswith(machine))
+
+
+# This will default to building the 32 bit VM even on machines that are
+# capable of running the 64 bit VM.
+def DefaultArch():
+ machine = platform.machine()
+ machine = machine.lower() # Windows 7 capitalizes 'AMD64'.
+ if machine.startswith('arm'):
+ return 'arm'
+ elif (not machine) or (not re.match('(x|i[3-6])86$', machine) is None):
+ return 'ia32'
+ elif machine == 'i86pc':
+ return 'ia32'
+ elif machine == 'x86_64':
+ return 'ia32'
+ elif machine == 'amd64':
+ return 'ia32'
+ else:
+ return None
+
+
+def GuessWordsize():
+ if '64' in platform.machine():
+ return '64'
+ else:
+ return '32'
+
+
+def IsWindows():
+ return GuessOS() == 'windows'
+
+
+def URLRetrieve(source, destination):
+ """urllib is broken for SSL connections via a proxy therefore we
+ can't use urllib.urlretrieve()."""
+ with open(destination, 'w') as f:
+ f.write(urllib2.urlopen(source).read())
diff --git a/tools/testrunner/local/verbose.py b/tools/testrunner/local/verbose.py
new file mode 100644
index 0000000..00c330d
--- /dev/null
+++ b/tools/testrunner/local/verbose.py
@@ -0,0 +1,99 @@
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import sys
+import time
+
+from . import statusfile
+
+
+REPORT_TEMPLATE = (
+"""Total: %(total)i tests
+ * %(skipped)4d tests will be skipped
+ * %(timeout)4d tests are expected to timeout sometimes
+ * %(nocrash)4d tests are expected to be flaky but not crash
+ * %(pass)4d tests are expected to pass
+ * %(fail_ok)4d tests are expected to fail that we won't fix
+ * %(fail)4d tests are expected to fail that we should fix""")
+
+
+def PrintReport(tests):
+ total = len(tests)
+ skipped = timeout = nocrash = passes = fail_ok = fail = 0
+ for t in tests:
+ if "outcomes" not in dir(t) or not t.outcomes:
+ passes += 1
+ continue
+ o = t.outcomes
+ if statusfile.DoSkip(o):
+ skipped += 1
+ continue
+ if statusfile.TIMEOUT in o: timeout += 1
+ if statusfile.IsPassOrFail(o): nocrash += 1
+ if list(o) == [statusfile.PASS]: passes += 1
+ if statusfile.IsFailOk(o): fail_ok += 1
+ if list(o) == [statusfile.FAIL]: fail += 1
+ print REPORT_TEMPLATE % {
+ "total": total,
+ "skipped": skipped,
+ "timeout": timeout,
+ "nocrash": nocrash,
+ "pass": passes,
+ "fail_ok": fail_ok,
+ "fail": fail
+ }
+
+
+def PrintTestSource(tests):
+ for test in tests:
+ suite = test.suite
+ source = suite.GetSourceForTest(test).strip()
+ if len(source) > 0:
+ print "--- begin source: %s/%s ---" % (suite.name, test.path)
+ print source
+ print "--- end source: %s/%s ---" % (suite.name, test.path)
+
+
+def FormatTime(d):
+ millis = round(d * 1000) % 1000
+ return time.strftime("%M:%S.", time.gmtime(d)) + ("%03i" % millis)
+
+
+def PrintTestDurations(suites, overall_time):
+ # Write the times to stderr to make it easy to separate from the
+ # test output.
+ print
+ sys.stderr.write("--- Total time: %s ---\n" % FormatTime(overall_time))
+ timed_tests = [ t for s in suites for t in s.tests
+ if t.duration is not None ]
+ timed_tests.sort(lambda a, b: cmp(b.duration, a.duration))
+ index = 1
+ for entry in timed_tests[:20]:
+ t = FormatTime(entry.duration)
+ sys.stderr.write("%4i (%s) %s\n" % (index, t, entry.GetLabel()))
+ index += 1