MultiTestRunner: Simplify, cleanup, and rename!
- MultiTestRunner will eventually be renamed to 'lit', for LLVM integrated
tester/testing. This has the pros of being pronouncable and short.
- "Project" level configuration lives in 'lit.cfg', which is also what lit uses
to find the root testing directory in some cases. This can be overridden for
use in project files which want to precisely specify where things are.
- TestRunner.py is not longer able to be invoked directly.
- Moved some code to Util.py.
- Introduced a configuration object.
- Cleaned up --help, removed a few not-very-useful options.
- Tried not to break anything that works. :)
git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@77665 91177308-0d34-0410-b5e6-96231b3b80d8
diff --git a/utils/test/MultiTestRunner.py b/utils/test/MultiTestRunner.py
index aaceb7f..9980e65 100755
--- a/utils/test/MultiTestRunner.py
+++ b/utils/test/MultiTestRunner.py
@@ -15,33 +15,40 @@
in a separate category).
"""
-# TOD
import os, sys, re, random, time
import threading
-import ProgressBar
-import TestRunner
-from TestRunner import TestStatus
from Queue import Queue
-kTestFileExtensions = set(['.mi','.i','.c','.cpp','.m','.mm','.ll'])
+import ProgressBar
+import TestRunner
+import Util
-def getTests(inputs):
+from TestingConfig import TestingConfig
+from TestRunner import TestStatus
+
+kConfigName = 'lit.cfg'
+
+def getTests(cfg, inputs):
for path in inputs:
if not os.path.exists(path):
- print >>sys.stderr,"WARNING: Invalid test \"%s\""%(path,)
+ Util.warning('Invalid test %r' % path)
continue
if not os.path.isdir(path):
yield path
+ foundOne = False
for dirpath,dirnames,filenames in os.walk(path):
# FIXME: This doesn't belong here
if 'Output' in dirnames:
dirnames.remove('Output')
for f in filenames:
base,ext = os.path.splitext(f)
- if ext in kTestFileExtensions:
+ if ext in cfg.suffixes:
yield os.path.join(dirpath,f)
+ foundOne = True
+ if not foundOne:
+ Util.warning('No tests in input directory %r' % path)
class TestingProgressDisplay:
def __init__(self, opts, numTests, progressBar=None):
@@ -109,7 +116,8 @@
return self.code in (TestStatus.Fail,TestStatus.XPass)
class TestProvider:
- def __init__(self, opts, tests, display):
+ def __init__(self, config, opts, tests, display):
+ self.config = config
self.opts = opts
self.tests = tests
self.index = 0
@@ -156,14 +164,12 @@
elapsed = None
try:
opts = self.provider.opts
- if opts.debugDoNotTest:
- code = None
- else:
- startTime = time.time()
- code, output = TestRunner.runOneTest(path, base,
- opts.clang, opts.clangcc,
- opts.useValgrind)
- elapsed = time.time() - startTime
+ startTime = time.time()
+ code, output = TestRunner.runOneTest(self.provider.config,
+ path, base,
+ opts.clang, opts.clangcc,
+ opts.useValgrind)
+ elapsed = time.time() - startTime
except KeyboardInterrupt:
# This is a sad hack. Unfortunately subprocess goes
# bonkers with ctrl-c and we start forking merrily.
@@ -172,102 +178,127 @@
self.provider.setResult(index, TestResult(path, code, output, elapsed))
-def detectCPUs():
- """
- Detects the number of CPUs on a system. Cribbed from pp.
- """
- # Linux, Unix and MacOS:
- if hasattr(os, "sysconf"):
- if os.sysconf_names.has_key("SC_NPROCESSORS_ONLN"):
- # Linux & Unix:
- ncpus = os.sysconf("SC_NPROCESSORS_ONLN")
- if isinstance(ncpus, int) and ncpus > 0:
- return ncpus
- else: # OSX:
- return int(os.popen2("sysctl -n hw.ncpu")[1].read())
- # Windows:
- if os.environ.has_key("NUMBER_OF_PROCESSORS"):
- ncpus = int(os.environ["NUMBER_OF_PROCESSORS"]);
- if ncpus > 0:
- return ncpus
- return 1 # Default
+def findConfigPath(root):
+ prev = None
+ while root != prev:
+ cfg = os.path.join(root, kConfigName)
+ if os.path.exists(cfg):
+ return cfg
+
+ prev,root = root,os.path.dirname(root)
+
+ raise ValueError,"Unable to find config file %r" % kConfigName
def main():
global options
- from optparse import OptionParser
- parser = OptionParser("usage: %prog [options] {inputs}")
- parser.add_option("-j", "--threads", dest="numThreads",
- help="Number of testing threads",
- type=int, action="store",
- default=detectCPUs())
- parser.add_option("", "--clang", dest="clang",
- help="Program to use as \"clang\"",
+ from optparse import OptionParser, OptionGroup
+ parser = OptionParser("usage: %prog [options] {file-or-path}")
+
+ parser.add_option("", "--root", dest="root",
+ help="Path to root test directory",
action="store", default=None)
- parser.add_option("", "--clang-cc", dest="clangcc",
- help="Program to use as \"clang-cc\"",
- action="store", default=None)
- parser.add_option("", "--vg", dest="useValgrind",
- help="Run tests under valgrind",
- action="store_true", default=False)
- parser.add_option("-v", "--verbose", dest="showOutput",
- help="Show all test output",
- action="store_true", default=False)
- parser.add_option("-q", "--quiet", dest="quiet",
- help="Suppress no error output",
- action="store_true", default=False)
- parser.add_option("-s", "--succinct", dest="succinct",
- help="Reduce amount of output",
- action="store_true", default=False)
- parser.add_option("", "--max-tests", dest="maxTests",
- help="Maximum number of tests to run",
- action="store", type=int, default=None)
- parser.add_option("", "--max-time", dest="maxTime",
- help="Maximum time to spend testing (in seconds)",
- action="store", type=float, default=None)
- parser.add_option("", "--shuffle", dest="shuffle",
- help="Run tests in random order",
- action="store_true", default=False)
- parser.add_option("", "--seed", dest="seed",
- help="Seed for random number generator (default: random)",
- action="store", default=None)
- parser.add_option("", "--no-progress-bar", dest="useProgressBar",
- help="Do not use curses based progress bar",
- action="store_false", default=True)
- parser.add_option("", "--debug-do-not-test", dest="debugDoNotTest",
- help="DEBUG: Skip running actual test script",
- action="store_true", default=False)
- parser.add_option("", "--time-tests", dest="timeTests",
- help="Track elapsed wall time for each test",
- action="store_true", default=False)
- parser.add_option("", "--path", dest="path",
- help="Additional paths to add to testing environment",
- action="append", type=str, default=[])
+ parser.add_option("", "--config", dest="config",
+ help="Testing configuration file [default='%default']",
+ action="store", default=kConfigName)
+
+ group = OptionGroup(parser, "Output Format")
+ # FIXME: I find these names very confusing, although I like the
+ # functionality.
+ group.add_option("-q", "--quiet", dest="quiet",
+ help="Suppress no error output",
+ action="store_true", default=False)
+ group.add_option("-s", "--succinct", dest="succinct",
+ help="Reduce amount of output",
+ action="store_true", default=False)
+ group.add_option("-v", "--verbose", dest="showOutput",
+ help="Show all test output",
+ action="store_true", default=False)
+ group.add_option("", "--no-progress-bar", dest="useProgressBar",
+ help="Do not use curses based progress bar",
+ action="store_false", default=True)
+ parser.add_option_group(group)
+
+ group = OptionGroup(parser, "Test Execution")
+ group.add_option("-j", "--threads", dest="numThreads",
+ help="Number of testing threads",
+ type=int, action="store",
+ default=None)
+ group.add_option("", "--clang", dest="clang",
+ help="Program to use as \"clang\"",
+ action="store", default=None)
+ group.add_option("", "--clang-cc", dest="clangcc",
+ help="Program to use as \"clang-cc\"",
+ action="store", default=None)
+ group.add_option("", "--path", dest="path",
+ help="Additional paths to add to testing environment",
+ action="append", type=str, default=[])
+ group.add_option("", "--vg", dest="useValgrind",
+ help="Run tests under valgrind",
+ action="store_true", default=False)
+ group.add_option("", "--time-tests", dest="timeTests",
+ help="Track elapsed wall time for each test",
+ action="store_true", default=False)
+ parser.add_option_group(group)
+
+ group = OptionGroup(parser, "Test Selection")
+ group.add_option("", "--max-tests", dest="maxTests",
+ help="Maximum number of tests to run",
+ action="store", type=int, default=None)
+ group.add_option("", "--max-time", dest="maxTime",
+ help="Maximum time to spend testing (in seconds)",
+ action="store", type=float, default=None)
+ group.add_option("", "--shuffle", dest="shuffle",
+ help="Run tests in random order",
+ action="store_true", default=False)
+ parser.add_option_group(group)
(opts, args) = parser.parse_args()
-
+
if not args:
parser.error('No inputs specified')
- # FIXME: Move into configuration object.
- TestRunner.kChildEnv["PATH"] = os.pathsep.join(opts.path +
- [TestRunner.kChildEnv['PATH']])
+ if opts.numThreads is None:
+ opts.numThreads = Util.detectCPUs()
+
+ inputs = args
+
+ # Resolve root if not given, either infer it from the config file if given,
+ # otherwise from the inputs.
+ if not opts.root:
+ if opts.config:
+ opts.root = os.path.dirname(opts.config)
+ else:
+ opts.root = os.path.commonprefix(inputs)
+
+ # Find the config file, if not specified.
+ if not opts.config:
+ try:
+ opts.config = findConfigPath(opts.root)
+ except ValueError,e:
+ parser.error(e.args[0])
+
+ cfg = TestingConfig.frompath(opts.config)
+
+ # Update the configuration based on the command line arguments.
+ for name in ('PATH','SYSTEMROOT'):
+ if name in cfg.environment:
+ parser.error("'%s' should not be set in configuration!" % name)
+
+ cfg.root = opts.root
+ cfg.environment['PATH'] = os.pathsep.join(opts.path +
+ [os.environ.get('PATH','')])
+ cfg.environment['SYSTEMROOT'] = os.environ.get('SYSTEMROOT','')
if opts.clang is None:
- opts.clang = TestRunner.inferClang()
+ opts.clang = TestRunner.inferClang(cfg)
if opts.clangcc is None:
- opts.clangcc = TestRunner.inferClangCC(opts.clang)
+ opts.clangcc = TestRunner.inferClangCC(cfg, opts.clang)
# FIXME: It could be worth loading these in parallel with testing.
- allTests = list(getTests(args))
+ allTests = list(getTests(cfg, args))
allTests.sort()
tests = allTests
- if opts.seed is not None:
- try:
- seed = int(opts.seed)
- except:
- parser.error('--seed argument should be an integer')
- random.seed(seed)
if opts.shuffle:
random.shuffle(tests)
if opts.maxTests is not None:
@@ -292,7 +323,7 @@
print header
display = TestingProgressDisplay(opts, len(tests), progressBar)
- provider = TestProvider(opts, tests, display)
+ provider = TestProvider(cfg, opts, tests, display)
testers = [Tester(provider) for i in range(opts.numThreads)]
startTime = time.time()
@@ -309,7 +340,7 @@
if not opts.quiet:
print 'Testing Time: %.2fs'%(time.time() - startTime)
- # List test results organized organized by kind.
+ # List test results organized by kind.
byCode = {}
for t in provider.results:
if t: