MultiTestRunner: Simplify, cleanup, and rename!
- MultiTestRunner will eventually be renamed to 'lit', for LLVM integrated
tester/testing. This has the pros of being pronouncable and short.
- "Project" level configuration lives in 'lit.cfg', which is also what lit uses
to find the root testing directory in some cases. This can be overridden for
use in project files which want to precisely specify where things are.
- TestRunner.py is not longer able to be invoked directly.
- Moved some code to Util.py.
- Introduced a configuration object.
- Cleaned up --help, removed a few not-very-useful options.
- Tried not to break anything that works. :)
git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@77665 91177308-0d34-0410-b5e6-96231b3b80d8
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 794e80b..58e7368 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -33,6 +33,7 @@
add_custom_target(clang-test-${testdir}
${PYTHON_EXECUTABLE}
${LLVM_SOURCE_DIR}/tools/clang/utils/test/MultiTestRunner.py
+ "--root=${LLVM_SOURCE_DIR}/tools/clang/test"
"--path=${LLVM_TOOLS_PATH}/${CMAKE_CFG_INTDIR}"
"--path=${LLVM_SOURCE_DIR}/test/Scripts"
-s ${CLANG_TEST_EXTRA_ARGS}
@@ -48,6 +49,7 @@
add_custom_target(clang-test
${PYTHON_EXECUTABLE}
${LLVM_SOURCE_DIR}/tools/clang/utils/test/MultiTestRunner.py
+ "--root=${LLVM_SOURCE_DIR}/tools/clang/test"
"--path=${LLVM_TOOLS_PATH}/${CMAKE_CFG_INTDIR}"
"--path=${LLVM_SOURCE_DIR}/test/Scripts"
-s ${CLANG_TEST_EXTRA_ARGS}
diff --git a/test/Makefile b/test/Makefile
index 97c9d27..13cbfe6 100644
--- a/test/Makefile
+++ b/test/Makefile
@@ -21,6 +21,7 @@
all::
@ echo '--- Running clang tests for $(TARGET_TRIPLE) ---'
@ $(PROJ_SRC_DIR)/../utils/test/MultiTestRunner.py \
+ --root $(PROJ_SRC_DIR) \
--path $(ToolDir) \
--path $(LLVM_SRC_ROOT)/test/Scripts \
$(TESTARGS) $(TESTDIRS) $(VGARG)
diff --git a/test/lit.cfg b/test/lit.cfg
new file mode 100644
index 0000000..89ca044
--- /dev/null
+++ b/test/lit.cfg
@@ -0,0 +1,12 @@
+# -*- Python -*-
+
+# Configuration file for the 'lit' test runner.
+
+# suffixes: A list of file extensions to treat as test files.
+suffixes = ['.c', '.cpp']
+
+# environment: The base environment to use when running test commands.
+#
+# The 'PATH' and 'SYSTEMROOT' variables will be set automatically from the lit
+# command line variables.
+environment = {}
diff --git a/utils/test/MultiTestRunner.py b/utils/test/MultiTestRunner.py
index aaceb7f..9980e65 100755
--- a/utils/test/MultiTestRunner.py
+++ b/utils/test/MultiTestRunner.py
@@ -15,33 +15,40 @@
in a separate category).
"""
-# TOD
import os, sys, re, random, time
import threading
-import ProgressBar
-import TestRunner
-from TestRunner import TestStatus
from Queue import Queue
-kTestFileExtensions = set(['.mi','.i','.c','.cpp','.m','.mm','.ll'])
+import ProgressBar
+import TestRunner
+import Util
-def getTests(inputs):
+from TestingConfig import TestingConfig
+from TestRunner import TestStatus
+
+kConfigName = 'lit.cfg'
+
+def getTests(cfg, inputs):
for path in inputs:
if not os.path.exists(path):
- print >>sys.stderr,"WARNING: Invalid test \"%s\""%(path,)
+ Util.warning('Invalid test %r' % path)
continue
if not os.path.isdir(path):
yield path
+ foundOne = False
for dirpath,dirnames,filenames in os.walk(path):
# FIXME: This doesn't belong here
if 'Output' in dirnames:
dirnames.remove('Output')
for f in filenames:
base,ext = os.path.splitext(f)
- if ext in kTestFileExtensions:
+ if ext in cfg.suffixes:
yield os.path.join(dirpath,f)
+ foundOne = True
+ if not foundOne:
+ Util.warning('No tests in input directory %r' % path)
class TestingProgressDisplay:
def __init__(self, opts, numTests, progressBar=None):
@@ -109,7 +116,8 @@
return self.code in (TestStatus.Fail,TestStatus.XPass)
class TestProvider:
- def __init__(self, opts, tests, display):
+ def __init__(self, config, opts, tests, display):
+ self.config = config
self.opts = opts
self.tests = tests
self.index = 0
@@ -156,14 +164,12 @@
elapsed = None
try:
opts = self.provider.opts
- if opts.debugDoNotTest:
- code = None
- else:
- startTime = time.time()
- code, output = TestRunner.runOneTest(path, base,
- opts.clang, opts.clangcc,
- opts.useValgrind)
- elapsed = time.time() - startTime
+ startTime = time.time()
+ code, output = TestRunner.runOneTest(self.provider.config,
+ path, base,
+ opts.clang, opts.clangcc,
+ opts.useValgrind)
+ elapsed = time.time() - startTime
except KeyboardInterrupt:
# This is a sad hack. Unfortunately subprocess goes
# bonkers with ctrl-c and we start forking merrily.
@@ -172,102 +178,127 @@
self.provider.setResult(index, TestResult(path, code, output, elapsed))
-def detectCPUs():
- """
- Detects the number of CPUs on a system. Cribbed from pp.
- """
- # Linux, Unix and MacOS:
- if hasattr(os, "sysconf"):
- if os.sysconf_names.has_key("SC_NPROCESSORS_ONLN"):
- # Linux & Unix:
- ncpus = os.sysconf("SC_NPROCESSORS_ONLN")
- if isinstance(ncpus, int) and ncpus > 0:
- return ncpus
- else: # OSX:
- return int(os.popen2("sysctl -n hw.ncpu")[1].read())
- # Windows:
- if os.environ.has_key("NUMBER_OF_PROCESSORS"):
- ncpus = int(os.environ["NUMBER_OF_PROCESSORS"]);
- if ncpus > 0:
- return ncpus
- return 1 # Default
+def findConfigPath(root):
+ prev = None
+ while root != prev:
+ cfg = os.path.join(root, kConfigName)
+ if os.path.exists(cfg):
+ return cfg
+
+ prev,root = root,os.path.dirname(root)
+
+ raise ValueError,"Unable to find config file %r" % kConfigName
def main():
global options
- from optparse import OptionParser
- parser = OptionParser("usage: %prog [options] {inputs}")
- parser.add_option("-j", "--threads", dest="numThreads",
- help="Number of testing threads",
- type=int, action="store",
- default=detectCPUs())
- parser.add_option("", "--clang", dest="clang",
- help="Program to use as \"clang\"",
+ from optparse import OptionParser, OptionGroup
+ parser = OptionParser("usage: %prog [options] {file-or-path}")
+
+ parser.add_option("", "--root", dest="root",
+ help="Path to root test directory",
action="store", default=None)
- parser.add_option("", "--clang-cc", dest="clangcc",
- help="Program to use as \"clang-cc\"",
- action="store", default=None)
- parser.add_option("", "--vg", dest="useValgrind",
- help="Run tests under valgrind",
- action="store_true", default=False)
- parser.add_option("-v", "--verbose", dest="showOutput",
- help="Show all test output",
- action="store_true", default=False)
- parser.add_option("-q", "--quiet", dest="quiet",
- help="Suppress no error output",
- action="store_true", default=False)
- parser.add_option("-s", "--succinct", dest="succinct",
- help="Reduce amount of output",
- action="store_true", default=False)
- parser.add_option("", "--max-tests", dest="maxTests",
- help="Maximum number of tests to run",
- action="store", type=int, default=None)
- parser.add_option("", "--max-time", dest="maxTime",
- help="Maximum time to spend testing (in seconds)",
- action="store", type=float, default=None)
- parser.add_option("", "--shuffle", dest="shuffle",
- help="Run tests in random order",
- action="store_true", default=False)
- parser.add_option("", "--seed", dest="seed",
- help="Seed for random number generator (default: random)",
- action="store", default=None)
- parser.add_option("", "--no-progress-bar", dest="useProgressBar",
- help="Do not use curses based progress bar",
- action="store_false", default=True)
- parser.add_option("", "--debug-do-not-test", dest="debugDoNotTest",
- help="DEBUG: Skip running actual test script",
- action="store_true", default=False)
- parser.add_option("", "--time-tests", dest="timeTests",
- help="Track elapsed wall time for each test",
- action="store_true", default=False)
- parser.add_option("", "--path", dest="path",
- help="Additional paths to add to testing environment",
- action="append", type=str, default=[])
+ parser.add_option("", "--config", dest="config",
+ help="Testing configuration file [default='%default']",
+ action="store", default=kConfigName)
+
+ group = OptionGroup(parser, "Output Format")
+ # FIXME: I find these names very confusing, although I like the
+ # functionality.
+ group.add_option("-q", "--quiet", dest="quiet",
+ help="Suppress no error output",
+ action="store_true", default=False)
+ group.add_option("-s", "--succinct", dest="succinct",
+ help="Reduce amount of output",
+ action="store_true", default=False)
+ group.add_option("-v", "--verbose", dest="showOutput",
+ help="Show all test output",
+ action="store_true", default=False)
+ group.add_option("", "--no-progress-bar", dest="useProgressBar",
+ help="Do not use curses based progress bar",
+ action="store_false", default=True)
+ parser.add_option_group(group)
+
+ group = OptionGroup(parser, "Test Execution")
+ group.add_option("-j", "--threads", dest="numThreads",
+ help="Number of testing threads",
+ type=int, action="store",
+ default=None)
+ group.add_option("", "--clang", dest="clang",
+ help="Program to use as \"clang\"",
+ action="store", default=None)
+ group.add_option("", "--clang-cc", dest="clangcc",
+ help="Program to use as \"clang-cc\"",
+ action="store", default=None)
+ group.add_option("", "--path", dest="path",
+ help="Additional paths to add to testing environment",
+ action="append", type=str, default=[])
+ group.add_option("", "--vg", dest="useValgrind",
+ help="Run tests under valgrind",
+ action="store_true", default=False)
+ group.add_option("", "--time-tests", dest="timeTests",
+ help="Track elapsed wall time for each test",
+ action="store_true", default=False)
+ parser.add_option_group(group)
+
+ group = OptionGroup(parser, "Test Selection")
+ group.add_option("", "--max-tests", dest="maxTests",
+ help="Maximum number of tests to run",
+ action="store", type=int, default=None)
+ group.add_option("", "--max-time", dest="maxTime",
+ help="Maximum time to spend testing (in seconds)",
+ action="store", type=float, default=None)
+ group.add_option("", "--shuffle", dest="shuffle",
+ help="Run tests in random order",
+ action="store_true", default=False)
+ parser.add_option_group(group)
(opts, args) = parser.parse_args()
-
+
if not args:
parser.error('No inputs specified')
- # FIXME: Move into configuration object.
- TestRunner.kChildEnv["PATH"] = os.pathsep.join(opts.path +
- [TestRunner.kChildEnv['PATH']])
+ if opts.numThreads is None:
+ opts.numThreads = Util.detectCPUs()
+
+ inputs = args
+
+ # Resolve root if not given, either infer it from the config file if given,
+ # otherwise from the inputs.
+ if not opts.root:
+ if opts.config:
+ opts.root = os.path.dirname(opts.config)
+ else:
+ opts.root = os.path.commonprefix(inputs)
+
+ # Find the config file, if not specified.
+ if not opts.config:
+ try:
+ opts.config = findConfigPath(opts.root)
+ except ValueError,e:
+ parser.error(e.args[0])
+
+ cfg = TestingConfig.frompath(opts.config)
+
+ # Update the configuration based on the command line arguments.
+ for name in ('PATH','SYSTEMROOT'):
+ if name in cfg.environment:
+ parser.error("'%s' should not be set in configuration!" % name)
+
+ cfg.root = opts.root
+ cfg.environment['PATH'] = os.pathsep.join(opts.path +
+ [os.environ.get('PATH','')])
+ cfg.environment['SYSTEMROOT'] = os.environ.get('SYSTEMROOT','')
if opts.clang is None:
- opts.clang = TestRunner.inferClang()
+ opts.clang = TestRunner.inferClang(cfg)
if opts.clangcc is None:
- opts.clangcc = TestRunner.inferClangCC(opts.clang)
+ opts.clangcc = TestRunner.inferClangCC(cfg, opts.clang)
# FIXME: It could be worth loading these in parallel with testing.
- allTests = list(getTests(args))
+ allTests = list(getTests(cfg, args))
allTests.sort()
tests = allTests
- if opts.seed is not None:
- try:
- seed = int(opts.seed)
- except:
- parser.error('--seed argument should be an integer')
- random.seed(seed)
if opts.shuffle:
random.shuffle(tests)
if opts.maxTests is not None:
@@ -292,7 +323,7 @@
print header
display = TestingProgressDisplay(opts, len(tests), progressBar)
- provider = TestProvider(opts, tests, display)
+ provider = TestProvider(cfg, opts, tests, display)
testers = [Tester(provider) for i in range(opts.numThreads)]
startTime = time.time()
@@ -309,7 +340,7 @@
if not opts.quiet:
print 'Testing Time: %.2fs'%(time.time() - startTime)
- # List test results organized organized by kind.
+ # List test results organized by kind.
byCode = {}
for t in provider.results:
if t:
diff --git a/utils/test/TestRunner.py b/utils/test/TestRunner.py
index d3ffef2..fb3d172 100755
--- a/utils/test/TestRunner.py
+++ b/utils/test/TestRunner.py
@@ -23,10 +23,7 @@
import subprocess
import sys
-# Increase determinism by explicitly choosing the environment.
-kChildEnv = {}
-for var in ('PATH', 'SYSTEMROOT'):
- kChildEnv[var] = os.environ.get(var, '')
+import Util
kSystemName = platform.system()
@@ -42,22 +39,7 @@
def getName(code):
return TestStatus.kNames[code]
-def mkdir_p(path):
- if not path:
- pass
- elif os.path.exists(path):
- pass
- else:
- parent = os.path.dirname(path)
- if parent != path:
- mkdir_p(parent)
- try:
- os.mkdir(path)
- except OSError,e:
- if e.errno != errno.EEXIST:
- raise
-
-def executeScript(script, commands, cwd, useValgrind):
+def executeScript(cfg, script, commands, cwd, useValgrind):
# Write script file
f = open(script,'w')
if kSystemName == 'Windows':
@@ -82,7 +64,7 @@
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
- env=kChildEnv)
+ env=cfg.environment)
out,err = p.communicate()
exitCode = p.wait()
@@ -93,14 +75,14 @@
return out, err, exitCode
import StringIO
-def runOneTest(testPath, tmpBase, clang, clangcc, useValgrind):
+def runOneTest(cfg, testPath, tmpBase, clang, clangcc, useValgrind):
# Make paths absolute.
tmpBase = os.path.abspath(tmpBase)
testPath = os.path.abspath(testPath)
# Create the output directory if it does not already exist.
- mkdir_p(os.path.dirname(tmpBase))
+ Util.mkdir_p(os.path.dirname(tmpBase))
script = tmpBase + '.script'
if kSystemName == 'Windows':
script += '.bat'
@@ -154,7 +136,7 @@
# Strip off '&&'
scriptLines[i] = ln[:-2]
- out, err, exitCode = executeScript(script, scriptLines,
+ out, err, exitCode = executeScript(cfg, script, scriptLines,
os.path.dirname(testPath),
useValgrind)
if xfailLines:
@@ -188,31 +170,7 @@
out,_ = p.communicate()
return out
-def which(command):
- # FIXME: Take configuration object.
-
- # Check for absolute match first.
- if os.path.exists(command):
- return command
-
- # Would be nice if Python had a lib function for this.
- paths = kChildEnv['PATH']
- if not paths:
- paths = os.defpath
-
- # Get suffixes to search.
- pathext = os.environ.get('PATHEXT', '').split(os.pathsep)
-
- # Search the paths...
- for path in paths.split(os.pathsep):
- for ext in pathext:
- p = os.path.join(path, command + ext)
- if os.path.exists(p):
- return p
-
- return None
-
-def inferClang():
+def inferClang(cfg):
# Determine which clang to use.
clang = os.getenv('CLANG')
@@ -222,7 +180,7 @@
return clang
# Otherwise look in the path.
- clang = which('clang')
+ clang = Util.which('clang', cfg.environment['PATH'])
if not clang:
print >>sys.stderr, "error: couldn't find 'clang' program, try setting CLANG in your environment"
@@ -230,7 +188,7 @@
return clang
-def inferClangCC(clang):
+def inferClangCC(cfg, clang):
clangcc = os.getenv('CLANGCC')
# If the user set clang in the environment, definitely use that and don't
@@ -244,7 +202,7 @@
clangccName = clang[:-4] + '-cc.exe'
else:
clangccName = clang + '-cc'
- clangcc = which(clangccName)
+ clangcc = Util.which(clangccName, cfg.environment['PATH'])
if not clangcc:
# Otherwise ask clang.
res = capture([clang, '-print-prog-name=clang-cc'])
@@ -267,43 +225,3 @@
return os.path.join(dir,
os.path.basename(os.path.dirname(testpath)),
os.path.basename(testpath))
-
-def main():
- global options
- from optparse import OptionParser
- parser = OptionParser("usage: %prog [options] {tests}")
- parser.add_option("", "--clang", dest="clang",
- help="Program to use as \"clang\"",
- action="store", default=None)
- parser.add_option("", "--clang-cc", dest="clangcc",
- help="Program to use as \"clang-cc\"",
- action="store", default=None)
- parser.add_option("", "--vg", dest="useValgrind",
- help="Run tests under valgrind",
- action="store_true", default=False)
- (opts, args) = parser.parse_args()
-
- if not args:
- parser.error('No tests specified')
-
- if opts.clang is None:
- opts.clang = inferClang()
- if opts.clangcc is None:
- opts.clangcc = inferClangCC(opts.clang)
-
- for path in args:
- base = getTestOutputBase('Output', path) + '.out'
-
- status,output = runOneTest(path, base, opts.clang, opts.clangcc,
- opts.useValgrind)
- print '%s: %s' % (TestStatus.getName(status).upper(), path)
- if status == TestStatus.Fail or status == TestStatus.XPass:
- print "%s TEST '%s' FAILED %s" % ('*'*20, path, '*'*20)
- sys.stdout.write(output)
- print "*" * 20
- sys.exit(1)
-
- sys.exit(0)
-
-if __name__=='__main__':
- main()
diff --git a/utils/test/TestingConfig.py b/utils/test/TestingConfig.py
new file mode 100644
index 0000000..6b33589
--- /dev/null
+++ b/utils/test/TestingConfig.py
@@ -0,0 +1,21 @@
+class TestingConfig:
+ """"
+ TestingConfig - Information on a how to run a group of tests.
+ """
+
+ @staticmethod
+ def frompath(path):
+ data = {}
+ f = open(path)
+ exec f in {},data
+
+ return TestingConfig(suffixes = data.get('suffixes', []),
+ environment = data.get('environment', {}))
+
+ def __init__(self, suffixes, environment):
+ self.root = None
+ self.suffixes = set(suffixes)
+ self.environment = dict(environment)
+
+
+
diff --git a/utils/test/Util.py b/utils/test/Util.py
new file mode 100644
index 0000000..bacdab8
--- /dev/null
+++ b/utils/test/Util.py
@@ -0,0 +1,70 @@
+import os, sys
+
+def warning(msg):
+ print >>sys.stderr, '%s: warning: %s' % (sys.argv[0], msg)
+
+def detectCPUs():
+ """
+ Detects the number of CPUs on a system. Cribbed from pp.
+ """
+ # Linux, Unix and MacOS:
+ if hasattr(os, "sysconf"):
+ if os.sysconf_names.has_key("SC_NPROCESSORS_ONLN"):
+ # Linux & Unix:
+ ncpus = os.sysconf("SC_NPROCESSORS_ONLN")
+ if isinstance(ncpus, int) and ncpus > 0:
+ return ncpus
+ else: # OSX:
+ return int(os.popen2("sysctl -n hw.ncpu")[1].read())
+ # Windows:
+ if os.environ.has_key("NUMBER_OF_PROCESSORS"):
+ ncpus = int(os.environ["NUMBER_OF_PROCESSORS"]);
+ if ncpus > 0:
+ return ncpus
+ return 1 # Default
+
+def mkdir_p(path):
+ """mkdir_p(path) - Make the "path" directory, if it does not exist; this
+ will also make directories for any missing parent directories."""
+
+ if not path or os.path.exists(path):
+ return
+
+ parent = os.path.dirname(path)
+ if parent != path:
+ mkdir_p(parent)
+
+ try:
+ os.mkdir(path)
+ except OSError,e:
+ # Ignore EEXIST, which may occur during a race condition.
+ if e.errno != errno.EEXIST:
+ raise
+
+def which(command, paths = None):
+ """which(command, [paths]) - Look up the given command in the paths string (or
+ the PATH environment variable, if unspecified)."""
+
+ if paths is None:
+ paths = os.environ.get('PATH','')
+
+ # Check for absolute match first.
+ if os.path.exists(command):
+ return command
+
+ # Would be nice if Python had a lib function for this.
+ if not paths:
+ paths = os.defpath
+
+ # Get suffixes to search.
+ pathext = os.environ.get('PATHEXT', '').split(os.pathsep)
+
+ # Search the paths...
+ for path in paths.split(os.pathsep):
+ for ext in pathext:
+ p = os.path.join(path, command + ext)
+ if os.path.exists(p):
+ return p
+
+ return None
+