blob: 728d406e62e201f8232ecc63175dbf7d3deec84c [file] [log] [blame]
Eli Friedman77a1fe92009-07-10 20:15:12 +00001#!/usr/bin/env python
Daniel Dunbar6fc0bdf2009-03-06 22:20:40 +00002#
3# TestRunner.py - This script is used to run arbitrary unit tests. Unit
4# tests must contain the command used to run them in the input file, starting
5# immediately after a "RUN:" string.
6#
7# This runner recognizes and replaces the following strings in the command:
8#
9# %s - Replaced with the input name of the program, or the program to
10# execute, as appropriate.
Daniel Dunbar4d8076a2009-04-10 19:49:21 +000011# %S - Replaced with the directory where the input resides.
Daniel Dunbar6fc0bdf2009-03-06 22:20:40 +000012# %llvmgcc - llvm-gcc command
13# %llvmgxx - llvm-g++ command
14# %prcontext - prcontext.tcl script
15# %t - temporary file name (derived from testcase name)
16#
17
Daniel Dunbar6fc0bdf2009-03-06 22:20:40 +000018import errno
Daniel Dunbar69e07a72009-06-17 21:33:37 +000019import os
Daniel Dunbar6fc0bdf2009-03-06 22:20:40 +000020import re
Daniel Dunbar69e07a72009-06-17 21:33:37 +000021import signal
22import subprocess
23import sys
24
25# Increase determinism for things that use the terminal width.
26#
27# FIXME: Find a better place for this hack.
28os.environ['COLUMNS'] = '0'
Daniel Dunbar6fc0bdf2009-03-06 22:20:40 +000029
30class TestStatus:
31 Pass = 0
32 XFail = 1
33 Fail = 2
34 XPass = 3
35 NoRunLine = 4
36 Invalid = 5
37
38 kNames = ['Pass','XFail','Fail','XPass','NoRunLine','Invalid']
Daniel Dunbarfbbb1e72009-07-02 23:58:07 +000039 @staticmethod
Daniel Dunbar8afede22009-07-02 23:56:37 +000040 def getName(code):
41 return TestStatus.kNames[code]
Daniel Dunbar6fc0bdf2009-03-06 22:20:40 +000042
43def mkdir_p(path):
44 if not path:
45 pass
46 elif os.path.exists(path):
47 pass
48 else:
49 parent = os.path.dirname(path)
50 if parent != path:
51 mkdir_p(parent)
52 try:
53 os.mkdir(path)
54 except OSError,e:
55 if e.errno != errno.EEXIST:
56 raise
57
58def remove(path):
59 try:
60 os.remove(path)
61 except OSError:
62 pass
63
64def cat(path, output):
65 f = open(path)
66 output.writelines(f)
67 f.close()
68
Daniel Dunbar69e07a72009-06-17 21:33:37 +000069def runOneTest(FILENAME, SUBST, OUTPUT, TESTNAME, CLANG, CLANGCC,
Daniel Dunbar6fc0bdf2009-03-06 22:20:40 +000070 useValgrind=False,
71 useDGCompat=False,
72 useScript=None,
73 output=sys.stdout):
Nuno Lopesa7afc452009-07-11 18:34:43 +000074 OUTPUT = os.path.abspath(OUTPUT)
Daniel Dunbar6fc0bdf2009-03-06 22:20:40 +000075 if useValgrind:
76 VG_OUTPUT = '%s.vg'%(OUTPUT,)
Nuno Lopesa7afc452009-07-11 18:34:43 +000077 os.system('rm -f %s.*'%(VG_OUTPUT))
78 VALGRIND = 'valgrind -q --tool=memcheck --leak-check=full --trace-children=yes --log-file=%s.%%p'%(VG_OUTPUT)
79 CLANG = '%s %s'%(VALGRIND, CLANG)
80 CLANGCC = '%s %s'%(VALGRIND, CLANGCC)
Daniel Dunbar6fc0bdf2009-03-06 22:20:40 +000081
82 # Create the output directory if it does not already exist.
83 mkdir_p(os.path.dirname(OUTPUT))
84
85 # FIXME
86 #ulimit -t 40
87
88 # FIXME: Load script once
89 # FIXME: Support "short" script syntax
90
91 if useScript:
92 scriptFile = useScript
93 else:
94 # See if we have a per-dir test script.
95 dirScriptFile = os.path.join(os.path.dirname(FILENAME), 'test.script')
96 if os.path.exists(dirScriptFile):
97 scriptFile = dirScriptFile
98 else:
99 scriptFile = FILENAME
100
101 # Verify the script contains a run line.
102 for ln in open(scriptFile):
103 if 'RUN:' in ln:
104 break
105 else:
106 print >>output, "******************** TEST '%s' HAS NO RUN LINE! ********************"%(TESTNAME,)
107 output.flush()
108 return TestStatus.NoRunLine
109
Daniel Dunbar6fc0bdf2009-03-06 22:20:40 +0000110 FILENAME = os.path.abspath(FILENAME)
111 SCRIPT = OUTPUT + '.script'
112 TEMPOUTPUT = OUTPUT + '.tmp'
113
114 substitutions = [('%s',SUBST),
Daniel Dunbar4d8076a2009-04-10 19:49:21 +0000115 ('%S',os.path.dirname(SUBST)),
Daniel Dunbar6fc0bdf2009-03-06 22:20:40 +0000116 ('%llvmgcc','llvm-gcc -emit-llvm -w'),
117 ('%llvmgxx','llvm-g++ -emit-llvm -w'),
118 ('%prcontext','prcontext.tcl'),
119 ('%t',TEMPOUTPUT),
Daniel Dunbar69e07a72009-06-17 21:33:37 +0000120 (' clang ', ' ' + CLANG + ' '),
121 (' clang-cc ', ' ' + CLANGCC + ' ')]
Daniel Dunbar6fc0bdf2009-03-06 22:20:40 +0000122 scriptLines = []
123 xfailLines = []
124 for ln in open(scriptFile):
125 if 'RUN:' in ln:
126 # Isolate run parameters
127 index = ln.index('RUN:')
128 ln = ln[index+4:]
129
130 # Apply substitutions
131 for a,b in substitutions:
132 ln = ln.replace(a,b)
133
134 if useDGCompat:
135 ln = re.sub(r'\{(.*)\}', r'"\1"', ln)
136 scriptLines.append(ln)
137 elif 'XFAIL' in ln:
138 xfailLines.append(ln)
139
140 if xfailLines:
141 print >>output, "XFAILED '%s':"%(TESTNAME,)
142 output.writelines(xfailLines)
143
144 # Write script file
145 f = open(SCRIPT,'w')
146 f.write(''.join(scriptLines))
147 f.close()
148
149 outputFile = open(OUTPUT,'w')
150 p = None
151 try:
152 p = subprocess.Popen(["/bin/sh",SCRIPT],
153 cwd=os.path.dirname(FILENAME),
154 stdout=subprocess.PIPE,
155 stderr=subprocess.PIPE)
156 out,err = p.communicate()
157 outputFile.write(out)
158 outputFile.write(err)
159 SCRIPT_STATUS = p.wait()
Daniel Dunbar69e07a72009-06-17 21:33:37 +0000160
161 # Detect Ctrl-C in subprocess.
162 if SCRIPT_STATUS == -signal.SIGINT:
163 raise KeyboardInterrupt
Daniel Dunbar6fc0bdf2009-03-06 22:20:40 +0000164 except KeyboardInterrupt:
Daniel Dunbar6fc0bdf2009-03-06 22:20:40 +0000165 raise
166 outputFile.close()
167
168 if xfailLines:
169 SCRIPT_STATUS = not SCRIPT_STATUS
170
171 if useValgrind:
Nuno Lopesa7afc452009-07-11 18:34:43 +0000172 VG_OUTPUT = capture(['/bin/sh','-c','cat %s.*'%(VG_OUTPUT)])
173 VG_STATUS = len(VG_OUTPUT)
Daniel Dunbar6fc0bdf2009-03-06 22:20:40 +0000174 else:
175 VG_STATUS = 0
176
177 if SCRIPT_STATUS or VG_STATUS:
178 print >>output, "******************** TEST '%s' FAILED! ********************"%(TESTNAME,)
179 print >>output, "Command: "
180 output.writelines(scriptLines)
181 if not SCRIPT_STATUS:
182 print >>output, "Output:"
183 else:
184 print >>output, "Incorrect Output:"
185 cat(OUTPUT, output)
186 if VG_STATUS:
187 print >>output, "Valgrind Output:"
Nuno Lopesa7afc452009-07-11 18:34:43 +0000188 print >>output, VG_OUTPUT
Daniel Dunbar6fc0bdf2009-03-06 22:20:40 +0000189 print >>output, "******************** TEST '%s' FAILED! ********************"%(TESTNAME,)
190 output.flush()
191 if xfailLines:
192 return TestStatus.XPass
193 else:
194 return TestStatus.Fail
195
196 if xfailLines:
197 return TestStatus.XFail
198 else:
199 return TestStatus.Pass
Daniel Dunbar69e07a72009-06-17 21:33:37 +0000200
201def capture(args):
202 p = subprocess.Popen(args, stdout=subprocess.PIPE)
203 out,_ = p.communicate()
204 return out
205
206def which(command):
207 # Would be nice if Python had a lib function for this.
208 res = capture(['which',command])
209 res = res.strip()
210 if res and os.path.exists(res):
211 return res
212 return None
213
214def inferClang():
215 # Determine which clang to use.
216 clang = os.getenv('CLANG')
217
218 # If the user set clang in the environment, definitely use that and don't
219 # try to validate.
220 if clang:
221 return clang
222
223 # Otherwise look in the path.
224 clang = which('clang')
225
226 if not clang:
227 print >>sys.stderr, "error: couldn't find 'clang' program, try setting CLANG in your environment"
228 sys.exit(1)
229
230 return clang
231
232def inferClangCC(clang):
233 clangcc = os.getenv('CLANGCC')
234
235 # If the user set clang in the environment, definitely use that and don't
236 # try to validate.
237 if clangcc:
238 return clangcc
239
240 # Otherwise try adding -cc since we expect to be looking in a build
241 # directory.
242 clangcc = which(clang + '-cc')
243 if not clangcc:
244 # Otherwise ask clang.
245 res = capture([clang, '-print-prog-name=clang-cc'])
246 res = res.strip()
247 if res and os.path.exists(res):
248 clangcc = res
249
250 if not clangcc:
251 print >>sys.stderr, "error: couldn't find 'clang-cc' program, try setting CLANGCC in your environment"
252 sys.exit(1)
253
254 return clangcc
Daniel Dunbar6fc0bdf2009-03-06 22:20:40 +0000255
256def main():
Daniel Dunbar69e07a72009-06-17 21:33:37 +0000257 global options
258 from optparse import OptionParser
259 parser = OptionParser("usage: %prog [options] {tests}")
260 parser.add_option("", "--clang", dest="clang",
261 help="Program to use as \"clang\"",
262 action="store", default=None)
263 parser.add_option("", "--clang-cc", dest="clangcc",
264 help="Program to use as \"clang-cc\"",
265 action="store", default=None)
266 parser.add_option("", "--vg", dest="useValgrind",
267 help="Run tests under valgrind",
268 action="store_true", default=False)
269 parser.add_option("", "--dg", dest="useDGCompat",
270 help="Use llvm dejagnu compatibility mode",
271 action="store_true", default=False)
272 (opts, args) = parser.parse_args()
Daniel Dunbar6fc0bdf2009-03-06 22:20:40 +0000273
Daniel Dunbar69e07a72009-06-17 21:33:37 +0000274 if not args:
275 parser.error('No tests specified')
Daniel Dunbar6fc0bdf2009-03-06 22:20:40 +0000276
Daniel Dunbar69e07a72009-06-17 21:33:37 +0000277 if opts.clang is None:
278 opts.clang = inferClang()
279 if opts.clangcc is None:
280 opts.clangcc = inferClangCC(opts.clang)
281
282 for path in args:
283 command = path
284 # Use hand concatentation here because we want to override
285 # absolute paths.
286 output = 'Output/' + path + '.out'
287 testname = path
288
289 res = runOneTest(path, command, output, testname,
290 opts.clang, opts.clangcc,
291 useValgrind=opts.useValgrind,
292 useDGCompat=opts.useDGCompat,
293 useScript=os.getenv("TEST_SCRIPT"))
Daniel Dunbar6fc0bdf2009-03-06 22:20:40 +0000294
295 sys.exit(res == TestStatus.Fail or res == TestStatus.XPass)
296
297if __name__=='__main__':
298 main()