Eli Friedman | 77a1fe9 | 2009-07-10 20:15:12 +0000 | [diff] [blame] | 1 | #!/usr/bin/env python |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 2 | # |
| 3 | # TestRunner.py - This script is used to run arbitrary unit tests. Unit |
| 4 | # tests must contain the command used to run them in the input file, starting |
| 5 | # immediately after a "RUN:" string. |
| 6 | # |
| 7 | # This runner recognizes and replaces the following strings in the command: |
| 8 | # |
| 9 | # %s - Replaced with the input name of the program, or the program to |
| 10 | # execute, as appropriate. |
Daniel Dunbar | 4d8076a | 2009-04-10 19:49:21 +0000 | [diff] [blame] | 11 | # %S - Replaced with the directory where the input resides. |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 12 | # %llvmgcc - llvm-gcc command |
| 13 | # %llvmgxx - llvm-g++ command |
| 14 | # %prcontext - prcontext.tcl script |
| 15 | # %t - temporary file name (derived from testcase name) |
| 16 | # |
| 17 | |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 18 | import errno |
Daniel Dunbar | 69e07a7 | 2009-06-17 21:33:37 +0000 | [diff] [blame] | 19 | import os |
Daniel Dunbar | 8fe83f1 | 2009-07-25 09:42:24 +0000 | [diff] [blame] | 20 | import platform |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 21 | import re |
Daniel Dunbar | 69e07a7 | 2009-06-17 21:33:37 +0000 | [diff] [blame] | 22 | import signal |
| 23 | import subprocess |
| 24 | import sys |
| 25 | |
| 26 | # Increase determinism for things that use the terminal width. |
| 27 | # |
| 28 | # FIXME: Find a better place for this hack. |
| 29 | os.environ['COLUMNS'] = '0' |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 30 | |
Daniel Dunbar | 8fe83f1 | 2009-07-25 09:42:24 +0000 | [diff] [blame] | 31 | kSystemName = platform.system() |
| 32 | |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 33 | class TestStatus: |
| 34 | Pass = 0 |
| 35 | XFail = 1 |
| 36 | Fail = 2 |
| 37 | XPass = 3 |
| 38 | NoRunLine = 4 |
| 39 | Invalid = 5 |
| 40 | |
| 41 | kNames = ['Pass','XFail','Fail','XPass','NoRunLine','Invalid'] |
Daniel Dunbar | fbbb1e7 | 2009-07-02 23:58:07 +0000 | [diff] [blame] | 42 | @staticmethod |
Daniel Dunbar | 8afede2 | 2009-07-02 23:56:37 +0000 | [diff] [blame] | 43 | def getName(code): |
| 44 | return TestStatus.kNames[code] |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 45 | |
| 46 | def mkdir_p(path): |
| 47 | if not path: |
| 48 | pass |
| 49 | elif os.path.exists(path): |
| 50 | pass |
| 51 | else: |
| 52 | parent = os.path.dirname(path) |
| 53 | if parent != path: |
| 54 | mkdir_p(parent) |
| 55 | try: |
| 56 | os.mkdir(path) |
| 57 | except OSError,e: |
| 58 | if e.errno != errno.EEXIST: |
| 59 | raise |
| 60 | |
| 61 | def remove(path): |
| 62 | try: |
| 63 | os.remove(path) |
| 64 | except OSError: |
| 65 | pass |
| 66 | |
| 67 | def cat(path, output): |
| 68 | f = open(path) |
| 69 | output.writelines(f) |
| 70 | f.close() |
| 71 | |
Daniel Dunbar | 69e07a7 | 2009-06-17 21:33:37 +0000 | [diff] [blame] | 72 | def runOneTest(FILENAME, SUBST, OUTPUT, TESTNAME, CLANG, CLANGCC, |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 73 | useValgrind=False, |
| 74 | useDGCompat=False, |
| 75 | useScript=None, |
| 76 | output=sys.stdout): |
Nuno Lopes | a7afc45 | 2009-07-11 18:34:43 +0000 | [diff] [blame] | 77 | OUTPUT = os.path.abspath(OUTPUT) |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 78 | if useValgrind: |
| 79 | VG_OUTPUT = '%s.vg'%(OUTPUT,) |
Nuno Lopes | a7afc45 | 2009-07-11 18:34:43 +0000 | [diff] [blame] | 80 | os.system('rm -f %s.*'%(VG_OUTPUT)) |
| 81 | VALGRIND = 'valgrind -q --tool=memcheck --leak-check=full --trace-children=yes --log-file=%s.%%p'%(VG_OUTPUT) |
| 82 | CLANG = '%s %s'%(VALGRIND, CLANG) |
| 83 | CLANGCC = '%s %s'%(VALGRIND, CLANGCC) |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 84 | |
| 85 | # Create the output directory if it does not already exist. |
| 86 | mkdir_p(os.path.dirname(OUTPUT)) |
| 87 | |
| 88 | # FIXME |
| 89 | #ulimit -t 40 |
| 90 | |
| 91 | # FIXME: Load script once |
| 92 | # FIXME: Support "short" script syntax |
| 93 | |
| 94 | if useScript: |
| 95 | scriptFile = useScript |
| 96 | else: |
| 97 | # See if we have a per-dir test script. |
| 98 | dirScriptFile = os.path.join(os.path.dirname(FILENAME), 'test.script') |
| 99 | if os.path.exists(dirScriptFile): |
| 100 | scriptFile = dirScriptFile |
| 101 | else: |
| 102 | scriptFile = FILENAME |
| 103 | |
| 104 | # Verify the script contains a run line. |
| 105 | for ln in open(scriptFile): |
| 106 | if 'RUN:' in ln: |
| 107 | break |
| 108 | else: |
| 109 | print >>output, "******************** TEST '%s' HAS NO RUN LINE! ********************"%(TESTNAME,) |
| 110 | output.flush() |
| 111 | return TestStatus.NoRunLine |
| 112 | |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 113 | FILENAME = os.path.abspath(FILENAME) |
| 114 | SCRIPT = OUTPUT + '.script' |
Daniel Dunbar | 8fe83f1 | 2009-07-25 09:42:24 +0000 | [diff] [blame] | 115 | if kSystemName == 'Windows': |
| 116 | SCRIPT += '.bat' |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 117 | TEMPOUTPUT = OUTPUT + '.tmp' |
| 118 | |
| 119 | substitutions = [('%s',SUBST), |
Daniel Dunbar | 4d8076a | 2009-04-10 19:49:21 +0000 | [diff] [blame] | 120 | ('%S',os.path.dirname(SUBST)), |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 121 | ('%llvmgcc','llvm-gcc -emit-llvm -w'), |
| 122 | ('%llvmgxx','llvm-g++ -emit-llvm -w'), |
| 123 | ('%prcontext','prcontext.tcl'), |
| 124 | ('%t',TEMPOUTPUT), |
Daniel Dunbar | 69e07a7 | 2009-06-17 21:33:37 +0000 | [diff] [blame] | 125 | (' clang ', ' ' + CLANG + ' '), |
| 126 | (' clang-cc ', ' ' + CLANGCC + ' ')] |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 127 | scriptLines = [] |
| 128 | xfailLines = [] |
| 129 | for ln in open(scriptFile): |
| 130 | if 'RUN:' in ln: |
| 131 | # Isolate run parameters |
| 132 | index = ln.index('RUN:') |
| 133 | ln = ln[index+4:] |
| 134 | |
| 135 | # Apply substitutions |
| 136 | for a,b in substitutions: |
| 137 | ln = ln.replace(a,b) |
| 138 | |
| 139 | if useDGCompat: |
| 140 | ln = re.sub(r'\{(.*)\}', r'"\1"', ln) |
| 141 | scriptLines.append(ln) |
| 142 | elif 'XFAIL' in ln: |
| 143 | xfailLines.append(ln) |
| 144 | |
| 145 | if xfailLines: |
| 146 | print >>output, "XFAILED '%s':"%(TESTNAME,) |
| 147 | output.writelines(xfailLines) |
| 148 | |
| 149 | # Write script file |
| 150 | f = open(SCRIPT,'w') |
| 151 | f.write(''.join(scriptLines)) |
| 152 | f.close() |
| 153 | |
| 154 | outputFile = open(OUTPUT,'w') |
| 155 | p = None |
| 156 | try: |
Daniel Dunbar | 8fe83f1 | 2009-07-25 09:42:24 +0000 | [diff] [blame] | 157 | if kSystemName == 'Windows': |
| 158 | command = ['cmd','/c', SCRIPT] |
| 159 | else: |
| 160 | command = ['/bin/sh', SCRIPT] |
| 161 | |
| 162 | p = subprocess.Popen(command, |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 163 | cwd=os.path.dirname(FILENAME), |
Daniel Dunbar | 02cee15 | 2009-07-13 21:24:28 +0000 | [diff] [blame] | 164 | stdin=subprocess.PIPE, |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 165 | stdout=subprocess.PIPE, |
| 166 | stderr=subprocess.PIPE) |
| 167 | out,err = p.communicate() |
| 168 | outputFile.write(out) |
| 169 | outputFile.write(err) |
| 170 | SCRIPT_STATUS = p.wait() |
Daniel Dunbar | 69e07a7 | 2009-06-17 21:33:37 +0000 | [diff] [blame] | 171 | |
| 172 | # Detect Ctrl-C in subprocess. |
| 173 | if SCRIPT_STATUS == -signal.SIGINT: |
| 174 | raise KeyboardInterrupt |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 175 | except KeyboardInterrupt: |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 176 | raise |
| 177 | outputFile.close() |
| 178 | |
| 179 | if xfailLines: |
| 180 | SCRIPT_STATUS = not SCRIPT_STATUS |
| 181 | |
| 182 | if useValgrind: |
Daniel Dunbar | 8fe83f1 | 2009-07-25 09:42:24 +0000 | [diff] [blame] | 183 | if kSystemName == 'Windows': |
| 184 | raise NotImplementedError,'Cannot run valgrind on windows' |
| 185 | else: |
| 186 | VG_OUTPUT = capture(['/bin/sh','-c','cat %s.*'%(VG_OUTPUT)]) |
Nuno Lopes | a7afc45 | 2009-07-11 18:34:43 +0000 | [diff] [blame] | 187 | VG_STATUS = len(VG_OUTPUT) |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 188 | else: |
| 189 | VG_STATUS = 0 |
| 190 | |
| 191 | if SCRIPT_STATUS or VG_STATUS: |
| 192 | print >>output, "******************** TEST '%s' FAILED! ********************"%(TESTNAME,) |
| 193 | print >>output, "Command: " |
| 194 | output.writelines(scriptLines) |
| 195 | if not SCRIPT_STATUS: |
| 196 | print >>output, "Output:" |
| 197 | else: |
| 198 | print >>output, "Incorrect Output:" |
| 199 | cat(OUTPUT, output) |
| 200 | if VG_STATUS: |
| 201 | print >>output, "Valgrind Output:" |
Nuno Lopes | a7afc45 | 2009-07-11 18:34:43 +0000 | [diff] [blame] | 202 | print >>output, VG_OUTPUT |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 203 | print >>output, "******************** TEST '%s' FAILED! ********************"%(TESTNAME,) |
| 204 | output.flush() |
| 205 | if xfailLines: |
| 206 | return TestStatus.XPass |
| 207 | else: |
| 208 | return TestStatus.Fail |
| 209 | |
| 210 | if xfailLines: |
| 211 | return TestStatus.XFail |
| 212 | else: |
| 213 | return TestStatus.Pass |
Daniel Dunbar | 69e07a7 | 2009-06-17 21:33:37 +0000 | [diff] [blame] | 214 | |
| 215 | def capture(args): |
Daniel Dunbar | 8fe83f1 | 2009-07-25 09:42:24 +0000 | [diff] [blame] | 216 | p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
Daniel Dunbar | 69e07a7 | 2009-06-17 21:33:37 +0000 | [diff] [blame] | 217 | out,_ = p.communicate() |
| 218 | return out |
| 219 | |
| 220 | def which(command): |
Daniel Dunbar | 8fe83f1 | 2009-07-25 09:42:24 +0000 | [diff] [blame] | 221 | # Check for absolute match first. |
| 222 | if os.path.exists(command): |
| 223 | return command |
| 224 | |
Daniel Dunbar | 69e07a7 | 2009-06-17 21:33:37 +0000 | [diff] [blame] | 225 | # Would be nice if Python had a lib function for this. |
Daniel Dunbar | 8fe83f1 | 2009-07-25 09:42:24 +0000 | [diff] [blame] | 226 | paths = os.environ.get('PATH') |
| 227 | if not paths: |
| 228 | paths = os.defpath |
| 229 | |
| 230 | # Get suffixes to search. |
| 231 | pathext = os.environ.get('PATHEXT', '').split(os.pathsep) |
| 232 | |
| 233 | # Search the paths... |
| 234 | for path in paths.split(os.pathsep): |
| 235 | for ext in pathext: |
| 236 | p = os.path.join(path, command + ext) |
| 237 | if os.path.exists(p): |
| 238 | return p |
| 239 | |
Daniel Dunbar | 69e07a7 | 2009-06-17 21:33:37 +0000 | [diff] [blame] | 240 | return None |
| 241 | |
| 242 | def inferClang(): |
| 243 | # Determine which clang to use. |
| 244 | clang = os.getenv('CLANG') |
| 245 | |
| 246 | # If the user set clang in the environment, definitely use that and don't |
| 247 | # try to validate. |
| 248 | if clang: |
| 249 | return clang |
| 250 | |
| 251 | # Otherwise look in the path. |
| 252 | clang = which('clang') |
| 253 | |
| 254 | if not clang: |
| 255 | print >>sys.stderr, "error: couldn't find 'clang' program, try setting CLANG in your environment" |
| 256 | sys.exit(1) |
| 257 | |
| 258 | return clang |
| 259 | |
| 260 | def inferClangCC(clang): |
| 261 | clangcc = os.getenv('CLANGCC') |
| 262 | |
| 263 | # If the user set clang in the environment, definitely use that and don't |
| 264 | # try to validate. |
| 265 | if clangcc: |
| 266 | return clangcc |
| 267 | |
| 268 | # Otherwise try adding -cc since we expect to be looking in a build |
| 269 | # directory. |
Daniel Dunbar | 8fe83f1 | 2009-07-25 09:42:24 +0000 | [diff] [blame] | 270 | if clang.endswith('.exe'): |
| 271 | clangccName = clang[:-4] + '-cc.exe' |
| 272 | else: |
| 273 | clangccName = clang + '-cc' |
| 274 | clangcc = which(clangccName) |
Daniel Dunbar | 69e07a7 | 2009-06-17 21:33:37 +0000 | [diff] [blame] | 275 | if not clangcc: |
| 276 | # Otherwise ask clang. |
| 277 | res = capture([clang, '-print-prog-name=clang-cc']) |
| 278 | res = res.strip() |
| 279 | if res and os.path.exists(res): |
| 280 | clangcc = res |
| 281 | |
| 282 | if not clangcc: |
| 283 | print >>sys.stderr, "error: couldn't find 'clang-cc' program, try setting CLANGCC in your environment" |
| 284 | sys.exit(1) |
| 285 | |
| 286 | return clangcc |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 287 | |
Daniel Dunbar | df08489 | 2009-07-25 09:53:43 +0000 | [diff] [blame^] | 288 | def getTestOutputBase(dir, testpath): |
| 289 | """getTestOutputPath(dir, testpath) - Get the full path for temporary files |
| 290 | corresponding to the given test path.""" |
| 291 | |
| 292 | # Form the output base out of the test parent directory name and the test |
| 293 | # name. FIXME: Find a better way to organize test results. |
| 294 | return os.path.join(dir, |
| 295 | os.path.basename(os.path.dirname(testpath)), |
| 296 | os.path.basename(testpath)) |
| 297 | |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 298 | def main(): |
Daniel Dunbar | 69e07a7 | 2009-06-17 21:33:37 +0000 | [diff] [blame] | 299 | global options |
| 300 | from optparse import OptionParser |
| 301 | parser = OptionParser("usage: %prog [options] {tests}") |
| 302 | parser.add_option("", "--clang", dest="clang", |
| 303 | help="Program to use as \"clang\"", |
| 304 | action="store", default=None) |
| 305 | parser.add_option("", "--clang-cc", dest="clangcc", |
| 306 | help="Program to use as \"clang-cc\"", |
| 307 | action="store", default=None) |
| 308 | parser.add_option("", "--vg", dest="useValgrind", |
| 309 | help="Run tests under valgrind", |
| 310 | action="store_true", default=False) |
| 311 | parser.add_option("", "--dg", dest="useDGCompat", |
| 312 | help="Use llvm dejagnu compatibility mode", |
| 313 | action="store_true", default=False) |
| 314 | (opts, args) = parser.parse_args() |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 315 | |
Daniel Dunbar | 69e07a7 | 2009-06-17 21:33:37 +0000 | [diff] [blame] | 316 | if not args: |
| 317 | parser.error('No tests specified') |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 318 | |
Daniel Dunbar | 69e07a7 | 2009-06-17 21:33:37 +0000 | [diff] [blame] | 319 | if opts.clang is None: |
| 320 | opts.clang = inferClang() |
| 321 | if opts.clangcc is None: |
| 322 | opts.clangcc = inferClangCC(opts.clang) |
| 323 | |
| 324 | for path in args: |
| 325 | command = path |
Daniel Dunbar | df08489 | 2009-07-25 09:53:43 +0000 | [diff] [blame^] | 326 | output = getTestOutputPath('Output', path) + '.out' |
Daniel Dunbar | 69e07a7 | 2009-06-17 21:33:37 +0000 | [diff] [blame] | 327 | testname = path |
| 328 | |
| 329 | res = runOneTest(path, command, output, testname, |
| 330 | opts.clang, opts.clangcc, |
| 331 | useValgrind=opts.useValgrind, |
| 332 | useDGCompat=opts.useDGCompat, |
| 333 | useScript=os.getenv("TEST_SCRIPT")) |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 334 | |
| 335 | sys.exit(res == TestStatus.Fail or res == TestStatus.XPass) |
| 336 | |
| 337 | if __name__=='__main__': |
| 338 | main() |