Eli Friedman | 77a1fe9 | 2009-07-10 20:15:12 +0000 | [diff] [blame] | 1 | #!/usr/bin/env python |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 2 | # |
| 3 | # TestRunner.py - This script is used to run arbitrary unit tests. Unit |
| 4 | # tests must contain the command used to run them in the input file, starting |
| 5 | # immediately after a "RUN:" string. |
| 6 | # |
| 7 | # This runner recognizes and replaces the following strings in the command: |
| 8 | # |
| 9 | # %s - Replaced with the input name of the program, or the program to |
| 10 | # execute, as appropriate. |
Daniel Dunbar | 4d8076a | 2009-04-10 19:49:21 +0000 | [diff] [blame] | 11 | # %S - Replaced with the directory where the input resides. |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 12 | # %llvmgcc - llvm-gcc command |
| 13 | # %llvmgxx - llvm-g++ command |
| 14 | # %prcontext - prcontext.tcl script |
| 15 | # %t - temporary file name (derived from testcase name) |
| 16 | # |
| 17 | |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 18 | import errno |
Daniel Dunbar | 69e07a7 | 2009-06-17 21:33:37 +0000 | [diff] [blame] | 19 | import os |
Daniel Dunbar | 8fe83f1 | 2009-07-25 09:42:24 +0000 | [diff] [blame] | 20 | import platform |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 21 | import re |
Daniel Dunbar | 69e07a7 | 2009-06-17 21:33:37 +0000 | [diff] [blame] | 22 | import signal |
| 23 | import subprocess |
| 24 | import sys |
| 25 | |
Daniel Dunbar | 78a88e7 | 2009-07-25 10:14:19 +0000 | [diff] [blame] | 26 | # Increase determinism by explicitly choosing the environment. |
Daniel Dunbar | 3b2505b | 2009-07-25 12:57:15 +0000 | [diff] [blame] | 27 | kChildEnv = {} |
| 28 | for var in ('PATH', 'SYSTEMROOT'): |
| 29 | kChildEnv[var] = os.environ.get(var, '') |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 30 | |
Daniel Dunbar | 8fe83f1 | 2009-07-25 09:42:24 +0000 | [diff] [blame] | 31 | kSystemName = platform.system() |
| 32 | |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 33 | class TestStatus: |
| 34 | Pass = 0 |
| 35 | XFail = 1 |
| 36 | Fail = 2 |
| 37 | XPass = 3 |
Daniel Dunbar | 8bf0ccd | 2009-07-25 12:47:38 +0000 | [diff] [blame] | 38 | Invalid = 4 |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 39 | |
Daniel Dunbar | 8bf0ccd | 2009-07-25 12:47:38 +0000 | [diff] [blame] | 40 | kNames = ['Pass','XFail','Fail','XPass','Invalid'] |
Daniel Dunbar | fbbb1e7 | 2009-07-02 23:58:07 +0000 | [diff] [blame] | 41 | @staticmethod |
Daniel Dunbar | 8afede2 | 2009-07-02 23:56:37 +0000 | [diff] [blame] | 42 | def getName(code): |
| 43 | return TestStatus.kNames[code] |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 44 | |
| 45 | def mkdir_p(path): |
| 46 | if not path: |
| 47 | pass |
| 48 | elif os.path.exists(path): |
| 49 | pass |
| 50 | else: |
| 51 | parent = os.path.dirname(path) |
| 52 | if parent != path: |
| 53 | mkdir_p(parent) |
| 54 | try: |
| 55 | os.mkdir(path) |
| 56 | except OSError,e: |
| 57 | if e.errno != errno.EEXIST: |
| 58 | raise |
| 59 | |
Daniel Dunbar | 10aebbb | 2009-07-25 15:26:08 +0000 | [diff] [blame] | 60 | def executeScript(script, commands, cwd): |
| 61 | # Write script file |
| 62 | f = open(script,'w') |
| 63 | if kSystemName == 'Windows': |
| 64 | f.write('\nif %ERRORLEVEL% NEQ 0 EXIT\n'.join(commands)) |
| 65 | else: |
| 66 | f.write(' &&\n'.join(commands)) |
| 67 | f.write('\n') |
| 68 | f.close() |
| 69 | |
| 70 | if kSystemName == 'Windows': |
| 71 | command = ['cmd','/c', script] |
| 72 | else: |
| 73 | command = ['/bin/sh', script] |
| 74 | |
| 75 | p = subprocess.Popen(command, cwd=cwd, |
| 76 | stdin=subprocess.PIPE, |
| 77 | stdout=subprocess.PIPE, |
| 78 | stderr=subprocess.PIPE, |
| 79 | env=kChildEnv) |
| 80 | out,err = p.communicate() |
| 81 | exitCode = p.wait() |
| 82 | |
| 83 | # Detect Ctrl-C in subprocess. |
| 84 | if exitCode == -signal.SIGINT: |
| 85 | raise KeyboardInterrupt |
| 86 | |
| 87 | return out, err, exitCode |
| 88 | |
Daniel Dunbar | a957d99 | 2009-07-25 14:46:05 +0000 | [diff] [blame] | 89 | import StringIO |
| 90 | def runOneTest(testPath, tmpBase, clang, clangcc): |
| 91 | # Make paths absolute. |
| 92 | tmpBase = os.path.abspath(tmpBase) |
| 93 | testPath = os.path.abspath(testPath) |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 94 | |
| 95 | # Create the output directory if it does not already exist. |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 96 | |
Daniel Dunbar | a957d99 | 2009-07-25 14:46:05 +0000 | [diff] [blame] | 97 | mkdir_p(os.path.dirname(tmpBase)) |
| 98 | script = tmpBase + '.script' |
Daniel Dunbar | 8fe83f1 | 2009-07-25 09:42:24 +0000 | [diff] [blame] | 99 | if kSystemName == 'Windows': |
Daniel Dunbar | a957d99 | 2009-07-25 14:46:05 +0000 | [diff] [blame] | 100 | script += '.bat' |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 101 | |
Daniel Dunbar | a957d99 | 2009-07-25 14:46:05 +0000 | [diff] [blame] | 102 | substitutions = [('%s', testPath), |
| 103 | ('%S', os.path.dirname(testPath)), |
| 104 | ('%t', tmpBase + '.tmp'), |
| 105 | (' clang ', ' ' + clang + ' '), |
| 106 | (' clang-cc ', ' ' + clangcc + ' ')] |
Daniel Dunbar | 025f80d | 2009-07-25 11:27:37 +0000 | [diff] [blame] | 107 | |
| 108 | # Collect the test lines from the script. |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 109 | scriptLines = [] |
| 110 | xfailLines = [] |
Daniel Dunbar | a957d99 | 2009-07-25 14:46:05 +0000 | [diff] [blame] | 111 | for ln in open(testPath): |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 112 | if 'RUN:' in ln: |
Daniel Dunbar | 025f80d | 2009-07-25 11:27:37 +0000 | [diff] [blame] | 113 | # Isolate the command to run. |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 114 | index = ln.index('RUN:') |
| 115 | ln = ln[index+4:] |
Daniel Dunbar | 025f80d | 2009-07-25 11:27:37 +0000 | [diff] [blame] | 116 | |
Daniel Dunbar | 322f789 | 2009-07-25 12:23:35 +0000 | [diff] [blame] | 117 | # Strip trailing newline. |
| 118 | scriptLines.append(ln) |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 119 | elif 'XFAIL' in ln: |
| 120 | xfailLines.append(ln) |
Daniel Dunbar | 025f80d | 2009-07-25 11:27:37 +0000 | [diff] [blame] | 121 | |
| 122 | # FIXME: Support something like END, in case we need to process large |
| 123 | # files. |
Daniel Dunbar | a957d99 | 2009-07-25 14:46:05 +0000 | [diff] [blame] | 124 | |
| 125 | # Verify the script contains a run line. |
| 126 | if not scriptLines: |
| 127 | return (TestStatus.Fail, "Test has no run line!") |
Daniel Dunbar | 322f789 | 2009-07-25 12:23:35 +0000 | [diff] [blame] | 128 | |
| 129 | # Apply substitutions to the script. |
| 130 | def processLine(ln): |
| 131 | # Apply substitutions |
| 132 | for a,b in substitutions: |
| 133 | ln = ln.replace(a,b) |
| 134 | |
Daniel Dunbar | 322f789 | 2009-07-25 12:23:35 +0000 | [diff] [blame] | 135 | # Strip the trailing newline and any extra whitespace. |
| 136 | return ln.strip() |
| 137 | scriptLines = map(processLine, scriptLines) |
Daniel Dunbar | 025f80d | 2009-07-25 11:27:37 +0000 | [diff] [blame] | 138 | |
| 139 | # Validate interior lines for '&&', a lovely historical artifact. |
| 140 | for i in range(len(scriptLines) - 1): |
| 141 | ln = scriptLines[i] |
| 142 | |
| 143 | if not ln.endswith('&&'): |
Daniel Dunbar | a957d99 | 2009-07-25 14:46:05 +0000 | [diff] [blame] | 144 | return (TestStatus.Fail, |
Daniel Dunbar | 6779647 | 2009-07-27 19:01:13 +0000 | [diff] [blame^] | 145 | ("MISSING \'&&\': %s\n" + |
| 146 | "FOLLOWED BY : %s\n") % (ln, scriptLines[i + 1])) |
Daniel Dunbar | 025f80d | 2009-07-25 11:27:37 +0000 | [diff] [blame] | 147 | |
| 148 | # Strip off '&&' |
| 149 | scriptLines[i] = ln[:-2] |
Daniel Dunbar | 025f80d | 2009-07-25 11:27:37 +0000 | [diff] [blame] | 150 | |
Daniel Dunbar | 10aebbb | 2009-07-25 15:26:08 +0000 | [diff] [blame] | 151 | out, err, exitCode = executeScript(script, scriptLines, |
| 152 | cwd=os.path.dirname(testPath)) |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 153 | if xfailLines: |
Daniel Dunbar | a957d99 | 2009-07-25 14:46:05 +0000 | [diff] [blame] | 154 | ok = exitCode != 0 |
| 155 | status = (TestStatus.XPass, TestStatus.XFail)[ok] |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 156 | else: |
Daniel Dunbar | a957d99 | 2009-07-25 14:46:05 +0000 | [diff] [blame] | 157 | ok = exitCode == 0 |
| 158 | status = (TestStatus.Fail, TestStatus.Pass)[ok] |
| 159 | |
| 160 | if ok: |
| 161 | return (status,'') |
| 162 | |
| 163 | output = StringIO.StringIO() |
| 164 | print >>output, "Script:" |
| 165 | print >>output, "--" |
| 166 | print >>output, '\n'.join(scriptLines) |
| 167 | print >>output, "--" |
| 168 | print >>output, "Exit Code: %r" % exitCode |
| 169 | print >>output, "Command Output (stdout):" |
| 170 | print >>output, "--" |
| 171 | output.write(out) |
| 172 | print >>output, "--" |
| 173 | print >>output, "Command Output (stderr):" |
| 174 | print >>output, "--" |
| 175 | output.write(err) |
| 176 | print >>output, "--" |
| 177 | return (status, output.getvalue()) |
Daniel Dunbar | 69e07a7 | 2009-06-17 21:33:37 +0000 | [diff] [blame] | 178 | |
| 179 | def capture(args): |
Daniel Dunbar | 8fe83f1 | 2009-07-25 09:42:24 +0000 | [diff] [blame] | 180 | p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
Daniel Dunbar | 69e07a7 | 2009-06-17 21:33:37 +0000 | [diff] [blame] | 181 | out,_ = p.communicate() |
| 182 | return out |
| 183 | |
| 184 | def which(command): |
Daniel Dunbar | 6779647 | 2009-07-27 19:01:13 +0000 | [diff] [blame^] | 185 | # FIXME: Take configuration object. |
| 186 | |
Daniel Dunbar | 8fe83f1 | 2009-07-25 09:42:24 +0000 | [diff] [blame] | 187 | # Check for absolute match first. |
| 188 | if os.path.exists(command): |
| 189 | return command |
| 190 | |
Daniel Dunbar | 69e07a7 | 2009-06-17 21:33:37 +0000 | [diff] [blame] | 191 | # Would be nice if Python had a lib function for this. |
Daniel Dunbar | 6779647 | 2009-07-27 19:01:13 +0000 | [diff] [blame^] | 192 | paths = kChildEnv['PATH'] |
Daniel Dunbar | 8fe83f1 | 2009-07-25 09:42:24 +0000 | [diff] [blame] | 193 | if not paths: |
| 194 | paths = os.defpath |
| 195 | |
| 196 | # Get suffixes to search. |
| 197 | pathext = os.environ.get('PATHEXT', '').split(os.pathsep) |
| 198 | |
| 199 | # Search the paths... |
| 200 | for path in paths.split(os.pathsep): |
| 201 | for ext in pathext: |
| 202 | p = os.path.join(path, command + ext) |
| 203 | if os.path.exists(p): |
| 204 | return p |
| 205 | |
Daniel Dunbar | 69e07a7 | 2009-06-17 21:33:37 +0000 | [diff] [blame] | 206 | return None |
| 207 | |
| 208 | def inferClang(): |
| 209 | # Determine which clang to use. |
| 210 | clang = os.getenv('CLANG') |
| 211 | |
| 212 | # If the user set clang in the environment, definitely use that and don't |
| 213 | # try to validate. |
| 214 | if clang: |
| 215 | return clang |
| 216 | |
| 217 | # Otherwise look in the path. |
| 218 | clang = which('clang') |
| 219 | |
| 220 | if not clang: |
| 221 | print >>sys.stderr, "error: couldn't find 'clang' program, try setting CLANG in your environment" |
| 222 | sys.exit(1) |
| 223 | |
| 224 | return clang |
| 225 | |
| 226 | def inferClangCC(clang): |
| 227 | clangcc = os.getenv('CLANGCC') |
| 228 | |
| 229 | # If the user set clang in the environment, definitely use that and don't |
| 230 | # try to validate. |
| 231 | if clangcc: |
| 232 | return clangcc |
| 233 | |
| 234 | # Otherwise try adding -cc since we expect to be looking in a build |
| 235 | # directory. |
Daniel Dunbar | 8fe83f1 | 2009-07-25 09:42:24 +0000 | [diff] [blame] | 236 | if clang.endswith('.exe'): |
| 237 | clangccName = clang[:-4] + '-cc.exe' |
| 238 | else: |
| 239 | clangccName = clang + '-cc' |
| 240 | clangcc = which(clangccName) |
Daniel Dunbar | 69e07a7 | 2009-06-17 21:33:37 +0000 | [diff] [blame] | 241 | if not clangcc: |
| 242 | # Otherwise ask clang. |
| 243 | res = capture([clang, '-print-prog-name=clang-cc']) |
| 244 | res = res.strip() |
| 245 | if res and os.path.exists(res): |
| 246 | clangcc = res |
| 247 | |
| 248 | if not clangcc: |
| 249 | print >>sys.stderr, "error: couldn't find 'clang-cc' program, try setting CLANGCC in your environment" |
| 250 | sys.exit(1) |
| 251 | |
| 252 | return clangcc |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 253 | |
Daniel Dunbar | df08489 | 2009-07-25 09:53:43 +0000 | [diff] [blame] | 254 | def getTestOutputBase(dir, testpath): |
Daniel Dunbar | fecdd00 | 2009-07-25 12:05:55 +0000 | [diff] [blame] | 255 | """getTestOutputBase(dir, testpath) - Get the full path for temporary files |
Daniel Dunbar | df08489 | 2009-07-25 09:53:43 +0000 | [diff] [blame] | 256 | corresponding to the given test path.""" |
| 257 | |
| 258 | # Form the output base out of the test parent directory name and the test |
| 259 | # name. FIXME: Find a better way to organize test results. |
| 260 | return os.path.join(dir, |
| 261 | os.path.basename(os.path.dirname(testpath)), |
| 262 | os.path.basename(testpath)) |
| 263 | |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 264 | def main(): |
Daniel Dunbar | 69e07a7 | 2009-06-17 21:33:37 +0000 | [diff] [blame] | 265 | global options |
| 266 | from optparse import OptionParser |
| 267 | parser = OptionParser("usage: %prog [options] {tests}") |
| 268 | parser.add_option("", "--clang", dest="clang", |
| 269 | help="Program to use as \"clang\"", |
| 270 | action="store", default=None) |
| 271 | parser.add_option("", "--clang-cc", dest="clangcc", |
| 272 | help="Program to use as \"clang-cc\"", |
| 273 | action="store", default=None) |
Daniel Dunbar | 69e07a7 | 2009-06-17 21:33:37 +0000 | [diff] [blame] | 274 | (opts, args) = parser.parse_args() |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 275 | |
Daniel Dunbar | 69e07a7 | 2009-06-17 21:33:37 +0000 | [diff] [blame] | 276 | if not args: |
| 277 | parser.error('No tests specified') |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 278 | |
Daniel Dunbar | 69e07a7 | 2009-06-17 21:33:37 +0000 | [diff] [blame] | 279 | if opts.clang is None: |
| 280 | opts.clang = inferClang() |
| 281 | if opts.clangcc is None: |
| 282 | opts.clangcc = inferClangCC(opts.clang) |
| 283 | |
| 284 | for path in args: |
Daniel Dunbar | a957d99 | 2009-07-25 14:46:05 +0000 | [diff] [blame] | 285 | base = getTestOutputBase('Output', path) + '.out' |
Daniel Dunbar | 69e07a7 | 2009-06-17 21:33:37 +0000 | [diff] [blame] | 286 | |
Daniel Dunbar | a957d99 | 2009-07-25 14:46:05 +0000 | [diff] [blame] | 287 | status,output = runOneTest(path, base, opts.clang, opts.clangcc) |
| 288 | print '%s: %s' % (TestStatus.getName(status).upper(), path) |
| 289 | if status == TestStatus.Fail or status == TestStatus.XPass: |
| 290 | print "%s TEST '%s' FAILED %s" % ('*'*20, path, '*'*20) |
| 291 | sys.stdout.write(output) |
| 292 | print "*" * 20 |
| 293 | sys.exit(1) |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 294 | |
Daniel Dunbar | a957d99 | 2009-07-25 14:46:05 +0000 | [diff] [blame] | 295 | sys.exit(0) |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 296 | |
| 297 | if __name__=='__main__': |
| 298 | main() |