Daniel Dunbar | 69e07a7 | 2009-06-17 21:33:37 +0000 | [diff] [blame] | 1 | import os |
Daniel Dunbar | 8fe83f1 | 2009-07-25 09:42:24 +0000 | [diff] [blame] | 2 | import platform |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 3 | import re |
Daniel Dunbar | 69e07a7 | 2009-06-17 21:33:37 +0000 | [diff] [blame] | 4 | import signal |
| 5 | import subprocess |
| 6 | import sys |
| 7 | |
Daniel Dunbar | 1db467f | 2009-07-31 05:54:17 +0000 | [diff] [blame] | 8 | import Util |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 9 | |
Daniel Dunbar | 8fe83f1 | 2009-07-25 09:42:24 +0000 | [diff] [blame] | 10 | kSystemName = platform.system() |
| 11 | |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 12 | class TestStatus: |
| 13 | Pass = 0 |
| 14 | XFail = 1 |
| 15 | Fail = 2 |
| 16 | XPass = 3 |
Daniel Dunbar | 8bf0ccd | 2009-07-25 12:47:38 +0000 | [diff] [blame] | 17 | Invalid = 4 |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 18 | |
Daniel Dunbar | 8bf0ccd | 2009-07-25 12:47:38 +0000 | [diff] [blame] | 19 | kNames = ['Pass','XFail','Fail','XPass','Invalid'] |
Daniel Dunbar | fbbb1e7 | 2009-07-02 23:58:07 +0000 | [diff] [blame] | 20 | @staticmethod |
Daniel Dunbar | 8afede2 | 2009-07-02 23:56:37 +0000 | [diff] [blame] | 21 | def getName(code): |
| 22 | return TestStatus.kNames[code] |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 23 | |
Daniel Dunbar | 5928ccd | 2009-08-01 04:06:02 +0000 | [diff] [blame] | 24 | def executeScript(cfg, script, commands, cwd): |
Daniel Dunbar | 10aebbb | 2009-07-25 15:26:08 +0000 | [diff] [blame] | 25 | # Write script file |
| 26 | f = open(script,'w') |
| 27 | if kSystemName == 'Windows': |
| 28 | f.write('\nif %ERRORLEVEL% NEQ 0 EXIT\n'.join(commands)) |
| 29 | else: |
| 30 | f.write(' &&\n'.join(commands)) |
| 31 | f.write('\n') |
| 32 | f.close() |
| 33 | |
| 34 | if kSystemName == 'Windows': |
| 35 | command = ['cmd','/c', script] |
| 36 | else: |
| 37 | command = ['/bin/sh', script] |
Daniel Dunbar | 5928ccd | 2009-08-01 04:06:02 +0000 | [diff] [blame] | 38 | if cfg.useValgrind: |
Daniel Dunbar | 9a676b7 | 2009-07-29 02:57:25 +0000 | [diff] [blame] | 39 | # FIXME: Running valgrind on sh is overkill. We probably could just |
Daniel Dunbar | 5928ccd | 2009-08-01 04:06:02 +0000 | [diff] [blame] | 40 | # run on clang with no real loss. |
Daniel Dunbar | 9a676b7 | 2009-07-29 02:57:25 +0000 | [diff] [blame] | 41 | command = ['valgrind', '-q', |
| 42 | '--tool=memcheck', '--leak-check=no', '--trace-children=yes', |
| 43 | '--error-exitcode=123'] + command |
Daniel Dunbar | 10aebbb | 2009-07-25 15:26:08 +0000 | [diff] [blame] | 44 | |
| 45 | p = subprocess.Popen(command, cwd=cwd, |
| 46 | stdin=subprocess.PIPE, |
| 47 | stdout=subprocess.PIPE, |
| 48 | stderr=subprocess.PIPE, |
Daniel Dunbar | 1db467f | 2009-07-31 05:54:17 +0000 | [diff] [blame] | 49 | env=cfg.environment) |
Daniel Dunbar | 10aebbb | 2009-07-25 15:26:08 +0000 | [diff] [blame] | 50 | out,err = p.communicate() |
| 51 | exitCode = p.wait() |
| 52 | |
| 53 | # Detect Ctrl-C in subprocess. |
| 54 | if exitCode == -signal.SIGINT: |
| 55 | raise KeyboardInterrupt |
| 56 | |
| 57 | return out, err, exitCode |
| 58 | |
Daniel Dunbar | a957d99 | 2009-07-25 14:46:05 +0000 | [diff] [blame] | 59 | import StringIO |
Daniel Dunbar | 5928ccd | 2009-08-01 04:06:02 +0000 | [diff] [blame] | 60 | def runOneTest(cfg, testPath, tmpBase): |
Daniel Dunbar | a957d99 | 2009-07-25 14:46:05 +0000 | [diff] [blame] | 61 | # Make paths absolute. |
| 62 | tmpBase = os.path.abspath(tmpBase) |
| 63 | testPath = os.path.abspath(testPath) |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 64 | |
| 65 | # Create the output directory if it does not already exist. |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 66 | |
Daniel Dunbar | 1db467f | 2009-07-31 05:54:17 +0000 | [diff] [blame] | 67 | Util.mkdir_p(os.path.dirname(tmpBase)) |
Daniel Dunbar | a957d99 | 2009-07-25 14:46:05 +0000 | [diff] [blame] | 68 | script = tmpBase + '.script' |
Daniel Dunbar | 8fe83f1 | 2009-07-25 09:42:24 +0000 | [diff] [blame] | 69 | if kSystemName == 'Windows': |
Daniel Dunbar | a957d99 | 2009-07-25 14:46:05 +0000 | [diff] [blame] | 70 | script += '.bat' |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 71 | |
Daniel Dunbar | a957d99 | 2009-07-25 14:46:05 +0000 | [diff] [blame] | 72 | substitutions = [('%s', testPath), |
| 73 | ('%S', os.path.dirname(testPath)), |
| 74 | ('%t', tmpBase + '.tmp'), |
Daniel Dunbar | 5928ccd | 2009-08-01 04:06:02 +0000 | [diff] [blame] | 75 | (' clang ', ' ' + cfg.clang + ' '), |
| 76 | (' clang-cc ', ' ' + cfg.clangcc + ' ')] |
Daniel Dunbar | 025f80d | 2009-07-25 11:27:37 +0000 | [diff] [blame] | 77 | |
| 78 | # Collect the test lines from the script. |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 79 | scriptLines = [] |
| 80 | xfailLines = [] |
Daniel Dunbar | a957d99 | 2009-07-25 14:46:05 +0000 | [diff] [blame] | 81 | for ln in open(testPath): |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 82 | if 'RUN:' in ln: |
Daniel Dunbar | 025f80d | 2009-07-25 11:27:37 +0000 | [diff] [blame] | 83 | # Isolate the command to run. |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 84 | index = ln.index('RUN:') |
| 85 | ln = ln[index+4:] |
Daniel Dunbar | 025f80d | 2009-07-25 11:27:37 +0000 | [diff] [blame] | 86 | |
Daniel Dunbar | 322f789 | 2009-07-25 12:23:35 +0000 | [diff] [blame] | 87 | # Strip trailing newline. |
| 88 | scriptLines.append(ln) |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 89 | elif 'XFAIL' in ln: |
| 90 | xfailLines.append(ln) |
Daniel Dunbar | 025f80d | 2009-07-25 11:27:37 +0000 | [diff] [blame] | 91 | |
| 92 | # FIXME: Support something like END, in case we need to process large |
| 93 | # files. |
Daniel Dunbar | a957d99 | 2009-07-25 14:46:05 +0000 | [diff] [blame] | 94 | |
| 95 | # Verify the script contains a run line. |
| 96 | if not scriptLines: |
| 97 | return (TestStatus.Fail, "Test has no run line!") |
Daniel Dunbar | 322f789 | 2009-07-25 12:23:35 +0000 | [diff] [blame] | 98 | |
| 99 | # Apply substitutions to the script. |
| 100 | def processLine(ln): |
| 101 | # Apply substitutions |
| 102 | for a,b in substitutions: |
| 103 | ln = ln.replace(a,b) |
| 104 | |
Daniel Dunbar | 322f789 | 2009-07-25 12:23:35 +0000 | [diff] [blame] | 105 | # Strip the trailing newline and any extra whitespace. |
| 106 | return ln.strip() |
| 107 | scriptLines = map(processLine, scriptLines) |
Daniel Dunbar | 025f80d | 2009-07-25 11:27:37 +0000 | [diff] [blame] | 108 | |
| 109 | # Validate interior lines for '&&', a lovely historical artifact. |
| 110 | for i in range(len(scriptLines) - 1): |
| 111 | ln = scriptLines[i] |
| 112 | |
| 113 | if not ln.endswith('&&'): |
Daniel Dunbar | a957d99 | 2009-07-25 14:46:05 +0000 | [diff] [blame] | 114 | return (TestStatus.Fail, |
Daniel Dunbar | 6779647 | 2009-07-27 19:01:13 +0000 | [diff] [blame] | 115 | ("MISSING \'&&\': %s\n" + |
| 116 | "FOLLOWED BY : %s\n") % (ln, scriptLines[i + 1])) |
Daniel Dunbar | 025f80d | 2009-07-25 11:27:37 +0000 | [diff] [blame] | 117 | |
| 118 | # Strip off '&&' |
| 119 | scriptLines[i] = ln[:-2] |
Daniel Dunbar | 025f80d | 2009-07-25 11:27:37 +0000 | [diff] [blame] | 120 | |
Daniel Dunbar | 1db467f | 2009-07-31 05:54:17 +0000 | [diff] [blame] | 121 | out, err, exitCode = executeScript(cfg, script, scriptLines, |
Daniel Dunbar | 5928ccd | 2009-08-01 04:06:02 +0000 | [diff] [blame] | 122 | os.path.dirname(testPath)) |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 123 | if xfailLines: |
Daniel Dunbar | a957d99 | 2009-07-25 14:46:05 +0000 | [diff] [blame] | 124 | ok = exitCode != 0 |
| 125 | status = (TestStatus.XPass, TestStatus.XFail)[ok] |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 126 | else: |
Daniel Dunbar | a957d99 | 2009-07-25 14:46:05 +0000 | [diff] [blame] | 127 | ok = exitCode == 0 |
| 128 | status = (TestStatus.Fail, TestStatus.Pass)[ok] |
| 129 | |
| 130 | if ok: |
| 131 | return (status,'') |
| 132 | |
| 133 | output = StringIO.StringIO() |
| 134 | print >>output, "Script:" |
| 135 | print >>output, "--" |
| 136 | print >>output, '\n'.join(scriptLines) |
| 137 | print >>output, "--" |
| 138 | print >>output, "Exit Code: %r" % exitCode |
| 139 | print >>output, "Command Output (stdout):" |
| 140 | print >>output, "--" |
| 141 | output.write(out) |
| 142 | print >>output, "--" |
| 143 | print >>output, "Command Output (stderr):" |
| 144 | print >>output, "--" |
| 145 | output.write(err) |
| 146 | print >>output, "--" |
| 147 | return (status, output.getvalue()) |
Daniel Dunbar | 69e07a7 | 2009-06-17 21:33:37 +0000 | [diff] [blame] | 148 | |
| 149 | def capture(args): |
Daniel Dunbar | 8fe83f1 | 2009-07-25 09:42:24 +0000 | [diff] [blame] | 150 | p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
Daniel Dunbar | 69e07a7 | 2009-06-17 21:33:37 +0000 | [diff] [blame] | 151 | out,_ = p.communicate() |
| 152 | return out |
| 153 | |
Daniel Dunbar | 1db467f | 2009-07-31 05:54:17 +0000 | [diff] [blame] | 154 | def inferClang(cfg): |
Daniel Dunbar | 69e07a7 | 2009-06-17 21:33:37 +0000 | [diff] [blame] | 155 | # Determine which clang to use. |
| 156 | clang = os.getenv('CLANG') |
| 157 | |
| 158 | # If the user set clang in the environment, definitely use that and don't |
| 159 | # try to validate. |
| 160 | if clang: |
| 161 | return clang |
| 162 | |
| 163 | # Otherwise look in the path. |
Daniel Dunbar | 1db467f | 2009-07-31 05:54:17 +0000 | [diff] [blame] | 164 | clang = Util.which('clang', cfg.environment['PATH']) |
Daniel Dunbar | 69e07a7 | 2009-06-17 21:33:37 +0000 | [diff] [blame] | 165 | |
| 166 | if not clang: |
| 167 | print >>sys.stderr, "error: couldn't find 'clang' program, try setting CLANG in your environment" |
| 168 | sys.exit(1) |
| 169 | |
| 170 | return clang |
| 171 | |
Daniel Dunbar | 1db467f | 2009-07-31 05:54:17 +0000 | [diff] [blame] | 172 | def inferClangCC(cfg, clang): |
Daniel Dunbar | 69e07a7 | 2009-06-17 21:33:37 +0000 | [diff] [blame] | 173 | clangcc = os.getenv('CLANGCC') |
| 174 | |
| 175 | # If the user set clang in the environment, definitely use that and don't |
| 176 | # try to validate. |
| 177 | if clangcc: |
| 178 | return clangcc |
| 179 | |
| 180 | # Otherwise try adding -cc since we expect to be looking in a build |
| 181 | # directory. |
Daniel Dunbar | 8fe83f1 | 2009-07-25 09:42:24 +0000 | [diff] [blame] | 182 | if clang.endswith('.exe'): |
| 183 | clangccName = clang[:-4] + '-cc.exe' |
| 184 | else: |
| 185 | clangccName = clang + '-cc' |
Daniel Dunbar | 1db467f | 2009-07-31 05:54:17 +0000 | [diff] [blame] | 186 | clangcc = Util.which(clangccName, cfg.environment['PATH']) |
Daniel Dunbar | 69e07a7 | 2009-06-17 21:33:37 +0000 | [diff] [blame] | 187 | if not clangcc: |
| 188 | # Otherwise ask clang. |
| 189 | res = capture([clang, '-print-prog-name=clang-cc']) |
| 190 | res = res.strip() |
| 191 | if res and os.path.exists(res): |
| 192 | clangcc = res |
| 193 | |
| 194 | if not clangcc: |
| 195 | print >>sys.stderr, "error: couldn't find 'clang-cc' program, try setting CLANGCC in your environment" |
| 196 | sys.exit(1) |
| 197 | |
| 198 | return clangcc |
Daniel Dunbar | 6fc0bdf | 2009-03-06 22:20:40 +0000 | [diff] [blame] | 199 | |
Daniel Dunbar | df08489 | 2009-07-25 09:53:43 +0000 | [diff] [blame] | 200 | def getTestOutputBase(dir, testpath): |
Daniel Dunbar | fecdd00 | 2009-07-25 12:05:55 +0000 | [diff] [blame] | 201 | """getTestOutputBase(dir, testpath) - Get the full path for temporary files |
Daniel Dunbar | df08489 | 2009-07-25 09:53:43 +0000 | [diff] [blame] | 202 | corresponding to the given test path.""" |
| 203 | |
| 204 | # Form the output base out of the test parent directory name and the test |
| 205 | # name. FIXME: Find a better way to organize test results. |
| 206 | return os.path.join(dir, |
| 207 | os.path.basename(os.path.dirname(testpath)), |
| 208 | os.path.basename(testpath)) |