Daniel Dunbar | 3bc6a98 | 2013-02-12 19:28:51 +0000 | [diff] [blame] | 1 | # -*- Python -*- vim: set syntax=python tabstop=4 expandtab cc=80: |
Daniel Dunbar | f5eadcd | 2010-09-15 03:57:04 +0000 | [diff] [blame] | 2 | |
| 3 | # Configuration file for the 'lit' test runner. |
| 4 | |
| 5 | import os |
Michael J. Spencer | 626916f | 2010-12-10 19:47:54 +0000 | [diff] [blame] | 6 | import sys |
Daniel Dunbar | f5eadcd | 2010-09-15 03:57:04 +0000 | [diff] [blame] | 7 | import platform |
| 8 | import tempfile |
| 9 | import signal |
| 10 | import subprocess |
Howard Hinnant | b4ebb0e | 2013-01-14 17:12:54 +0000 | [diff] [blame] | 11 | import errno |
| 12 | import time |
Daniel Dunbar | 3bc6a98 | 2013-02-12 19:28:51 +0000 | [diff] [blame] | 13 | import shlex |
Daniel Dunbar | f5eadcd | 2010-09-15 03:57:04 +0000 | [diff] [blame] | 14 | |
Daniel Dunbar | 81d1ef7 | 2013-02-05 21:03:25 +0000 | [diff] [blame] | 15 | # FIXME: For now, this is cribbed from lit.TestRunner, to avoid introducing a |
| 16 | # dependency there. What we more ideally would like to do is lift the "xfail" |
| 17 | # and "requires" handling to be a core lit framework feature. |
| 18 | def isExpectedFail(test, xfails): |
| 19 | # Check if any of the xfails match an available feature or the target. |
| 20 | for item in xfails: |
| 21 | # If this is the wildcard, it always fails. |
| 22 | if item == '*': |
| 23 | return True |
| 24 | |
Daniel Dunbar | 5f4841f | 2013-02-06 00:04:52 +0000 | [diff] [blame] | 25 | # If this is a part of any of the features, it fails. |
| 26 | for feature in test.config.available_features: |
| 27 | if item in feature: |
| 28 | return True |
Daniel Dunbar | 81d1ef7 | 2013-02-05 21:03:25 +0000 | [diff] [blame] | 29 | |
| 30 | # If this is a part of the target triple, it fails. |
| 31 | if item in test.suite.config.target_triple: |
| 32 | return True |
| 33 | |
| 34 | return False |
| 35 | |
Daniel Dunbar | f5eadcd | 2010-09-15 03:57:04 +0000 | [diff] [blame] | 36 | class LibcxxTestFormat(lit.formats.FileBasedTest): |
| 37 | """ |
| 38 | Custom test format handler for use with the test format use by libc++. |
| 39 | |
| 40 | Tests fall into two categories: |
| 41 | FOO.pass.cpp - Executable test which should compile, run, and exit with |
| 42 | code 0. |
| 43 | FOO.fail.cpp - Negative test case which is expected to fail compilation. |
| 44 | """ |
| 45 | |
Daniel Dunbar | cccf255 | 2013-02-05 18:03:49 +0000 | [diff] [blame] | 46 | def __init__(self, cxx_under_test, cpp_flags, ld_flags, exec_env): |
Daniel Dunbar | 7e0c57b | 2010-09-15 04:11:29 +0000 | [diff] [blame] | 47 | self.cxx_under_test = cxx_under_test |
Daniel Dunbar | 611581b | 2010-09-15 04:31:58 +0000 | [diff] [blame] | 48 | self.cpp_flags = list(cpp_flags) |
| 49 | self.ld_flags = list(ld_flags) |
Daniel Dunbar | cccf255 | 2013-02-05 18:03:49 +0000 | [diff] [blame] | 50 | self.exec_env = dict(exec_env) |
Daniel Dunbar | f5eadcd | 2010-09-15 03:57:04 +0000 | [diff] [blame] | 51 | |
Howard Hinnant | b4ebb0e | 2013-01-14 17:12:54 +0000 | [diff] [blame] | 52 | def execute_command(self, command, in_dir=None): |
| 53 | kwargs = { |
| 54 | 'stdin' :subprocess.PIPE, |
| 55 | 'stdout':subprocess.PIPE, |
| 56 | 'stderr':subprocess.PIPE, |
| 57 | } |
| 58 | if in_dir: |
| 59 | kwargs['cwd'] = in_dir |
| 60 | p = subprocess.Popen(command, **kwargs) |
Daniel Dunbar | f5eadcd | 2010-09-15 03:57:04 +0000 | [diff] [blame] | 61 | out,err = p.communicate() |
| 62 | exitCode = p.wait() |
| 63 | |
| 64 | # Detect Ctrl-C in subprocess. |
| 65 | if exitCode == -signal.SIGINT: |
| 66 | raise KeyboardInterrupt |
| 67 | |
| 68 | return out, err, exitCode |
| 69 | |
| 70 | def execute(self, test, lit_config): |
Howard Hinnant | b4ebb0e | 2013-01-14 17:12:54 +0000 | [diff] [blame] | 71 | while True: |
| 72 | try: |
| 73 | return self._execute(test, lit_config) |
| 74 | except OSError, oe: |
| 75 | if oe.errno != errno.ETXTBSY: |
| 76 | raise |
| 77 | time.sleep(0.1) |
| 78 | |
| 79 | def _execute(self, test, lit_config): |
Daniel Dunbar | 81d1ef7 | 2013-02-05 21:03:25 +0000 | [diff] [blame] | 80 | # Extract test metadata from the test file. |
| 81 | xfails = [] |
| 82 | requires = [] |
| 83 | with open(test.getSourcePath()) as f: |
| 84 | for ln in f: |
| 85 | if 'XFAIL:' in ln: |
| 86 | items = ln[ln.index('XFAIL:') + 6:].split(',') |
| 87 | xfails.extend([s.strip() for s in items]) |
| 88 | elif 'REQUIRES:' in ln: |
| 89 | items = ln[ln.index('REQUIRES:') + 9:].split(',') |
| 90 | requires.extend([s.strip() for s in items]) |
| 91 | elif not ln.startswith("//") and ln.strip(): |
| 92 | # Stop at the first non-empty line that is not a C++ |
| 93 | # comment. |
| 94 | break |
| 95 | |
| 96 | # Check that we have the required features. |
| 97 | # |
| 98 | # FIXME: For now, this is cribbed from lit.TestRunner, to avoid |
| 99 | # introducing a dependency there. What we more ideally would like to do |
Daniel Dunbar | 3bc6a98 | 2013-02-12 19:28:51 +0000 | [diff] [blame] | 100 | # is lift the "xfail" and "requires" handling to be a core lit |
| 101 | # framework feature. |
Daniel Dunbar | 81d1ef7 | 2013-02-05 21:03:25 +0000 | [diff] [blame] | 102 | missing_required_features = [f for f in requires |
| 103 | if f not in test.config.available_features] |
| 104 | if missing_required_features: |
| 105 | return (lit.Test.UNSUPPORTED, |
| 106 | "Test requires the following features: %s" % ( |
| 107 | ', '.join(missing_required_features),)) |
| 108 | |
| 109 | # Determine if this test is an expected failure. |
| 110 | isXFail = isExpectedFail(test, xfails) |
| 111 | |
| 112 | # Evaluate the test. |
| 113 | result, report = self._evaluate_test(test, lit_config) |
| 114 | |
| 115 | # Convert the test result based on whether this is an expected failure. |
| 116 | if isXFail: |
| 117 | if result != lit.Test.FAIL: |
| 118 | report += "\n\nTest was expected to FAIL, but did not.\n" |
| 119 | result = lit.Test.XPASS |
| 120 | else: |
| 121 | result = lit.Test.XFAIL |
| 122 | |
| 123 | return result, report |
| 124 | |
| 125 | def _evaluate_test(self, test, lit_config): |
Daniel Dunbar | f5eadcd | 2010-09-15 03:57:04 +0000 | [diff] [blame] | 126 | name = test.path_in_suite[-1] |
| 127 | source_path = test.getSourcePath() |
Howard Hinnant | b4ebb0e | 2013-01-14 17:12:54 +0000 | [diff] [blame] | 128 | source_dir = os.path.dirname(source_path) |
Daniel Dunbar | f5eadcd | 2010-09-15 03:57:04 +0000 | [diff] [blame] | 129 | |
| 130 | # Check what kind of test this is. |
| 131 | assert name.endswith('.pass.cpp') or name.endswith('.fail.cpp') |
| 132 | expected_compile_fail = name.endswith('.fail.cpp') |
| 133 | |
| 134 | # If this is a compile (failure) test, build it and check for failure. |
| 135 | if expected_compile_fail: |
| 136 | cmd = [self.cxx_under_test, '-c', |
Daniel Dunbar | 611581b | 2010-09-15 04:31:58 +0000 | [diff] [blame] | 137 | '-o', '/dev/null', source_path] + self.cpp_flags |
Daniel Dunbar | f5eadcd | 2010-09-15 03:57:04 +0000 | [diff] [blame] | 138 | out, err, exitCode = self.execute_command(cmd) |
| 139 | if exitCode == 1: |
| 140 | return lit.Test.PASS, "" |
| 141 | else: |
| 142 | report = """Command: %s\n""" % ' '.join(["'%s'" % a |
| 143 | for a in cmd]) |
| 144 | report += """Exit Code: %d\n""" % exitCode |
| 145 | if out: |
| 146 | report += """Standard Output:\n--\n%s--""" % out |
| 147 | if err: |
| 148 | report += """Standard Error:\n--\n%s--""" % err |
| 149 | report += "\n\nExpected compilation to fail!" |
Daniel Dunbar | 611581b | 2010-09-15 04:31:58 +0000 | [diff] [blame] | 150 | return lit.Test.FAIL, report |
Daniel Dunbar | f5eadcd | 2010-09-15 03:57:04 +0000 | [diff] [blame] | 151 | else: |
| 152 | exec_file = tempfile.NamedTemporaryFile(suffix="exe", delete=False) |
| 153 | exec_path = exec_file.name |
| 154 | exec_file.close() |
| 155 | |
| 156 | try: |
Michael J. Spencer | 626916f | 2010-12-10 19:47:54 +0000 | [diff] [blame] | 157 | compile_cmd = [self.cxx_under_test, '-o', exec_path, |
Daniel Dunbar | 611581b | 2010-09-15 04:31:58 +0000 | [diff] [blame] | 158 | source_path] + self.cpp_flags + self.ld_flags |
Michael J. Spencer | 626916f | 2010-12-10 19:47:54 +0000 | [diff] [blame] | 159 | cmd = compile_cmd |
Daniel Dunbar | f5eadcd | 2010-09-15 03:57:04 +0000 | [diff] [blame] | 160 | out, err, exitCode = self.execute_command(cmd) |
| 161 | if exitCode != 0: |
Daniel Dunbar | f5eadcd | 2010-09-15 03:57:04 +0000 | [diff] [blame] | 162 | report = """Command: %s\n""" % ' '.join(["'%s'" % a |
| 163 | for a in cmd]) |
| 164 | report += """Exit Code: %d\n""" % exitCode |
| 165 | if out: |
| 166 | report += """Standard Output:\n--\n%s--""" % out |
| 167 | if err: |
| 168 | report += """Standard Error:\n--\n%s--""" % err |
| 169 | report += "\n\nCompilation failed unexpectedly!" |
| 170 | return lit.Test.FAIL, report |
| 171 | |
Daniel Dunbar | cccf255 | 2013-02-05 18:03:49 +0000 | [diff] [blame] | 172 | cmd = [] |
| 173 | if self.exec_env: |
| 174 | cmd.append('env') |
| 175 | cmd.extend('%s=%s' % (name, value) |
| 176 | for name,value in self.exec_env.items()) |
| 177 | cmd.append(exec_path) |
Howard Hinnant | 63b2f4f | 2012-08-02 18:36:47 +0000 | [diff] [blame] | 178 | if lit_config.useValgrind: |
| 179 | cmd = lit_config.valgrindArgs + cmd |
Howard Hinnant | b4ebb0e | 2013-01-14 17:12:54 +0000 | [diff] [blame] | 180 | out, err, exitCode = self.execute_command(cmd, source_dir) |
Daniel Dunbar | f5eadcd | 2010-09-15 03:57:04 +0000 | [diff] [blame] | 181 | if exitCode != 0: |
Daniel Dunbar | 3bc6a98 | 2013-02-12 19:28:51 +0000 | [diff] [blame] | 182 | report = """Compiled With: %s\n""" % \ |
| 183 | ' '.join(["'%s'" % a for a in compile_cmd]) |
| 184 | report += """Command: %s\n""" % \ |
| 185 | ' '.join(["'%s'" % a for a in cmd]) |
Daniel Dunbar | f5eadcd | 2010-09-15 03:57:04 +0000 | [diff] [blame] | 186 | report += """Exit Code: %d\n""" % exitCode |
| 187 | if out: |
| 188 | report += """Standard Output:\n--\n%s--""" % out |
| 189 | if err: |
| 190 | report += """Standard Error:\n--\n%s--""" % err |
| 191 | report += "\n\nCompiled test failed unexpectedly!" |
| 192 | return lit.Test.FAIL, report |
| 193 | finally: |
| 194 | try: |
| 195 | os.remove(exec_path) |
| 196 | except: |
| 197 | pass |
| 198 | return lit.Test.PASS, "" |
| 199 | |
| 200 | # name: The name of this test suite. |
| 201 | config.name = 'libc++' |
| 202 | |
| 203 | # suffixes: A list of file extensions to treat as test files. |
| 204 | config.suffixes = ['.cpp'] |
| 205 | |
| 206 | # test_source_root: The root path where tests are located. |
| 207 | config.test_source_root = os.path.dirname(__file__) |
| 208 | |
Daniel Dunbar | af01e70 | 2012-11-27 23:56:28 +0000 | [diff] [blame] | 209 | # Gather various compiler parameters. |
Daniel Dunbar | f5eadcd | 2010-09-15 03:57:04 +0000 | [diff] [blame] | 210 | cxx_under_test = lit.params.get('cxx_under_test', None) |
| 211 | if cxx_under_test is None: |
Michael J. Spencer | 626916f | 2010-12-10 19:47:54 +0000 | [diff] [blame] | 212 | cxx_under_test = getattr(config, 'cxx_under_test', None) |
Daniel Dunbar | 7fa0ca7 | 2013-02-06 20:24:23 +0000 | [diff] [blame] | 213 | |
| 214 | # If no specific cxx_under_test was given, attempt to infer it as clang++. |
| 215 | clangxx = lit.util.which('clang++', config.environment['PATH']) |
| 216 | if clangxx is not None: |
| 217 | cxx_under_test = clangxx |
| 218 | lit.note("inferred cxx_under_test as: %r" % (cxx_under_test,)) |
| 219 | if cxx_under_test is None: |
| 220 | lit.fatal('must specify user parameter cxx_under_test ' |
| 221 | '(e.g., --param=cxx_under_test=clang++)') |
Michael J. Spencer | 626916f | 2010-12-10 19:47:54 +0000 | [diff] [blame] | 222 | |
Daniel Dunbar | af01e70 | 2012-11-27 23:56:28 +0000 | [diff] [blame] | 223 | libcxx_src_root = lit.params.get('libcxx_src_root', None) |
| 224 | if libcxx_src_root is None: |
| 225 | libcxx_src_root = getattr(config, 'libcxx_src_root', None) |
| 226 | if libcxx_src_root is None: |
| 227 | libcxx_src_root = os.path.dirname(config.test_source_root) |
Michael J. Spencer | 626916f | 2010-12-10 19:47:54 +0000 | [diff] [blame] | 228 | |
Daniel Dunbar | af01e70 | 2012-11-27 23:56:28 +0000 | [diff] [blame] | 229 | libcxx_obj_root = lit.params.get('libcxx_obj_root', None) |
| 230 | if libcxx_obj_root is None: |
| 231 | libcxx_obj_root = getattr(config, 'libcxx_obj_root', None) |
| 232 | if libcxx_obj_root is None: |
| 233 | libcxx_obj_root = libcxx_src_root |
| 234 | |
| 235 | cxx_has_stdcxx0x_flag_str = lit.params.get('cxx_has_stdcxx0x_flag', None) |
| 236 | if cxx_has_stdcxx0x_flag_str is not None: |
Daniel Dunbar | cccf255 | 2013-02-05 18:03:49 +0000 | [diff] [blame] | 237 | if cxx_has_stdcxx0x_flag_str.lower() in ('1', 'true'): |
Daniel Dunbar | af01e70 | 2012-11-27 23:56:28 +0000 | [diff] [blame] | 238 | cxx_has_stdcxx0x_flag = True |
Daniel Dunbar | cccf255 | 2013-02-05 18:03:49 +0000 | [diff] [blame] | 239 | elif cxx_has_stdcxx0x_flag_str.lower() in ('', '0', 'false'): |
Daniel Dunbar | af01e70 | 2012-11-27 23:56:28 +0000 | [diff] [blame] | 240 | cxx_has_stdcxx0x_flag = False |
| 241 | else: |
| 242 | lit.fatal('user parameter cxx_has_stdcxx0x_flag_str should be 0 or 1') |
Michael J. Spencer | 626916f | 2010-12-10 19:47:54 +0000 | [diff] [blame] | 243 | else: |
Daniel Dunbar | af01e70 | 2012-11-27 23:56:28 +0000 | [diff] [blame] | 244 | cxx_has_stdcxx0x_flag = getattr(config, 'cxx_has_stdcxx0x_flag', True) |
Michael J. Spencer | 626916f | 2010-12-10 19:47:54 +0000 | [diff] [blame] | 245 | |
Daniel Dunbar | cccf255 | 2013-02-05 18:03:49 +0000 | [diff] [blame] | 246 | # This test suite supports testing against either the system library or the |
| 247 | # locally built one; the former mode is useful for testing ABI compatibility |
Daniel Dunbar | 88dec1e | 2013-02-06 17:47:08 +0000 | [diff] [blame] | 248 | # between the current headers and a shipping dynamic library. |
Daniel Dunbar | cccf255 | 2013-02-05 18:03:49 +0000 | [diff] [blame] | 249 | use_system_lib_str = lit.params.get('use_system_lib', None) |
| 250 | if use_system_lib_str is not None: |
| 251 | if use_system_lib_str.lower() in ('1', 'true'): |
| 252 | use_system_lib = True |
| 253 | elif use_system_lib_str.lower() in ('', '0', 'false'): |
| 254 | use_system_lib = False |
| 255 | else: |
| 256 | lit.fatal('user parameter use_system_lib should be 0 or 1') |
| 257 | else: |
Daniel Dunbar | 88dec1e | 2013-02-06 17:47:08 +0000 | [diff] [blame] | 258 | # Default to testing against the locally built libc++ library. |
| 259 | use_system_lib = False |
Daniel Dunbar | 7fa0ca7 | 2013-02-06 20:24:23 +0000 | [diff] [blame] | 260 | lit.note("inferred use_system_lib as: %r" % (use_system_lib,)) |
Daniel Dunbar | cccf255 | 2013-02-05 18:03:49 +0000 | [diff] [blame] | 261 | |
Daniel Dunbar | 3bc6a98 | 2013-02-12 19:28:51 +0000 | [diff] [blame] | 262 | link_flags = [] |
| 263 | link_flags_str = lit.params.get('link_flags', None) |
| 264 | if link_flags_str is None: |
| 265 | link_flags_str = getattr(config, 'link_flags', None) |
| 266 | if link_flags_str is None: |
| 267 | if sys.platform == 'darwin': |
| 268 | link_flags += ['-lSystem'] |
| 269 | elif sys.platform == 'linux2': |
| 270 | link_flags += ['-lsupc++', '-lgcc_eh', '-lc', '-lm', '-lpthread', |
| 271 | '-lrt', '-lgcc_s'] |
| 272 | else: |
| 273 | lit.fatal("unrecognized system") |
| 274 | lit.note("inferred link_flags as: %r" % (link_flags,)) |
| 275 | if not link_flags_str is None: |
| 276 | link_flags += shlex.split(link_flags_str) |
| 277 | |
Chandler Carruth | e76496c | 2011-01-23 01:05:20 +0000 | [diff] [blame] | 278 | # Configure extra compiler flags. |
Daniel Dunbar | 3bc6a98 | 2013-02-12 19:28:51 +0000 | [diff] [blame] | 279 | include_paths = ['-I' + libcxx_src_root + '/include', |
| 280 | '-I' + libcxx_src_root + '/test/support'] |
Daniel Dunbar | af01e70 | 2012-11-27 23:56:28 +0000 | [diff] [blame] | 281 | library_paths = ['-L' + libcxx_obj_root + '/lib'] |
Chandler Carruth | e76496c | 2011-01-23 01:05:20 +0000 | [diff] [blame] | 282 | compile_flags = [] |
Daniel Dunbar | af01e70 | 2012-11-27 23:56:28 +0000 | [diff] [blame] | 283 | if cxx_has_stdcxx0x_flag: |
Daniel Dunbar | cccf255 | 2013-02-05 18:03:49 +0000 | [diff] [blame] | 284 | compile_flags += ['-std=c++0x'] |
Chandler Carruth | e76496c | 2011-01-23 01:05:20 +0000 | [diff] [blame] | 285 | |
Daniel Dunbar | 3bc6a98 | 2013-02-12 19:28:51 +0000 | [diff] [blame] | 286 | # Configure extra linker parameters. |
Daniel Dunbar | cccf255 | 2013-02-05 18:03:49 +0000 | [diff] [blame] | 287 | exec_env = {} |
Michael J. Spencer | 626916f | 2010-12-10 19:47:54 +0000 | [diff] [blame] | 288 | if sys.platform == 'darwin': |
Daniel Dunbar | cccf255 | 2013-02-05 18:03:49 +0000 | [diff] [blame] | 289 | if not use_system_lib: |
| 290 | exec_env['DYLD_LIBRARY_PATH'] = os.path.join(libcxx_obj_root, 'lib') |
| 291 | elif sys.platform == 'linux2': |
Daniel Dunbar | 6b8b992 | 2013-02-06 17:45:53 +0000 | [diff] [blame] | 292 | if not use_system_lib: |
Daniel Dunbar | 3bc6a98 | 2013-02-12 19:28:51 +0000 | [diff] [blame] | 293 | link_flags += ['-Wl,-R', libcxx_obj_root + '/lib'] |
| 294 | compile_flags += ['-D__STDC_FORMAT_MACROS', '-D__STDC_LIMIT_MACROS', |
| 295 | '-D__STDC_CONSTANT_MACROS'] |
Daniel Dunbar | cccf255 | 2013-02-05 18:03:49 +0000 | [diff] [blame] | 296 | else: |
| 297 | lit.fatal("unrecognized system") |
Michael J. Spencer | 626916f | 2010-12-10 19:47:54 +0000 | [diff] [blame] | 298 | |
Daniel Dunbar | af01e70 | 2012-11-27 23:56:28 +0000 | [diff] [blame] | 299 | config.test_format = LibcxxTestFormat( |
| 300 | cxx_under_test, |
| 301 | cpp_flags = ['-nostdinc++'] + compile_flags + include_paths, |
Daniel Dunbar | 3bc6a98 | 2013-02-12 19:28:51 +0000 | [diff] [blame] | 302 | ld_flags = ['-nodefaultlibs'] + library_paths + ['-lc++'] + link_flags, |
Daniel Dunbar | cccf255 | 2013-02-05 18:03:49 +0000 | [diff] [blame] | 303 | exec_env = exec_env) |
Daniel Dunbar | f5eadcd | 2010-09-15 03:57:04 +0000 | [diff] [blame] | 304 | |
Daniel Dunbar | 4cceb7a | 2013-02-05 22:28:03 +0000 | [diff] [blame] | 305 | # Get or infer the target triple. |
| 306 | config.target_triple = lit.params.get('target_triple', None) |
| 307 | # If no target triple was given, try to infer it from the compiler under test. |
| 308 | if config.target_triple is None: |
| 309 | config.target_triple = lit.util.capture( |
| 310 | [cxx_under_test, '-dumpmachine']).strip() |
Daniel Dunbar | 7fa0ca7 | 2013-02-06 20:24:23 +0000 | [diff] [blame] | 311 | lit.note("inferred target_triple as: %r" % (config.target_triple,)) |
Daniel Dunbar | a5b5196 | 2013-02-05 21:43:30 +0000 | [diff] [blame] | 312 | |
| 313 | # Write an "available feature" that combines the triple when use_system_lib is |
| 314 | # enabled. This is so that we can easily write XFAIL markers for tests that are |
| 315 | # known to fail with versions of libc++ as were shipped with a particular |
| 316 | # triple. |
| 317 | if use_system_lib: |
| 318 | config.available_features.add('with_system_lib=%s' % ( |
| 319 | config.target_triple,)) |