blob: 37f6357b9141ac7d66f794ec1af213546aa4f9b6 [file] [log] [blame]
Daniel Dunbar62b94392013-02-12 19:28:51 +00001# -*- Python -*- vim: set syntax=python tabstop=4 expandtab cc=80:
Daniel Dunbar42ea4632010-09-15 03:57:04 +00002
3# Configuration file for the 'lit' test runner.
4
Daniel Dunbarb6b3e502013-08-30 19:52:12 +00005import errno
Dan Alberta85b27f2014-08-04 18:44:48 +00006import locale
Daniel Dunbar42ea4632010-09-15 03:57:04 +00007import os
8import platform
Daniel Dunbarb6b3e502013-08-30 19:52:12 +00009import re
10import shlex
Daniel Dunbar42ea4632010-09-15 03:57:04 +000011import signal
12import subprocess
Daniel Dunbarb6b3e502013-08-30 19:52:12 +000013import sys
14import tempfile
Howard Hinnant3778f272013-01-14 17:12:54 +000015import time
Daniel Dunbar42ea4632010-09-15 03:57:04 +000016
Daniel Dunbar4a381292013-08-09 14:44:11 +000017import lit.Test
18import lit.formats
19import lit.util
20
Daniel Dunbar42ea4632010-09-15 03:57:04 +000021class LibcxxTestFormat(lit.formats.FileBasedTest):
22 """
23 Custom test format handler for use with the test format use by libc++.
24
25 Tests fall into two categories:
26 FOO.pass.cpp - Executable test which should compile, run, and exit with
27 code 0.
28 FOO.fail.cpp - Negative test case which is expected to fail compilation.
29 """
30
Justin Bogner33a2a2e2014-09-03 04:32:08 +000031 def __init__(self, cxx_under_test, use_verify_for_fail,
32 cpp_flags, ld_flags, exec_env):
Daniel Dunbarbc9a8482010-09-15 04:11:29 +000033 self.cxx_under_test = cxx_under_test
Justin Bogner33a2a2e2014-09-03 04:32:08 +000034 self.use_verify_for_fail = use_verify_for_fail
Daniel Dunbar5f09d9e02010-09-15 04:31:58 +000035 self.cpp_flags = list(cpp_flags)
36 self.ld_flags = list(ld_flags)
Daniel Dunbar84958712013-02-05 18:03:49 +000037 self.exec_env = dict(exec_env)
Daniel Dunbar42ea4632010-09-15 03:57:04 +000038
Howard Hinnant3778f272013-01-14 17:12:54 +000039 def execute_command(self, command, in_dir=None):
40 kwargs = {
41 'stdin' :subprocess.PIPE,
42 'stdout':subprocess.PIPE,
43 'stderr':subprocess.PIPE,
44 }
45 if in_dir:
46 kwargs['cwd'] = in_dir
47 p = subprocess.Popen(command, **kwargs)
Daniel Dunbar42ea4632010-09-15 03:57:04 +000048 out,err = p.communicate()
49 exitCode = p.wait()
50
51 # Detect Ctrl-C in subprocess.
52 if exitCode == -signal.SIGINT:
53 raise KeyboardInterrupt
54
55 return out, err, exitCode
56
57 def execute(self, test, lit_config):
Howard Hinnant3778f272013-01-14 17:12:54 +000058 while True:
59 try:
60 return self._execute(test, lit_config)
61 except OSError, oe:
62 if oe.errno != errno.ETXTBSY:
63 raise
64 time.sleep(0.1)
65
66 def _execute(self, test, lit_config):
Daniel Dunbarf51f0312013-02-05 21:03:25 +000067 # Extract test metadata from the test file.
Daniel Dunbarf51f0312013-02-05 21:03:25 +000068 requires = []
Eric Fiselier339bdc32014-08-18 06:43:06 +000069 unsupported = []
Justin Bogner2cfd1fe2014-09-03 06:01:52 +000070 use_verify = False
Daniel Dunbarf51f0312013-02-05 21:03:25 +000071 with open(test.getSourcePath()) as f:
72 for ln in f:
73 if 'XFAIL:' in ln:
74 items = ln[ln.index('XFAIL:') + 6:].split(',')
Daniel Dunbar019c5902013-08-21 23:06:32 +000075 test.xfails.extend([s.strip() for s in items])
Daniel Dunbarf51f0312013-02-05 21:03:25 +000076 elif 'REQUIRES:' in ln:
77 items = ln[ln.index('REQUIRES:') + 9:].split(',')
78 requires.extend([s.strip() for s in items])
Eric Fiselier339bdc32014-08-18 06:43:06 +000079 elif 'UNSUPPORTED:' in ln:
80 items = ln[ln.index('UNSUPPORTED:') + 12:].split(',')
81 unsupported.extend([s.strip() for s in items])
Justin Bogner2cfd1fe2014-09-03 06:01:52 +000082 elif 'USE_VERIFY' in ln and self.use_verify_for_fail:
83 use_verify = True
Eric Fiselier993dfb12014-07-31 22:56:52 +000084 elif not ln.strip().startswith("//") and ln.strip():
Daniel Dunbarf51f0312013-02-05 21:03:25 +000085 # Stop at the first non-empty line that is not a C++
86 # comment.
87 break
88
89 # Check that we have the required features.
90 #
91 # FIXME: For now, this is cribbed from lit.TestRunner, to avoid
92 # introducing a dependency there. What we more ideally would like to do
Daniel Dunbar019c5902013-08-21 23:06:32 +000093 # is lift the "requires" handling to be a core lit framework feature.
Daniel Dunbarf51f0312013-02-05 21:03:25 +000094 missing_required_features = [f for f in requires
95 if f not in test.config.available_features]
96 if missing_required_features:
97 return (lit.Test.UNSUPPORTED,
98 "Test requires the following features: %s" % (
99 ', '.join(missing_required_features),))
100
Eric Fiselier339bdc32014-08-18 06:43:06 +0000101 unsupported_features = [f for f in unsupported
102 if f in test.config.available_features]
103 if unsupported_features:
104 return (lit.Test.UNSUPPORTED,
105 "Test is unsupported with the following features: %s" % (
106 ', '.join(unsupported_features),))
107
Daniel Dunbarf51f0312013-02-05 21:03:25 +0000108 # Evaluate the test.
Justin Bogner2cfd1fe2014-09-03 06:01:52 +0000109 return self._evaluate_test(test, use_verify, lit_config)
Daniel Dunbarf51f0312013-02-05 21:03:25 +0000110
Justin Bogner2cfd1fe2014-09-03 06:01:52 +0000111 def _evaluate_test(self, test, use_verify, lit_config):
Daniel Dunbar42ea4632010-09-15 03:57:04 +0000112 name = test.path_in_suite[-1]
113 source_path = test.getSourcePath()
Howard Hinnant3778f272013-01-14 17:12:54 +0000114 source_dir = os.path.dirname(source_path)
Daniel Dunbar42ea4632010-09-15 03:57:04 +0000115
116 # Check what kind of test this is.
117 assert name.endswith('.pass.cpp') or name.endswith('.fail.cpp')
118 expected_compile_fail = name.endswith('.fail.cpp')
119
120 # If this is a compile (failure) test, build it and check for failure.
121 if expected_compile_fail:
122 cmd = [self.cxx_under_test, '-c',
Daniel Dunbar5f09d9e02010-09-15 04:31:58 +0000123 '-o', '/dev/null', source_path] + self.cpp_flags
Justin Bogner33a2a2e2014-09-03 04:32:08 +0000124 expected_rc = 1
Justin Bogner2cfd1fe2014-09-03 06:01:52 +0000125 if use_verify:
Justin Bogner33a2a2e2014-09-03 04:32:08 +0000126 cmd += ['-Xclang', '-verify']
127 expected_rc = 0
128 out, err, rc = self.execute_command(cmd)
129 if rc == expected_rc:
Daniel Dunbar42ea4632010-09-15 03:57:04 +0000130 return lit.Test.PASS, ""
131 else:
132 report = """Command: %s\n""" % ' '.join(["'%s'" % a
133 for a in cmd])
Justin Bogner33a2a2e2014-09-03 04:32:08 +0000134 report += """Exit Code: %d\n""" % rc
Daniel Dunbar42ea4632010-09-15 03:57:04 +0000135 if out:
136 report += """Standard Output:\n--\n%s--""" % out
137 if err:
138 report += """Standard Error:\n--\n%s--""" % err
139 report += "\n\nExpected compilation to fail!"
Daniel Dunbar5f09d9e02010-09-15 04:31:58 +0000140 return lit.Test.FAIL, report
Daniel Dunbar42ea4632010-09-15 03:57:04 +0000141 else:
142 exec_file = tempfile.NamedTemporaryFile(suffix="exe", delete=False)
143 exec_path = exec_file.name
144 exec_file.close()
145
146 try:
Michael J. Spencerf5799be2010-12-10 19:47:54 +0000147 compile_cmd = [self.cxx_under_test, '-o', exec_path,
Daniel Dunbar5f09d9e02010-09-15 04:31:58 +0000148 source_path] + self.cpp_flags + self.ld_flags
Michael J. Spencerf5799be2010-12-10 19:47:54 +0000149 cmd = compile_cmd
Daniel Dunbar42ea4632010-09-15 03:57:04 +0000150 out, err, exitCode = self.execute_command(cmd)
151 if exitCode != 0:
Daniel Dunbar42ea4632010-09-15 03:57:04 +0000152 report = """Command: %s\n""" % ' '.join(["'%s'" % a
153 for a in cmd])
154 report += """Exit Code: %d\n""" % exitCode
155 if out:
156 report += """Standard Output:\n--\n%s--""" % out
157 if err:
158 report += """Standard Error:\n--\n%s--""" % err
159 report += "\n\nCompilation failed unexpectedly!"
160 return lit.Test.FAIL, report
161
Daniel Dunbar84958712013-02-05 18:03:49 +0000162 cmd = []
163 if self.exec_env:
164 cmd.append('env')
165 cmd.extend('%s=%s' % (name, value)
166 for name,value in self.exec_env.items())
167 cmd.append(exec_path)
Howard Hinnantc1a45fb2012-08-02 18:36:47 +0000168 if lit_config.useValgrind:
169 cmd = lit_config.valgrindArgs + cmd
Howard Hinnant3778f272013-01-14 17:12:54 +0000170 out, err, exitCode = self.execute_command(cmd, source_dir)
Daniel Dunbar42ea4632010-09-15 03:57:04 +0000171 if exitCode != 0:
Daniel Dunbar62b94392013-02-12 19:28:51 +0000172 report = """Compiled With: %s\n""" % \
173 ' '.join(["'%s'" % a for a in compile_cmd])
174 report += """Command: %s\n""" % \
175 ' '.join(["'%s'" % a for a in cmd])
Daniel Dunbar42ea4632010-09-15 03:57:04 +0000176 report += """Exit Code: %d\n""" % exitCode
177 if out:
178 report += """Standard Output:\n--\n%s--""" % out
179 if err:
180 report += """Standard Error:\n--\n%s--""" % err
181 report += "\n\nCompiled test failed unexpectedly!"
182 return lit.Test.FAIL, report
183 finally:
184 try:
185 os.remove(exec_path)
186 except:
187 pass
188 return lit.Test.PASS, ""
189
Dan Albertae2526b2014-08-21 17:30:44 +0000190
191class Configuration(object):
192 def __init__(self, lit_config, config):
193 self.lit_config = lit_config
194 self.config = config
195 self.cxx = None
196 self.src_root = None
197 self.obj_root = None
198 self.env = {}
199 self.compile_flags = []
200 self.link_flags = []
201 self.use_system_lib = False
Justin Bogner33a2a2e2014-09-03 04:32:08 +0000202 self.use_clang_verify = False
Dan Albertae2526b2014-08-21 17:30:44 +0000203
204 if platform.system() not in ('Darwin', 'FreeBSD', 'Linux'):
205 self.lit_config.fatal("unrecognized system")
206
207 def get_lit_conf(self, name, default=None):
208 val = self.lit_config.params.get(name, None)
209 if val is None:
210 val = getattr(self.config, name, None)
211 if val is None:
212 val = default
213 return val
214
Justin Bogner33a2a2e2014-09-03 04:32:08 +0000215 def get_lit_bool(self, name):
216 conf = self.get_lit_conf(name)
217 if conf is None:
218 return None
219 if conf.lower() in ('1', 'true'):
220 return True
221 if conf.lower() in ('', '0', 'false'):
222 return False
223 self.lit_config.fatal(
224 "parameter '{}' should be true or false".format(name))
225
Dan Albertae2526b2014-08-21 17:30:44 +0000226 def configure(self):
227 self.configure_cxx()
228 self.configure_triple()
229 self.configure_src_root()
230 self.configure_obj_root()
231 self.configure_use_system_lib()
Justin Bogner33a2a2e2014-09-03 04:32:08 +0000232 self.configure_use_clang_verify()
Dan Albertae2526b2014-08-21 17:30:44 +0000233 self.configure_env()
234 self.configure_std_flag()
235 self.configure_compile_flags()
236 self.configure_link_flags()
237 self.configure_sanitizer()
238 self.configure_features()
239
240 def get_test_format(self):
241 return LibcxxTestFormat(
242 self.cxx,
Justin Bogner33a2a2e2014-09-03 04:32:08 +0000243 self.use_clang_verify,
Dan Albertae2526b2014-08-21 17:30:44 +0000244 cpp_flags=['-nostdinc++'] + self.compile_flags,
245 ld_flags=['-nodefaultlibs'] + self.link_flags,
246 exec_env=self.env)
247
248 def configure_cxx(self):
249 # Gather various compiler parameters.
250 self.cxx = self.get_lit_conf('cxx_under_test')
251
252 # If no specific cxx_under_test was given, attempt to infer it as
253 # clang++.
254 if self.cxx is None:
255 clangxx = lit.util.which('clang++',
256 self.config.environment['PATH'])
257 if clangxx:
258 self.cxx = clangxx
259 self.lit_config.note(
260 "inferred cxx_under_test as: %r" % self.cxx)
261 if not self.cxx:
262 self.lit_config.fatal('must specify user parameter cxx_under_test '
263 '(e.g., --param=cxx_under_test=clang++)')
264
265 def configure_src_root(self):
266 self.src_root = self.get_lit_conf(
267 'libcxx_src_root', os.path.dirname(self.config.test_source_root))
268
269 def configure_obj_root(self):
270 self.obj_root = self.get_lit_conf('libcxx_obj_root', self.src_root)
271
272 def configure_use_system_lib(self):
273 # This test suite supports testing against either the system library or
274 # the locally built one; the former mode is useful for testing ABI
275 # compatibility between the current headers and a shipping dynamic
276 # library.
Justin Bogner33a2a2e2014-09-03 04:32:08 +0000277 self.use_system_lib = self.get_lit_bool('use_system_lib')
278 if self.use_system_lib is None:
Dan Albertae2526b2014-08-21 17:30:44 +0000279 # Default to testing against the locally built libc++ library.
280 self.use_system_lib = False
281 self.lit_config.note(
282 "inferred use_system_lib as: %r" % self.use_system_lib)
283
Justin Bogner33a2a2e2014-09-03 04:32:08 +0000284 def configure_use_clang_verify(self):
285 '''If set, run clang with -verify on failing tests.'''
286 self.use_clang_verify = self.get_lit_bool('use_clang_verify')
287 if self.use_clang_verify is None:
288 # TODO: Default this to True when using clang.
289 self.use_clang_verify = False
290 self.lit_config.note(
291 "inferred use_clang_verify as: %r" % self.use_clang_verify)
292
Dan Albertae2526b2014-08-21 17:30:44 +0000293 def configure_features(self):
294 # Figure out which of the required locales we support
295 locales = {
296 'Darwin': {
297 'en_US.UTF-8': 'en_US.UTF-8',
298 'cs_CZ.ISO8859-2': 'cs_CZ.ISO8859-2',
299 'fr_FR.UTF-8': 'fr_FR.UTF-8',
300 'fr_CA.ISO8859-1': 'cs_CZ.ISO8859-1',
301 'ru_RU.UTF-8': 'ru_RU.UTF-8',
302 'zh_CN.UTF-8': 'zh_CN.UTF-8',
303 },
304 'FreeBSD': {
305 'en_US.UTF-8': 'en_US.UTF-8',
306 'cs_CZ.ISO8859-2': 'cs_CZ.ISO8859-2',
307 'fr_FR.UTF-8': 'fr_FR.UTF-8',
308 'fr_CA.ISO8859-1': 'fr_CA.ISO8859-1',
309 'ru_RU.UTF-8': 'ru_RU.UTF-8',
310 'zh_CN.UTF-8': 'zh_CN.UTF-8',
311 },
312 'Linux': {
313 'en_US.UTF-8': 'en_US.UTF-8',
314 'cs_CZ.ISO8859-2': 'cs_CZ.ISO-8859-2',
315 'fr_FR.UTF-8': 'fr_FR.UTF-8',
316 'fr_CA.ISO8859-1': 'fr_CA.ISO-8859-1',
317 'ru_RU.UTF-8': 'ru_RU.UTF-8',
318 'zh_CN.UTF-8': 'zh_CN.UTF-8',
319 },
320 'Windows': {
321 'en_US.UTF-8': 'English_United States.1252',
322 'cs_CZ.ISO8859-2': 'Czech_Czech Republic.1250',
323 'fr_FR.UTF-8': 'French_France.1252',
324 'fr_CA.ISO8859-1': 'French_Canada.1252',
325 'ru_RU.UTF-8': 'Russian_Russia.1251',
326 'zh_CN.UTF-8': 'Chinese_China.936',
327 },
328 }
329
330 default_locale = locale.setlocale(locale.LC_ALL)
331 for feature, loc in locales[platform.system()].items():
332 try:
333 locale.setlocale(locale.LC_ALL, loc)
Eric Fiselier561dfb52014-08-23 04:02:21 +0000334 self.config.available_features.add('locale.{0}'.format(feature))
Dan Albertae2526b2014-08-21 17:30:44 +0000335 except:
Eric Fiselier561dfb52014-08-23 04:02:21 +0000336 self.lit_config.warning('The locale {0} is not supported by '
Dan Albertae2526b2014-08-21 17:30:44 +0000337 'your platform. Some tests will be '
338 'unsupported.'.format(loc))
339 locale.setlocale(locale.LC_ALL, default_locale)
340
341 # Write an "available feature" that combines the triple when
342 # use_system_lib is enabled. This is so that we can easily write XFAIL
343 # markers for tests that are known to fail with versions of libc++ as
344 # were shipped with a particular triple.
345 if self.use_system_lib:
346 # Drop sub-major version components from the triple, because the
347 # current XFAIL handling expects exact matches for feature checks.
348 sanitized_triple = re.sub(
349 r"([^-]+)-([^-]+)-([^-.]+).*", r"\1-\2-\3",
350 self.config.target_triple)
351 self.config.available_features.add(
352 'with_system_lib=%s' % sanitized_triple)
353
354 def configure_compile_flags(self):
355 # Configure extra compiler flags.
356 self.compile_flags += ['-I' + self.src_root + '/include',
357 '-I' + self.src_root + '/test/support']
358
359 def configure_link_flags(self):
360 self.link_flags += ['-L' + self.obj_root + '/lib', '-lc++']
361 link_flags_str = self.get_lit_conf('link_flags')
362 if link_flags_str is None:
363 cxx_abi = self.get_lit_conf('cxx_abi', 'libcxxabi')
364 if cxx_abi == 'libstdc++':
365 self.link_flags += ['-lstdc++']
366 elif cxx_abi == 'libsupc++':
367 self.link_flags += ['-lsupc++']
368 elif cxx_abi == 'libcxxabi':
369 self.link_flags += ['-lc++abi']
370 elif cxx_abi == 'libcxxrt':
371 self.link_flags += ['-lcxxrt']
372 elif cxx_abi == 'none':
373 pass
374 else:
375 self.lit_config.fatal(
376 'C++ ABI setting %s unsupported for tests' % cxx_abi)
377
378 if sys.platform == 'darwin':
379 self.link_flags += ['-lSystem']
380 elif sys.platform == 'linux2':
381 self.link_flags += ['-lgcc_eh', '-lc', '-lm', '-lpthread',
382 '-lrt', '-lgcc_s']
383 elif sys.platform.startswith('freebsd'):
384 self.link_flags += ['-lc', '-lm', '-pthread', '-lgcc_s']
385 else:
386 self.lit_config.fatal("unrecognized system")
387
388 self.lit_config.note(
389 "inferred link_flags as: %r" % self.link_flags)
390 if link_flags_str:
391 self.link_flags += shlex.split(link_flags_str)
392
393 if sys.platform == 'linux2':
394 if not self.use_system_lib:
395 self.link_flags += ['-Wl,-R', self.obj_root + '/lib']
396 self.compile_flags += ['-D__STDC_FORMAT_MACROS',
397 '-D__STDC_LIMIT_MACROS',
398 '-D__STDC_CONSTANT_MACROS']
399 elif sys.platform.startswith('freebsd'):
400 if not self.use_system_lib:
401 self.link_flags += ['-Wl,-R', self.obj_root + '/lib']
402
403 def configure_std_flag(self):
404 # Try and get the std version from the command line. Fall back to
405 # default given in lit.site.cfg is not present. If default is not
406 # present then force c++11.
407 std = self.get_lit_conf('std')
408 if std is None:
409 std = 'c++11'
410 self.lit_config.note('using default std: \'-std=c++11\'')
Eric Fiselier561dfb52014-08-23 04:02:21 +0000411 self.compile_flags += ['-std={0}'.format(std)]
Dan Albertae2526b2014-08-21 17:30:44 +0000412 self.config.available_features.add(std)
413
414 def configure_sanitizer(self):
415 san = self.get_lit_conf('llvm_use_sanitizer', '').strip()
416 if san:
417 self.compile_flags += ['-fno-omit-frame-pointer']
418 if san == 'Address':
419 self.compile_flags += ['-fsanitize=address']
420 self.config.available_features.add('asan')
421 elif san == 'Memory' or san == 'MemoryWithOrigins':
422 self.compile_flags += ['-fsanitize=memory']
423 if san == 'MemoryWithOrigins':
424 self.compile_flags += ['-fsanitize-memory-track-origins']
425 self.config.available_features.add('msan')
426 else:
427 self.lit_config.fatal('unsupported value for '
Eric Fiselier561dfb52014-08-23 04:02:21 +0000428 'libcxx_use_san: {0}'.format(san))
Dan Albertae2526b2014-08-21 17:30:44 +0000429
430 def configure_triple(self):
431 # Get or infer the target triple.
432 self.config.target_triple = self.get_lit_conf('target_triple')
433 # If no target triple was given, try to infer it from the compiler
434 # under test.
435 if not self.config.target_triple:
436 self.config.target_triple = lit.util.capture(
437 [self.cxx, '-dumpmachine']).strip()
438 self.lit_config.note(
439 "inferred target_triple as: %r" % self.config.target_triple)
440
441 def configure_env(self):
442 # Configure extra linker parameters.
443 if sys.platform == 'darwin':
444 if not self.use_system_lib:
445 self.env['DYLD_LIBRARY_PATH'] = os.path.join(self.obj_root,
446 'lib')
447
448
Daniel Dunbar42ea4632010-09-15 03:57:04 +0000449# name: The name of this test suite.
450config.name = 'libc++'
451
452# suffixes: A list of file extensions to treat as test files.
453config.suffixes = ['.cpp']
454
455# test_source_root: The root path where tests are located.
456config.test_source_root = os.path.dirname(__file__)
457
Dan Albertae2526b2014-08-21 17:30:44 +0000458configuration = Configuration(lit_config, config)
459configuration.configure()
460config.test_format = configuration.get_test_format()