libcxx/test/lit.cfg

# -*- Python -*-

# Configuration file for the 'lit' test runner.

import os
import sys
import platform
import tempfile
import signal
import subprocess
import errno
import time

# FIXME: For now, this is cribbed from lit.TestRunner, to avoid introducing a
# dependency there. What we more ideally would like to do is lift the "xfail"
# and "requires" handling to be a core lit framework feature.
def isExpectedFail(test, xfails):
    # Check if any of the xfails match an available feature or the target.
    for item in xfails:
        # If this is the wildcard, it always fails.
        if item == '*':
            return True

        # If this is a part of any of the features, it fails.
        for feature in test.config.available_features:
            if item in feature:
                return True

        # If this is a part of the target triple, it fails.
        if item in test.suite.config.target_triple:
            return True

    return False

class LibcxxTestFormat(lit.formats.FileBasedTest):
    """
    Custom test format handler for use with the test format use by libc++.

    Tests fall into two categories:
      FOO.pass.cpp - Executable test which should compile, run, and exit with
                     code 0.
      FOO.fail.cpp - Negative test case which is expected to fail compilation.
    """

    def __init__(self, cxx_under_test, cpp_flags, ld_flags, exec_env):
        self.cxx_under_test = cxx_under_test
        self.cpp_flags = list(cpp_flags)
        self.ld_flags = list(ld_flags)
        self.exec_env = dict(exec_env)

    def execute_command(self, command, in_dir=None):
        kwargs = {
            'stdin' :subprocess.PIPE,
            'stdout':subprocess.PIPE,
            'stderr':subprocess.PIPE,
        }
        if in_dir:
            kwargs['cwd'] = in_dir
        p = subprocess.Popen(command, **kwargs)
        out,err = p.communicate()
        exitCode = p.wait()

        # Detect Ctrl-C in subprocess.
        if exitCode == -signal.SIGINT:
            raise KeyboardInterrupt

        return out, err, exitCode

    def execute(self, test, lit_config):
        while True:
            try:
                return self._execute(test, lit_config)
            except OSError, oe:
                if oe.errno != errno.ETXTBSY:
                    raise
                time.sleep(0.1)

    def _execute(self, test, lit_config):
        # Extract test metadata from the test file.
        xfails = []
        requires = []
        with open(test.getSourcePath()) as f:
            for ln in f:
                if 'XFAIL:' in ln:
                    items = ln[ln.index('XFAIL:') + 6:].split(',')
                    xfails.extend([s.strip() for s in items])
                elif 'REQUIRES:' in ln:
                    items = ln[ln.index('REQUIRES:') + 9:].split(',')
                    requires.extend([s.strip() for s in items])
                elif not ln.startswith("//") and ln.strip():
                    # Stop at the first non-empty line that is not a C++
                    # comment.
                    break

        # Check that we have the required features.
        #
        # FIXME: For now, this is cribbed from lit.TestRunner, to avoid
        # introducing a dependency there. What we more ideally would like to do
        # is lift the "xfail" and "requires" handling to be a core lit framework
        # feature.
        missing_required_features = [f for f in requires
                                     if f not in test.config.available_features]
        if missing_required_features:
            return (lit.Test.UNSUPPORTED,
                    "Test requires the following features: %s" % (
                      ', '.join(missing_required_features),))

        # Determine if this test is an expected failure.
        isXFail = isExpectedFail(test, xfails)

        # Evaluate the test.
        result, report = self._evaluate_test(test, lit_config)

        # Convert the test result based on whether this is an expected failure.
        if isXFail:
            if result != lit.Test.FAIL:
                report += "\n\nTest was expected to FAIL, but did not.\n"
                result = lit.Test.XPASS
            else:
                result = lit.Test.XFAIL

        return result, report

    def _evaluate_test(self, test, lit_config):
        name = test.path_in_suite[-1]
        source_path = test.getSourcePath()
        source_dir = os.path.dirname(source_path)

        # Check what kind of test this is.
        assert name.endswith('.pass.cpp') or name.endswith('.fail.cpp')
        expected_compile_fail = name.endswith('.fail.cpp')

        # If this is a compile (failure) test, build it and check for failure.
        if expected_compile_fail:
            cmd = [self.cxx_under_test, '-c',
                   '-o', '/dev/null', source_path] + self.cpp_flags
            out, err, exitCode = self.execute_command(cmd)
            if exitCode == 1:
                return lit.Test.PASS, ""
            else:
                report = """Command: %s\n""" % ' '.join(["'%s'" % a
                                                         for a in cmd])
                report += """Exit Code: %d\n""" % exitCode
                if out:
                    report += """Standard Output:\n--\n%s--""" % out
                if err:
                    report += """Standard Error:\n--\n%s--""" % err
                report += "\n\nExpected compilation to fail!"
                return lit.Test.FAIL, report
        else:
            exec_file = tempfile.NamedTemporaryFile(suffix="exe", delete=False)
            exec_path = exec_file.name
            exec_file.close()

            try:
                compile_cmd = [self.cxx_under_test, '-o', exec_path,
                       source_path] + self.cpp_flags + self.ld_flags
                cmd = compile_cmd
                out, err, exitCode = self.execute_command(cmd)
                if exitCode != 0:
                    report = """Command: %s\n""" % ' '.join(["'%s'" % a
                                                             for a in cmd])
                    report += """Exit Code: %d\n""" % exitCode
                    if out:
                        report += """Standard Output:\n--\n%s--""" % out
                    if err:
                        report += """Standard Error:\n--\n%s--""" % err
                    report += "\n\nCompilation failed unexpectedly!"
                    return lit.Test.FAIL, report

                cmd = []
                if self.exec_env:
                    cmd.append('env')
                    cmd.extend('%s=%s' % (name, value)
                               for name,value in self.exec_env.items())
                cmd.append(exec_path)
                if lit_config.useValgrind:
                    cmd = lit_config.valgrindArgs + cmd
                out, err, exitCode = self.execute_command(cmd, source_dir)
                if exitCode != 0:
                    report = """Compiled With: %s\n""" % ' '.join(["'%s'" % a
                                                                   for a in compile_cmd])
                    report += """Command: %s\n""" % ' '.join(["'%s'" % a
                                                             for a in cmd])
                    report += """Exit Code: %d\n""" % exitCode
                    if out:
                        report += """Standard Output:\n--\n%s--""" % out
                    if err:
                        report += """Standard Error:\n--\n%s--""" % err
                    report += "\n\nCompiled test failed unexpectedly!"
                    return lit.Test.FAIL, report
            finally:
                try:
                    os.remove(exec_path)
                except:
                    pass
        return lit.Test.PASS, ""

# name: The name of this test suite.
config.name = 'libc++'

# suffixes: A list of file extensions to treat as test files.
config.suffixes = ['.cpp']

# test_source_root: The root path where tests are located.
config.test_source_root = os.path.dirname(__file__)

# Gather various compiler parameters.
cxx_under_test = lit.params.get('cxx_under_test', None)
if cxx_under_test is None:
    cxx_under_test = getattr(config, 'cxx_under_test', None)

    # If no specific cxx_under_test was given, attempt to infer it as clang++.
    clangxx = lit.util.which('clang++', config.environment['PATH'])
    if clangxx is not None:
        cxx_under_test = clangxx
        lit.note("inferred cxx_under_test as: %r" % (cxx_under_test,))
if cxx_under_test is None:
    lit.fatal('must specify user parameter cxx_under_test '
              '(e.g., --param=cxx_under_test=clang++)')

libcxx_src_root = lit.params.get('libcxx_src_root', None)
if libcxx_src_root is None:
    libcxx_src_root = getattr(config, 'libcxx_src_root', None)
    if libcxx_src_root is None:
        libcxx_src_root = os.path.dirname(config.test_source_root)

libcxx_obj_root = lit.params.get('libcxx_obj_root', None)
if libcxx_obj_root is None:
    libcxx_obj_root = getattr(config, 'libcxx_obj_root', None)
    if libcxx_obj_root is None:
        libcxx_obj_root = libcxx_src_root

cxx_has_stdcxx0x_flag_str = lit.params.get('cxx_has_stdcxx0x_flag', None)
if cxx_has_stdcxx0x_flag_str is not None:
    if cxx_has_stdcxx0x_flag_str.lower() in ('1', 'true'):
        cxx_has_stdcxx0x_flag = True
    elif cxx_has_stdcxx0x_flag_str.lower() in ('', '0', 'false'):
        cxx_has_stdcxx0x_flag = False
    else:
        lit.fatal('user parameter cxx_has_stdcxx0x_flag_str should be 0 or 1')
else:
    cxx_has_stdcxx0x_flag = getattr(config, 'cxx_has_stdcxx0x_flag', True)

# This test suite supports testing against either the system library or the
# locally built one; the former mode is useful for testing ABI compatibility
# between the current headers and a shipping dynamic library.
use_system_lib_str = lit.params.get('use_system_lib', None)
if use_system_lib_str is not None:
    if use_system_lib_str.lower() in ('1', 'true'):
        use_system_lib = True
    elif use_system_lib_str.lower() in ('', '0', 'false'):
        use_system_lib = False
    else:
        lit.fatal('user parameter use_system_lib should be 0 or 1')
else:
    # Default to testing against the locally built libc++ library.
    use_system_lib = False
    lit.note("inferred use_system_lib as: %r" % (use_system_lib,))

# Configure extra compiler flags.
include_paths = ['-I' + libcxx_src_root + '/include', '-I' + libcxx_src_root + '/test/support']
library_paths = ['-L' + libcxx_obj_root + '/lib']
compile_flags = []
if cxx_has_stdcxx0x_flag:
    compile_flags += ['-std=c++0x']

# Configure extra libraries.
exec_env = {}
libraries = []
if sys.platform == 'darwin':
    libraries += ['-lSystem']
    if not use_system_lib:
        exec_env['DYLD_LIBRARY_PATH'] = os.path.join(libcxx_obj_root, 'lib')
elif sys.platform == 'linux2':
    libraries += ['-lsupc++', '-lgcc_eh', '-lc', '-lm', '-lpthread', '-lrt', '-lgcc_s']
    if not use_system_lib:
        libraries += ['-Wl,-R', libcxx_obj_root + '/lib']
    compile_flags += ['-D__STDC_FORMAT_MACROS', '-D__STDC_LIMIT_MACROS', '-D__STDC_CONSTANT_MACROS']
else:
    lit.fatal("unrecognized system")

config.test_format = LibcxxTestFormat(
    cxx_under_test,
    cpp_flags = ['-nostdinc++'] + compile_flags + include_paths,
    ld_flags = ['-nodefaultlibs'] + library_paths + ['-lc++'] + libraries,
    exec_env = exec_env)

# Get or infer the target triple.
config.target_triple = lit.params.get('target_triple', None)
# If no target triple was given, try to infer it from the compiler under test.
if config.target_triple is None:
    config.target_triple = lit.util.capture(
        [cxx_under_test, '-dumpmachine']).strip()
    lit.note("inferred target_triple as: %r" % (config.target_triple,))

# Write an "available feature" that combines the triple when use_system_lib is
# enabled. This is so that we can easily write XFAIL markers for tests that are
# known to fail with versions of libc++ as were shipped with a particular
# triple.
if use_system_lib:
    config.available_features.add('with_system_lib=%s' % (
            config.target_triple,))