libcxx/test/lit.cfg

# -*- Python -*- vim: set syntax=python tabstop=4 expandtab cc=80:

# Configuration file for the 'lit' test runner.

import os
import sys
import platform
import tempfile
import signal
import subprocess
import errno
import time
import shlex

import lit.Test
import lit.formats
import lit.util

# FIXME: For now, this is cribbed from lit.TestRunner, to avoid introducing a
# dependency there. What we more ideally would like to do is lift the "xfail"
# and "requires" handling to be a core lit framework feature.
def isExpectedFail(test, xfails):
    # Check if any of the xfails match an available feature or the target.
    for item in xfails:
        # If this is the wildcard, it always fails.
        if item == '*':
            return True

        # If this is a part of any of the features, it fails.
        for feature in test.config.available_features:
            if item in feature:
                return True

        # If this is a part of the target triple, it fails.
        if item in test.suite.config.target_triple:
            return True

    return False

class LibcxxTestFormat(lit.formats.FileBasedTest):
    """
    Custom test format handler for use with the test format use by libc++.

    Tests fall into two categories:
      FOO.pass.cpp - Executable test which should compile, run, and exit with
                     code 0.
      FOO.fail.cpp - Negative test case which is expected to fail compilation.
    """

    def __init__(self, cxx_under_test, cpp_flags, ld_flags, exec_env):
        self.cxx_under_test = cxx_under_test
        self.cpp_flags = list(cpp_flags)
        self.ld_flags = list(ld_flags)
        self.exec_env = dict(exec_env)

    def execute_command(self, command, in_dir=None):
        kwargs = {
            'stdin' :subprocess.PIPE,
            'stdout':subprocess.PIPE,
            'stderr':subprocess.PIPE,
        }
        if in_dir:
            kwargs['cwd'] = in_dir
        p = subprocess.Popen(command, **kwargs)
        out,err = p.communicate()
        exitCode = p.wait()

        # Detect Ctrl-C in subprocess.
        if exitCode == -signal.SIGINT:
            raise KeyboardInterrupt

        return out, err, exitCode

    def execute(self, test, lit_config):
        while True:
            try:
                return self._execute(test, lit_config)
            except OSError, oe:
                if oe.errno != errno.ETXTBSY:
                    raise
                time.sleep(0.1)

    def _execute(self, test, lit_config):
        # Extract test metadata from the test file.
        xfails = []
        requires = []
        with open(test.getSourcePath()) as f:
            for ln in f:
                if 'XFAIL:' in ln:
                    items = ln[ln.index('XFAIL:') + 6:].split(',')
                    xfails.extend([s.strip() for s in items])
                elif 'REQUIRES:' in ln:
                    items = ln[ln.index('REQUIRES:') + 9:].split(',')
                    requires.extend([s.strip() for s in items])
                elif not ln.startswith("//") and ln.strip():
                    # Stop at the first non-empty line that is not a C++
                    # comment.
                    break

        # Check that we have the required features.
        #
        # FIXME: For now, this is cribbed from lit.TestRunner, to avoid
        # introducing a dependency there. What we more ideally would like to do
        # is lift the "xfail" and "requires" handling to be a core lit
        # framework feature.
        missing_required_features = [f for f in requires
                                     if f not in test.config.available_features]
        if missing_required_features:
            return (lit.Test.UNSUPPORTED,
                    "Test requires the following features: %s" % (
                      ', '.join(missing_required_features),))

        # Determine if this test is an expected failure.
        isXFail = isExpectedFail(test, xfails)

        # Evaluate the test.
        result, report = self._evaluate_test(test, lit_config)

        # Convert the test result based on whether this is an expected failure.
        if isXFail:
            if result != lit.Test.FAIL:
                report += "\n\nTest was expected to FAIL, but did not.\n"
                result = lit.Test.XPASS
            else:
                result = lit.Test.XFAIL

        return result, report

    def _evaluate_test(self, test, lit_config):
        name = test.path_in_suite[-1]
        source_path = test.getSourcePath()
        source_dir = os.path.dirname(source_path)

        # Check what kind of test this is.
        assert name.endswith('.pass.cpp') or name.endswith('.fail.cpp')
        expected_compile_fail = name.endswith('.fail.cpp')

        # If this is a compile (failure) test, build it and check for failure.
        if expected_compile_fail:
            cmd = [self.cxx_under_test, '-c',
                   '-o', '/dev/null', source_path] + self.cpp_flags
            out, err, exitCode = self.execute_command(cmd)
            if exitCode == 1:
                return lit.Test.PASS, ""
            else:
                report = """Command: %s\n""" % ' '.join(["'%s'" % a
                                                         for a in cmd])
                report += """Exit Code: %d\n""" % exitCode
                if out:
                    report += """Standard Output:\n--\n%s--""" % out
                if err:
                    report += """Standard Error:\n--\n%s--""" % err
                report += "\n\nExpected compilation to fail!"
                return lit.Test.FAIL, report
        else:
            exec_file = tempfile.NamedTemporaryFile(suffix="exe", delete=False)
            exec_path = exec_file.name
            exec_file.close()

            try:
                compile_cmd = [self.cxx_under_test, '-o', exec_path,
                       source_path] + self.cpp_flags + self.ld_flags
                cmd = compile_cmd
                out, err, exitCode = self.execute_command(cmd)
                if exitCode != 0:
                    report = """Command: %s\n""" % ' '.join(["'%s'" % a
                                                             for a in cmd])
                    report += """Exit Code: %d\n""" % exitCode
                    if out:
                        report += """Standard Output:\n--\n%s--""" % out
                    if err:
                        report += """Standard Error:\n--\n%s--""" % err
                    report += "\n\nCompilation failed unexpectedly!"
                    return lit.Test.FAIL, report

                cmd = []
                if self.exec_env:
                    cmd.append('env')
                    cmd.extend('%s=%s' % (name, value)
                               for name,value in self.exec_env.items())
                cmd.append(exec_path)
                if lit_config.useValgrind:
                    cmd = lit_config.valgrindArgs + cmd
                out, err, exitCode = self.execute_command(cmd, source_dir)
                if exitCode != 0:
                    report = """Compiled With: %s\n""" % \
                        ' '.join(["'%s'" % a for a in compile_cmd])
                    report += """Command: %s\n""" % \
                        ' '.join(["'%s'" % a for a in cmd])
                    report += """Exit Code: %d\n""" % exitCode
                    if out:
                        report += """Standard Output:\n--\n%s--""" % out
                    if err:
                        report += """Standard Error:\n--\n%s--""" % err
                    report += "\n\nCompiled test failed unexpectedly!"
                    return lit.Test.FAIL, report
            finally:
                try:
                    os.remove(exec_path)
                except:
                    pass
        return lit.Test.PASS, ""

# name: The name of this test suite.
config.name = 'libc++'

# suffixes: A list of file extensions to treat as test files.
config.suffixes = ['.cpp']

# test_source_root: The root path where tests are located.
config.test_source_root = os.path.dirname(__file__)

# Gather various compiler parameters.
cxx_under_test = lit_config.params.get('cxx_under_test', None)
if cxx_under_test is None:
    cxx_under_test = getattr(config, 'cxx_under_test', None)

    # If no specific cxx_under_test was given, attempt to infer it as clang++.
    clangxx = lit.util.which('clang++', config.environment['PATH'])
    if clangxx is not None:
        cxx_under_test = clangxx
        lit_config.note("inferred cxx_under_test as: %r" % (cxx_under_test,))
if cxx_under_test is None:
    lit_config.fatal('must specify user parameter cxx_under_test '
                     '(e.g., --param=cxx_under_test=clang++)')

libcxx_src_root = lit_config.params.get('libcxx_src_root', None)
if libcxx_src_root is None:
    libcxx_src_root = getattr(config, 'libcxx_src_root', None)
    if libcxx_src_root is None:
        libcxx_src_root = os.path.dirname(config.test_source_root)

libcxx_obj_root = lit_config.params.get('libcxx_obj_root', None)
if libcxx_obj_root is None:
    libcxx_obj_root = getattr(config, 'libcxx_obj_root', None)
    if libcxx_obj_root is None:
        libcxx_obj_root = libcxx_src_root

cxx_has_stdcxx0x_flag_str = lit_config.params.get('cxx_has_stdcxx0x_flag', None)
if cxx_has_stdcxx0x_flag_str is not None:
    if cxx_has_stdcxx0x_flag_str.lower() in ('1', 'true'):
        cxx_has_stdcxx0x_flag = True
    elif cxx_has_stdcxx0x_flag_str.lower() in ('', '0', 'false'):
        cxx_has_stdcxx0x_flag = False
    else:
        lit_config.fatal(
            'user parameter cxx_has_stdcxx0x_flag_str should be 0 or 1')
else:
    cxx_has_stdcxx0x_flag = getattr(config, 'cxx_has_stdcxx0x_flag', True)

# This test suite supports testing against either the system library or the
# locally built one; the former mode is useful for testing ABI compatibility
# between the current headers and a shipping dynamic library.
use_system_lib_str = lit_config.params.get('use_system_lib', None)
if use_system_lib_str is not None:
    if use_system_lib_str.lower() in ('1', 'true'):
        use_system_lib = True
    elif use_system_lib_str.lower() in ('', '0', 'false'):
        use_system_lib = False
    else:
        lit_config.fatal('user parameter use_system_lib should be 0 or 1')
else:
    # Default to testing against the locally built libc++ library.
    use_system_lib = False
    lit_config.note("inferred use_system_lib as: %r" % (use_system_lib,))

link_flags = []
link_flags_str = lit_config.params.get('link_flags', None)
if link_flags_str is None:
    link_flags_str = getattr(config, 'link_flags', None)
    if link_flags_str is None:
        if sys.platform == 'darwin':
            link_flags += ['-lSystem']
        elif sys.platform == 'linux2':
            link_flags += ['-lsupc++', '-lgcc_eh', '-lc', '-lm', '-lpthread',
                '-lrt', '-lgcc_s']
        else:
            lit_config.fatal("unrecognized system")
        lit_config.note("inferred link_flags as: %r" % (link_flags,))
if not link_flags_str is None:
    link_flags += shlex.split(link_flags_str)

# Configure extra compiler flags.
include_paths = ['-I' + libcxx_src_root + '/include',
    '-I' + libcxx_src_root + '/test/support']
library_paths = ['-L' + libcxx_obj_root + '/lib']
compile_flags = []
if cxx_has_stdcxx0x_flag:
    compile_flags += ['-std=c++0x']

# Configure extra linker parameters.
exec_env = {}
if sys.platform == 'darwin':
    if not use_system_lib:
        exec_env['DYLD_LIBRARY_PATH'] = os.path.join(libcxx_obj_root, 'lib')
elif sys.platform == 'linux2':
    if not use_system_lib:
        link_flags += ['-Wl,-R', libcxx_obj_root + '/lib']
    compile_flags += ['-D__STDC_FORMAT_MACROS', '-D__STDC_LIMIT_MACROS',
        '-D__STDC_CONSTANT_MACROS']
else:
    lit_config.fatal("unrecognized system")

config.test_format = LibcxxTestFormat(
    cxx_under_test,
    cpp_flags = ['-nostdinc++'] + compile_flags + include_paths,
    ld_flags = ['-nodefaultlibs'] + library_paths + ['-lc++'] + link_flags,
    exec_env = exec_env)

# Get or infer the target triple.
config.target_triple = lit_config.params.get('target_triple', None)
# If no target triple was given, try to infer it from the compiler under test.
if config.target_triple is None:
    config.target_triple = lit.util.capture(
        [cxx_under_test, '-dumpmachine']).strip()
    lit_config.note("inferred target_triple as: %r" % (config.target_triple,))

# Write an "available feature" that combines the triple when use_system_lib is
# enabled. This is so that we can easily write XFAIL markers for tests that are
# known to fail with versions of libc++ as were shipped with a particular
# triple.
if use_system_lib:
    config.available_features.add('with_system_lib=%s' % (
            config.target_triple,))