[lit] Allow boolean expressions in REQUIRES and XFAIL and UNSUPPORTED
A `lit` condition line is now a comma-separated list of boolean expressions.
Comma-separated expressions act as if each expression were on its own
condition line:
For REQUIRES, if every expression is true then the test will run.
For UNSUPPORTED, if every expression is false then the test will run.
For XFAIL, if every expression is false then the test is expected to succeed.
As a special case "XFAIL: *" expects the test to fail.
Examples:
# Test is expected fail on 64-bit Apple simulators and pass everywhere else
XFAIL: x86_64 && apple && !macosx
# Test is unsupported on Windows and on non-Ubuntu Linux
# and supported everywhere else
UNSUPPORTED: linux && !ubuntu, system-windows
Syntax:
* '&&', '||', '!', '(', ')'. 'true' is true. 'false' is false.
* Each test feature is a true identifier.
* Substrings of the target triple are true identifiers for UNSUPPORTED
and XFAIL, but not for REQUIRES. (This matches the current behavior.)
* All other identifiers are false.
* Identifiers are [-+=._a-zA-Z0-9]+
Differential Revision: https://reviews.llvm.org/D18185
llvm-svn: 292896
diff --git a/llvm/utils/lit/tests/Inputs/shtest-format/requires-any-missing.txt b/llvm/utils/lit/tests/Inputs/shtest-format/requires-any-missing.txt
deleted file mode 100644
index c977ee9..0000000
--- a/llvm/utils/lit/tests/Inputs/shtest-format/requires-any-missing.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-RUN: true
-REQUIRES-ANY: a-missing-feature, a-missing-feature-2
diff --git a/llvm/utils/lit/tests/Inputs/shtest-format/requires-any-present.txt b/llvm/utils/lit/tests/Inputs/shtest-format/requires-any-present.txt
deleted file mode 100644
index f3be518..0000000
--- a/llvm/utils/lit/tests/Inputs/shtest-format/requires-any-present.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-RUN: true
-REQUIRES-ANY: a-missing-feature, a-present-feature
diff --git a/llvm/utils/lit/tests/Inputs/shtest-format/requires-any.txt b/llvm/utils/lit/tests/Inputs/shtest-format/requires-any.txt
new file mode 100644
index 0000000..14c8dd5
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-format/requires-any.txt
@@ -0,0 +1,3 @@
+# REQUIRES-ANY is no longer supported. Test should not run.
+REQUIRES-ANY: true, a-present-feature
+RUN: false
diff --git a/llvm/utils/lit/tests/Inputs/shtest-format/requires-missing.txt b/llvm/utils/lit/tests/Inputs/shtest-format/requires-missing.txt
index 9e6648d..d643e57 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-format/requires-missing.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-format/requires-missing.txt
@@ -1,2 +1,5 @@
-RUN: true
-REQUIRES: a-missing-feature
+# REQUIRES with a false clause. Test should not run.
+REQUIRES: true
+REQUIRES: a-missing-feature, true
+REQUIRES: true
+RUN: false
diff --git a/llvm/utils/lit/tests/Inputs/shtest-format/requires-present.txt b/llvm/utils/lit/tests/Inputs/shtest-format/requires-present.txt
index 064f707..9fcbdca 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-format/requires-present.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-format/requires-present.txt
@@ -1,2 +1,4 @@
+# REQUIRES with only true clauses. Test should run.
+REQUIRES: a-present-feature, true, !not-true
+REQUIRES: true
RUN: true
-REQUIRES: a-present-feature
diff --git a/llvm/utils/lit/tests/Inputs/shtest-format/requires-star.txt b/llvm/utils/lit/tests/Inputs/shtest-format/requires-star.txt
new file mode 100644
index 0000000..5566d8b
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-format/requires-star.txt
@@ -0,0 +1,3 @@
+# '*' only works in XFAIL
+REQUIRES: *
+RUN: false
diff --git a/llvm/utils/lit/tests/Inputs/shtest-format/requires-triple.txt b/llvm/utils/lit/tests/Inputs/shtest-format/requires-triple.txt
new file mode 100644
index 0000000..6470bf4
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-format/requires-triple.txt
@@ -0,0 +1,3 @@
+# REQUIRES line that uses target triple, which doesn't work. Test should not run
+REQUIRES: x86_64
+RUN: false
diff --git a/llvm/utils/lit/tests/Inputs/shtest-format/unsupported-expr-false.txt b/llvm/utils/lit/tests/Inputs/shtest-format/unsupported-expr-false.txt
new file mode 100644
index 0000000..00c6160
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-format/unsupported-expr-false.txt
@@ -0,0 +1,9 @@
+# UNSUPPORTED with only false clauses. Test should run.
+UNSUPPORTED: false
+UNSUPPORTED: false, not-true
+UNSUPPORTED: false
+UNSUPPORTED: still-not-true
+UNSUPPORTED: false
+UNSUPPORTED: false
+UNSUPPORTED: false
+RUN: true
diff --git a/llvm/utils/lit/tests/Inputs/shtest-format/unsupported-expr-true.txt b/llvm/utils/lit/tests/Inputs/shtest-format/unsupported-expr-true.txt
new file mode 100644
index 0000000..f48ba7b
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-format/unsupported-expr-true.txt
@@ -0,0 +1,4 @@
+# UNSUPPORTED with a true clause. Test should not run.
+UNSUPPORTED: false
+UNSUPPORTED: false, false, false, _64-unk && a-present-feature, false
+RUN: false
diff --git a/llvm/utils/lit/tests/Inputs/shtest-format/unsupported-star.txt b/llvm/utils/lit/tests/Inputs/shtest-format/unsupported-star.txt
new file mode 100644
index 0000000..1663020
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-format/unsupported-star.txt
@@ -0,0 +1,3 @@
+# '*' only works in XFAIL
+UNSUPPORTED: *
+RUN: false
diff --git a/llvm/utils/lit/tests/Inputs/shtest-format/xfail-expr-false.txt b/llvm/utils/lit/tests/Inputs/shtest-format/xfail-expr-false.txt
new file mode 100644
index 0000000..83b0de1
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-format/xfail-expr-false.txt
@@ -0,0 +1,3 @@
+# XFAIL with only false clauses. Test should run.
+XFAIL: false, a-missing-feature || ! a-present-feature || ! x86_64, false
+RUN: true
diff --git a/llvm/utils/lit/tests/Inputs/shtest-format/xfail-expr-true.txt b/llvm/utils/lit/tests/Inputs/shtest-format/xfail-expr-true.txt
new file mode 100644
index 0000000..3c19748
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-format/xfail-expr-true.txt
@@ -0,0 +1,4 @@
+# XFAIL with a true clause. Test should not run.
+XFAIL: false
+XFAIL: false, a-present-feature && ! a-missing-feature && x86_64
+RUN: false
diff --git a/llvm/utils/lit/tests/boolean-parsing.py b/llvm/utils/lit/tests/boolean-parsing.py
new file mode 100644
index 0000000..372a94d
--- /dev/null
+++ b/llvm/utils/lit/tests/boolean-parsing.py
@@ -0,0 +1,4 @@
+# Test the boolean expression parser
+# used for REQUIRES and UNSUPPORTED and XFAIL
+
+# RUN: %{python} -m lit.BooleanExpression
diff --git a/llvm/utils/lit/tests/shtest-format.py b/llvm/utils/lit/tests/shtest-format.py
index 20884f8..4ae25a4 100644
--- a/llvm/utils/lit/tests/shtest-format.py
+++ b/llvm/utils/lit/tests/shtest-format.py
@@ -46,11 +46,18 @@
# CHECK: UNRESOLVED: shtest-format :: no-test-line.txt
# CHECK: PASS: shtest-format :: pass.txt
-# CHECK: UNSUPPORTED: shtest-format :: requires-any-missing.txt
-# CHECK: PASS: shtest-format :: requires-any-present.txt
+# CHECK: UNRESOLVED: shtest-format :: requires-any.txt
+# CHECK: ValueError: `REQUIRES-ANY: a, b, c` not supported
# CHECK: UNSUPPORTED: shtest-format :: requires-missing.txt
# CHECK: PASS: shtest-format :: requires-present.txt
+# CHECK: UNRESOLVED: shtest-format :: requires-star.txt
+# CHECK: UNSUPPORTED: shtest-format :: requires-triple.txt
+# CHECK: PASS: shtest-format :: unsupported-expr-false.txt
+# CHECK: UNSUPPORTED: shtest-format :: unsupported-expr-true.txt
+# CHECK: UNRESOLVED: shtest-format :: unsupported-star.txt
# CHECK: UNSUPPORTED: shtest-format :: unsupported_dir/some-test.txt
+# CHECK: PASS: shtest-format :: xfail-expr-false.txt
+# CHECK: XFAIL: shtest-format :: xfail-expr-true.txt
# CHECK: XFAIL: shtest-format :: xfail-feature.txt
# CHECK: XFAIL: shtest-format :: xfail-target.txt
# CHECK: XFAIL: shtest-format :: xfail.txt
@@ -70,9 +77,9 @@
# CHECK: shtest-format :: external_shell/fail_with_bad_encoding.txt
# CHECK: shtest-format :: fail.txt
-# CHECK: Expected Passes : 5
-# CHECK: Expected Failures : 3
-# CHECK: Unsupported Tests : 3
-# CHECK: Unresolved Tests : 1
+# CHECK: Expected Passes : 6
+# CHECK: Expected Failures : 4
+# CHECK: Unsupported Tests : 4
+# CHECK: Unresolved Tests : 4
# CHECK: Unexpected Passes : 1
# CHECK: Unexpected Failures: 3
diff --git a/llvm/utils/lit/tests/unit/TestRunner.py b/llvm/utils/lit/tests/unit/TestRunner.py
index ff11834..d8b0064 100644
--- a/llvm/utils/lit/tests/unit/TestRunner.py
+++ b/llvm/utils/lit/tests/unit/TestRunner.py
@@ -108,6 +108,71 @@
value = custom_parser.getValue()
self.assertItemsEqual(value, ['a', 'b', 'c'])
+ def test_bad_keywords(self):
+ def custom_parse(line_number, line, output):
+ return output
+
+ try:
+ IntegratedTestKeywordParser("TAG_NO_SUFFIX", ParserKind.TAG),
+ self.fail("TAG_NO_SUFFIX failed to raise an exception")
+ except ValueError as e:
+ pass
+ except BaseException as e:
+ self.fail("TAG_NO_SUFFIX raised the wrong exception: %r" % e)
+
+ try:
+ IntegratedTestKeywordParser("TAG_WITH_COLON:", ParserKind.TAG),
+ self.fail("TAG_WITH_COLON: failed to raise an exception")
+ except ValueError as e:
+ pass
+ except BaseException as e:
+ self.fail("TAG_WITH_COLON: raised the wrong exception: %r" % e)
+
+ try:
+ IntegratedTestKeywordParser("LIST_WITH_DOT.", ParserKind.LIST),
+ self.fail("LIST_WITH_DOT. failed to raise an exception")
+ except ValueError as e:
+ pass
+ except BaseException as e:
+ self.fail("LIST_WITH_DOT. raised the wrong exception: %r" % e)
+
+ try:
+ IntegratedTestKeywordParser("CUSTOM_NO_SUFFIX",
+ ParserKind.CUSTOM, custom_parse),
+ self.fail("CUSTOM_NO_SUFFIX failed to raise an exception")
+ except ValueError as e:
+ pass
+ except BaseException as e:
+ self.fail("CUSTOM_NO_SUFFIX raised the wrong exception: %r" % e)
+
+ # Both '.' and ':' are allowed for CUSTOM keywords.
+ try:
+ IntegratedTestKeywordParser("CUSTOM_WITH_DOT.",
+ ParserKind.CUSTOM, custom_parse),
+ except BaseException as e:
+ self.fail("CUSTOM_WITH_DOT. raised an exception: %r" % e)
+ try:
+ IntegratedTestKeywordParser("CUSTOM_WITH_COLON:",
+ ParserKind.CUSTOM, custom_parse),
+ except BaseException as e:
+ self.fail("CUSTOM_WITH_COLON: raised an exception: %r" % e)
+
+ try:
+ IntegratedTestKeywordParser("CUSTOM_NO_PARSER:",
+ ParserKind.CUSTOM),
+ self.fail("CUSTOM_NO_PARSER: failed to raise an exception")
+ except ValueError as e:
+ pass
+ except BaseException as e:
+ self.fail("CUSTOM_NO_PARSER: raised the wrong exception: %r" % e)
+
+ # REQUIRES-ANY: has a built-in parser that generates an error,
+ # but it may be overridden by a custom parser.
+ try:
+ IntegratedTestKeywordParser("REQUIRES-ANY:",
+ ParserKind.CUSTOM, custom_parse),
+ except BaseException as e:
+ self.fail("REQUIRES-ANY: raised an exception: %r" % e)
if __name__ == '__main__':
TestIntegratedTestKeywordParser.load_keyword_parser_lit_tests()