Don’t include tests that rely on varying two-phase-lookup behaviour in approval tests
diff --git a/projects/SelfTest/Baselines/console.std.approved.txt b/projects/SelfTest/Baselines/console.std.approved.txt
index f5c53cd..61b3390 100644
--- a/projects/SelfTest/Baselines/console.std.approved.txt
+++ b/projects/SelfTest/Baselines/console.std.approved.txt
@@ -830,6 +830,6 @@
"first" == "second"
===============================================================================
-test cases: 159 | 115 passed | 42 failed | 2 failed as expected
-assertions: 915 | 819 passed | 78 failed | 18 failed as expected
+test cases: 157 | 113 passed | 42 failed | 2 failed as expected
+assertions: 913 | 817 passed | 78 failed | 18 failed as expected
diff --git a/projects/SelfTest/Baselines/console.sw.approved.txt b/projects/SelfTest/Baselines/console.sw.approved.txt
index 4ac83ca..2222058 100644
--- a/projects/SelfTest/Baselines/console.sw.approved.txt
+++ b/projects/SelfTest/Baselines/console.sw.approved.txt
@@ -8758,32 +8758,6 @@
"{ StringMaker<has_maker> }"
-------------------------------------------------------------------------------
-toString( vectors<has_maker_and_toString )
--------------------------------------------------------------------------------
-ToStringWhich.cpp:<line number>
-...............................................................................
-
-ToStringWhich.cpp:<line number>:
-PASSED:
- REQUIRE( Catch::toString( v ) == "{ StringMaker<has_maker_and_toString> }" )
-with expansion:
- "{ StringMaker<has_maker_and_toString> }"
- ==
- "{ StringMaker<has_maker_and_toString> }"
-
--------------------------------------------------------------------------------
-toString( vectors<has_toString )
--------------------------------------------------------------------------------
-ToStringWhich.cpp:<line number>
-...............................................................................
-
-ToStringWhich.cpp:<line number>:
-PASSED:
- REQUIRE( Catch::toString( v ) == "{ {?} }" )
-with expansion:
- "{ {?} }" == "{ {?} }"
-
--------------------------------------------------------------------------------
toString(enum w/operator<<)
-------------------------------------------------------------------------------
EnumToString.cpp:<line number>
@@ -9052,6 +9026,6 @@
PASSED:
===============================================================================
-test cases: 159 | 114 passed | 43 failed | 2 failed as expected
-assertions: 917 | 819 passed | 80 failed | 18 failed as expected
+test cases: 157 | 112 passed | 43 failed | 2 failed as expected
+assertions: 915 | 817 passed | 80 failed | 18 failed as expected
diff --git a/projects/SelfTest/Baselines/junit.sw.approved.txt b/projects/SelfTest/Baselines/junit.sw.approved.txt
index 43b808a..81eb097 100644
--- a/projects/SelfTest/Baselines/junit.sw.approved.txt
+++ b/projects/SelfTest/Baselines/junit.sw.approved.txt
@@ -1,6 +1,6 @@
<?xml version="1.1" encoding="UTF-8"?>
<testsuites>
- <testsuite name="<exe-name>" errors="13" failures="68" tests="918" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
+ <testsuite name="<exe-name>" errors="13" failures="68" tests="916" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<testcase classname="global" name="# A test name that starts with a #" time="{duration}"/>
<testcase classname="global" name="'Not' checks that should fail" time="{duration}">
<failure message="false != false" type="CHECK">
@@ -654,8 +654,6 @@
<testcase classname="global" name="toString( has_maker_and_toString )" time="{duration}"/>
<testcase classname="global" name="toString( has_toString )" time="{duration}"/>
<testcase classname="global" name="toString( vectors<has_maker )" time="{duration}"/>
- <testcase classname="global" name="toString( vectors<has_maker_and_toString )" time="{duration}"/>
- <testcase classname="global" name="toString( vectors<has_toString )" time="{duration}"/>
<testcase classname="global" name="toString(enum w/operator<<)" time="{duration}"/>
<testcase classname="global" name="toString(enum)" time="{duration}"/>
<testcase classname="global" name="vector<int> -> toString" time="{duration}"/>
diff --git a/projects/SelfTest/Baselines/xml.sw.approved.txt b/projects/SelfTest/Baselines/xml.sw.approved.txt
index 3398978..0af5860 100644
--- a/projects/SelfTest/Baselines/xml.sw.approved.txt
+++ b/projects/SelfTest/Baselines/xml.sw.approved.txt
@@ -9246,30 +9246,6 @@
</Expression>
<OverallResult success="true"/>
</TestCase>
- <TestCase name="toString( vectors<has_maker_and_toString )">
- <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/ToStringWhich.cpp" >
- <Original>
- Catch::toString( v ) == "{ StringMaker<has_maker_and_toString> }"
- </Original>
- <Expanded>
- "{ StringMaker<has_maker_and_toString> }"
-==
-"{ StringMaker<has_maker_and_toString> }"
- </Expanded>
- </Expression>
- <OverallResult success="true"/>
- </TestCase>
- <TestCase name="toString( vectors<has_toString )">
- <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/ToStringWhich.cpp" >
- <Original>
- Catch::toString( v ) == "{ {?} }"
- </Original>
- <Expanded>
- "{ {?} }" == "{ {?} }"
- </Expanded>
- </Expression>
- <OverallResult success="true"/>
- </TestCase>
<TestCase name="toString(enum w/operator<<)">
<Expression success="true" type="CHECK" filename="projects/<exe-name>/EnumToString.cpp" >
<Original>
@@ -9527,7 +9503,7 @@
</Section>
<OverallResult success="true"/>
</TestCase>
- <OverallResults successes="819" failures="81" expectedFailures="18"/>
+ <OverallResults successes="817" failures="81" expectedFailures="18"/>
</Group>
- <OverallResults successes="819" failures="80" expectedFailures="18"/>
+ <OverallResults successes="817" failures="80" expectedFailures="18"/>
</Catch>
diff --git a/projects/SelfTest/ToStringWhich.cpp b/projects/SelfTest/ToStringWhich.cpp
index 1d4aa89..a1d2a60 100644
--- a/projects/SelfTest/ToStringWhich.cpp
+++ b/projects/SelfTest/ToStringWhich.cpp
@@ -37,19 +37,21 @@
}
// Call the overload
-TEST_CASE( "toString( has_maker )", "[toString]" ) {
+TEST_CASE( "toString( has_maker )", "toString]" ) {
has_maker item;
REQUIRE( Catch::toString( item ) == "StringMaker<has_maker>" );
}
// Call the overload
-TEST_CASE( "toString( has_maker_and_toString )", "[toString]" ) {
+TEST_CASE( "toString( has_maker_and_toString )", "[.][toString]" ) {
has_maker_and_toString item;
REQUIRE( Catch::toString( item ) == "toString( has_maker_and_toString )" );
}
// Vectors...
-TEST_CASE( "toString( vectors<has_toString )", "[toString]" ) {
+
+// Don't run this in approval tests as it is sensitive to two phase lookup differences
+TEST_CASE( "toString( vectors<has_toString )", "[.][toString][!nonportable]" ) {
std::vector<has_toString> v(1);
// This invokes template<T> toString which actually gives us '{ ? }'
REQUIRE( Catch::toString( v ) == "{ {?} }" );
@@ -61,7 +63,8 @@
}
-TEST_CASE( "toString( vectors<has_maker_and_toString )", "[toString]" ) {
+// Don't run this in approval tests as it is sensitive to two phase lookup differences
+TEST_CASE( "toString( vectors<has_maker_and_toString )", "[.][toString][!nonportable]" ) {
std::vector<has_maker_and_toString> v(1);
// Note: This invokes the template<T> toString -> StringMaker
REQUIRE( Catch::toString( v ) == "{ StringMaker<has_maker_and_toString> }" );
diff --git a/scripts/approvalTests.py b/scripts/approvalTests.py
index e57241d..6c51aa4 100644
--- a/scripts/approvalTests.py
+++ b/scripts/approvalTests.py
@@ -137,15 +137,15 @@
print(" " + cmdPath)
# Standard console reporter
-approve("console.std", ["~[c++11]", "--order", "lex"])
+approve("console.std", ["~[c++11]~[!nonportable]", "--order", "lex"])
# console reporter, include passes, warn about No Assertions
-approve("console.sw", ["~[c++11]", "-s", "-w", "NoAssertions", "--order", "lex"])
+approve("console.sw", ["~[c++11]~[!nonportable]", "-s", "-w", "NoAssertions", "--order", "lex"])
# console reporter, include passes, warn about No Assertions, limit failures to first 4
-approve("console.swa4", ["~[c++11]", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"])
+approve("console.swa4", ["~[c++11]~[!nonportable]", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"])
# junit reporter, include passes, warn about No Assertions
-approve("junit.sw", ["~[c++11]", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"])
+approve("junit.sw", ["~[c++11]~[!nonportable]", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"])
# xml reporter, include passes, warn about No Assertions
-approve("xml.sw", ["~[c++11]", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"])
+approve("xml.sw", ["~[c++11]~[!nonportable]", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"])
if overallResult != 0:
print("If these differenecs are expected run approve.py to approve new baselines")