gm: add --ignoreTests flag
BUG=skia:1600
R=scroggo@google.com
Review URL: https://codereview.chromium.org/23526065
git-svn-id: http://skia.googlecode.com/svn/trunk@11374 2bbb7eff-a529-9590-31e7-b0007b416f81
diff --git a/gm/gm.cpp b/gm/gm.cpp
index 06d3c6d..4a8b3c3 100644
--- a/gm/gm.cpp
+++ b/gm/gm.cpp
@@ -14,6 +14,7 @@
fBGColor = SK_ColorWHITE;
fCanvasIsDeferred = false;
fHaveCalledOnceBeforeDraw = false;
+ fIgnoreFailures = false;
}
GM::~GM() {}
diff --git a/gm/gm.h b/gm/gm.h
index 2886f8c..3a65160 100644
--- a/gm/gm.h
+++ b/gm/gm.h
@@ -92,6 +92,11 @@
return gResourcePath;
}
+ bool isIgnoringFailures() const { return fIgnoreFailures; }
+ void setIgnoreFailures(bool val) {
+ fIgnoreFailures = val;
+ }
+
bool isCanvasDeferred() const { return fCanvasIsDeferred; }
void setCanvasIsDeferred(bool isDeferred) {
fCanvasIsDeferred = isDeferred;
@@ -117,6 +122,7 @@
SkColor fBGColor;
bool fCanvasIsDeferred; // work-around problem in srcmode.cpp
bool fHaveCalledOnceBeforeDraw;
+ bool fIgnoreFailures; // whether to file any failures as failure-ignored
};
typedef SkTRegistry<GM*(*)(void*)> GMRegistry;
diff --git a/gm/gm_expectations.h b/gm/gm_expectations.h
index 96c6d11..de89589 100644
--- a/gm/gm_expectations.h
+++ b/gm/gm_expectations.h
@@ -133,6 +133,11 @@
bool ignoreFailure() const { return this->fIgnoreFailure; }
/**
+ * Override default setting of fIgnoreFailure.
+ */
+ void setIgnoreFailure(bool val) { this->fIgnoreFailure = val; }
+
+ /**
* Returns true iff there are no allowed results.
*/
bool empty() const { return this->fAllowedResultDigests.empty(); }
diff --git a/gm/gmmain.cpp b/gm/gmmain.cpp
index 4939624..2cd8f47 100644
--- a/gm/gmmain.cpp
+++ b/gm/gmmain.cpp
@@ -905,6 +905,9 @@
* See comments above complete_bitmap() for more detail.
*/
Expectations expectations = expectationsSource->get(nameWithExtension.c_str());
+ if (gm->isIgnoringFailures()) {
+ expectations.setIgnoreFailure(true);
+ }
errors.add(compare_to_expectations(expectations, *actualBitmapAndDigest,
gm->shortName(), configName, "", true));
} else {
@@ -1380,6 +1383,8 @@
DEFINE_string(ignoreErrorTypes, kDefaultIgnorableErrorTypes.asString(" ").c_str(),
"Space-separated list of ErrorTypes that should be ignored. If any *other* error "
"types are encountered, the tool will exit with a nonzero return value.");
+DEFINE_string(ignoreTests, "", "Space delimited list of tests for which we should ignore "
+ "failures.");
DEFINE_string(match, "", "[~][^]substring[$] [...] of test name to run.\n"
"Multiple matches may be separated by spaces.\n"
"~ causes a matching test to always be skipped\n"
@@ -2163,6 +2168,9 @@
if (SkCommandLineFlags::ShouldSkip(FLAGS_match, shortName)) {
continue;
}
+ if (FLAGS_ignoreTests.contains(shortName)) {
+ gm->setIgnoreFailures(true);
+ }
gmsRun++;
SkISize size = gm->getISize();
diff --git a/gm/tests/outputs/ignoring-one-test/output-expected/command_line b/gm/tests/outputs/ignoring-one-test/output-expected/command_line
new file mode 100644
index 0000000..49025ad
--- /dev/null
+++ b/gm/tests/outputs/ignoring-one-test/output-expected/command_line
@@ -0,0 +1 @@
+out/Debug/gm --verbose --hierarchy --match selftest1 --ignoreTests selftest1 --config 8888 565 -r gm/tests/inputs/json/different-pixels.json --writeJsonSummaryPath gm/tests/outputs/ignoring-one-test/output-actual/json-summary.txt --writePath gm/tests/outputs/ignoring-one-test/output-actual/writePath --mismatchPath gm/tests/outputs/ignoring-one-test/output-actual/mismatchPath --missingExpectationsPath gm/tests/outputs/ignoring-one-test/output-actual/missingExpectationsPath
diff --git a/gm/tests/outputs/ignoring-one-test/output-expected/json-summary.txt b/gm/tests/outputs/ignoring-one-test/output-expected/json-summary.txt
new file mode 100644
index 0000000..5f8aa4e
--- /dev/null
+++ b/gm/tests/outputs/ignoring-one-test/output-expected/json-summary.txt
@@ -0,0 +1,25 @@
+{
+ "actual-results" : {
+ "failed" : null,
+ "failure-ignored" : {
+ "565/selftest1.png" : [ "bitmap-64bitMD5", 12927999507540085554 ],
+ "8888/selftest1.png" : [ "bitmap-64bitMD5", 1209453360120438698 ]
+ },
+ "no-comparison" : null,
+ "succeeded" : null
+ },
+ "expected-results" : {
+ "565/selftest1.png" : {
+ "allowed-digests" : [
+ [ "bitmap-64bitMD5", 8863920166200910451 ]
+ ],
+ "ignore-failure" : true
+ },
+ "8888/selftest1.png" : {
+ "allowed-digests" : [
+ [ "bitmap-64bitMD5", 13451349865803053525 ]
+ ],
+ "ignore-failure" : true
+ }
+ }
+}
diff --git a/gm/tests/outputs/ignoring-one-test/output-expected/mismatchPath/565/bogusfile b/gm/tests/outputs/ignoring-one-test/output-expected/mismatchPath/565/bogusfile
new file mode 100644
index 0000000..d86cd5b
--- /dev/null
+++ b/gm/tests/outputs/ignoring-one-test/output-expected/mismatchPath/565/bogusfile
@@ -0,0 +1 @@
+Created additional file to make sure directory isn't empty, because self-test cannot handle empty directories.
diff --git a/gm/tests/outputs/ignoring-one-test/output-expected/mismatchPath/565/selftest1.png b/gm/tests/outputs/ignoring-one-test/output-expected/mismatchPath/565/selftest1.png
new file mode 100644
index 0000000..c93c688
--- /dev/null
+++ b/gm/tests/outputs/ignoring-one-test/output-expected/mismatchPath/565/selftest1.png
@@ -0,0 +1 @@
+[contents of gm/tests/outputs/ignoring-one-test/output-actual/mismatchPath/565/selftest1.png]
diff --git a/gm/tests/outputs/ignoring-one-test/output-expected/mismatchPath/8888/bogusfile b/gm/tests/outputs/ignoring-one-test/output-expected/mismatchPath/8888/bogusfile
new file mode 100644
index 0000000..d86cd5b
--- /dev/null
+++ b/gm/tests/outputs/ignoring-one-test/output-expected/mismatchPath/8888/bogusfile
@@ -0,0 +1 @@
+Created additional file to make sure directory isn't empty, because self-test cannot handle empty directories.
diff --git a/gm/tests/outputs/ignoring-one-test/output-expected/mismatchPath/8888/selftest1.png b/gm/tests/outputs/ignoring-one-test/output-expected/mismatchPath/8888/selftest1.png
new file mode 100644
index 0000000..1ce89fe
--- /dev/null
+++ b/gm/tests/outputs/ignoring-one-test/output-expected/mismatchPath/8888/selftest1.png
@@ -0,0 +1 @@
+[contents of gm/tests/outputs/ignoring-one-test/output-actual/mismatchPath/8888/selftest1.png]
diff --git a/gm/tests/outputs/ignoring-one-test/output-expected/mismatchPath/bogusfile b/gm/tests/outputs/ignoring-one-test/output-expected/mismatchPath/bogusfile
new file mode 100644
index 0000000..d86cd5b
--- /dev/null
+++ b/gm/tests/outputs/ignoring-one-test/output-expected/mismatchPath/bogusfile
@@ -0,0 +1 @@
+Created additional file to make sure directory isn't empty, because self-test cannot handle empty directories.
diff --git a/gm/tests/outputs/ignoring-one-test/output-expected/missingExpectationsPath/565/bogusfile b/gm/tests/outputs/ignoring-one-test/output-expected/missingExpectationsPath/565/bogusfile
new file mode 100644
index 0000000..d86cd5b
--- /dev/null
+++ b/gm/tests/outputs/ignoring-one-test/output-expected/missingExpectationsPath/565/bogusfile
@@ -0,0 +1 @@
+Created additional file to make sure directory isn't empty, because self-test cannot handle empty directories.
diff --git a/gm/tests/outputs/ignoring-one-test/output-expected/missingExpectationsPath/8888/bogusfile b/gm/tests/outputs/ignoring-one-test/output-expected/missingExpectationsPath/8888/bogusfile
new file mode 100644
index 0000000..d86cd5b
--- /dev/null
+++ b/gm/tests/outputs/ignoring-one-test/output-expected/missingExpectationsPath/8888/bogusfile
@@ -0,0 +1 @@
+Created additional file to make sure directory isn't empty, because self-test cannot handle empty directories.
diff --git a/gm/tests/outputs/ignoring-one-test/output-expected/missingExpectationsPath/bogusfile b/gm/tests/outputs/ignoring-one-test/output-expected/missingExpectationsPath/bogusfile
new file mode 100644
index 0000000..d86cd5b
--- /dev/null
+++ b/gm/tests/outputs/ignoring-one-test/output-expected/missingExpectationsPath/bogusfile
@@ -0,0 +1 @@
+Created additional file to make sure directory isn't empty, because self-test cannot handle empty directories.
diff --git a/gm/tests/outputs/ignoring-one-test/output-expected/return_value b/gm/tests/outputs/ignoring-one-test/output-expected/return_value
new file mode 100644
index 0000000..ace9d03
--- /dev/null
+++ b/gm/tests/outputs/ignoring-one-test/output-expected/return_value
@@ -0,0 +1 @@
+255
diff --git a/gm/tests/outputs/ignoring-one-test/output-expected/stderr b/gm/tests/outputs/ignoring-one-test/output-expected/stderr
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/gm/tests/outputs/ignoring-one-test/output-expected/stderr
diff --git a/gm/tests/outputs/ignoring-one-test/output-expected/stdout b/gm/tests/outputs/ignoring-one-test/output-expected/stdout
new file mode 100644
index 0000000..ed498dd
--- /dev/null
+++ b/gm/tests/outputs/ignoring-one-test/output-expected/stdout
@@ -0,0 +1,18 @@
+GM: These configs will be run: 8888 565
+GM: reading expectations from JSON summary file gm/tests/inputs/json/different-pixels.json
+GM: writing to gm/tests/outputs/ignoring-one-test/output-actual/writePath
+GM: writing mismatches to gm/tests/outputs/ignoring-one-test/output-actual/mismatchPath
+GM: writing images without expectations to gm/tests/outputs/ignoring-one-test/output-actual/missingExpectationsPath
+GM: drawing... selftest1 [300 200]
+GM: Ran 1 GMs
+GM: ... over 2 configs ["8888", "565"]
+GM: ... and 0 modes []
+GM: ... so there should be a total of 2 tests.
+GM: Ran 2 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 ExpectationsMismatch=2 MissingExpectations=0 WritingReferenceImage=0
+GM: [*] 0 NoGpuContext:
+GM: [ ] 0 IntentionallySkipped:
+GM: [*] 0 RenderModeMismatch:
+GM: [*] 2 ExpectationsMismatch: 8888/selftest1 565/selftest1
+GM: [ ] 0 MissingExpectations:
+GM: [*] 0 WritingReferenceImage:
+GM: (results marked with [*] will cause nonzero return value)
diff --git a/gm/tests/outputs/ignoring-one-test/output-expected/writePath/565/bogusfile b/gm/tests/outputs/ignoring-one-test/output-expected/writePath/565/bogusfile
new file mode 100644
index 0000000..d86cd5b
--- /dev/null
+++ b/gm/tests/outputs/ignoring-one-test/output-expected/writePath/565/bogusfile
@@ -0,0 +1 @@
+Created additional file to make sure directory isn't empty, because self-test cannot handle empty directories.
diff --git a/gm/tests/outputs/ignoring-one-test/output-expected/writePath/565/selftest1.png b/gm/tests/outputs/ignoring-one-test/output-expected/writePath/565/selftest1.png
new file mode 100644
index 0000000..0e665b8
--- /dev/null
+++ b/gm/tests/outputs/ignoring-one-test/output-expected/writePath/565/selftest1.png
@@ -0,0 +1 @@
+[contents of gm/tests/outputs/ignoring-one-test/output-actual/writePath/565/selftest1.png]
diff --git a/gm/tests/outputs/ignoring-one-test/output-expected/writePath/8888/bogusfile b/gm/tests/outputs/ignoring-one-test/output-expected/writePath/8888/bogusfile
new file mode 100644
index 0000000..d86cd5b
--- /dev/null
+++ b/gm/tests/outputs/ignoring-one-test/output-expected/writePath/8888/bogusfile
@@ -0,0 +1 @@
+Created additional file to make sure directory isn't empty, because self-test cannot handle empty directories.
diff --git a/gm/tests/outputs/ignoring-one-test/output-expected/writePath/8888/selftest1.png b/gm/tests/outputs/ignoring-one-test/output-expected/writePath/8888/selftest1.png
new file mode 100644
index 0000000..5ea72ba
--- /dev/null
+++ b/gm/tests/outputs/ignoring-one-test/output-expected/writePath/8888/selftest1.png
@@ -0,0 +1 @@
+[contents of gm/tests/outputs/ignoring-one-test/output-actual/writePath/8888/selftest1.png]
diff --git a/gm/tests/outputs/ignoring-one-test/output-expected/writePath/bogusfile b/gm/tests/outputs/ignoring-one-test/output-expected/writePath/bogusfile
new file mode 100644
index 0000000..d86cd5b
--- /dev/null
+++ b/gm/tests/outputs/ignoring-one-test/output-expected/writePath/bogusfile
@@ -0,0 +1 @@
+Created additional file to make sure directory isn't empty, because self-test cannot handle empty directories.
diff --git a/gm/tests/run.sh b/gm/tests/run.sh
index a7d13f1..7a7bc13 100755
--- a/gm/tests/run.sh
+++ b/gm/tests/run.sh
@@ -201,6 +201,9 @@
gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/different-pixels" "$GM_OUTPUTS/compared-against-different-pixels-images"
gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels.json" "$GM_OUTPUTS/compared-against-different-pixels-json"
+# Exercise --ignoreTests flag.
+gm_test "--verbose --hierarchy --match selftest1 --ignoreTests selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels.json" "$GM_OUTPUTS/ignoring-one-test"
+
# Compare different pixels, but with a SUBSET of the expectations marked as
# ignore-failure.
gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels-ignore-some-failures.json" "$GM_OUTPUTS/ignoring-some-failures"
diff --git a/tools/flags/SkCommandLineFlags.h b/tools/flags/SkCommandLineFlags.h
index c324a1f..370198b 100644
--- a/tools/flags/SkCommandLineFlags.h
+++ b/tools/flags/SkCommandLineFlags.h
@@ -125,6 +125,18 @@
bool isEmpty() const { return this->count() == 0; }
+ /**
+ * Returns true iff string is equal to one of the strings in this array.
+ */
+ bool contains(const char* string) const {
+ for (int i = 0; i < fStrings.count(); i++) {
+ if (fStrings[i].equals(string)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
private:
void reset() { fStrings.reset(); }
