Truly ignore failures in skimage.
If the expectation is set to ignore failures, do not return a
negative 1 (indicating a failure) when a successful decode
does not match the expectation.
If the file could not be decoded at all, report this to the
user depending on the input expectations file:
No expectations:
Report failure. The user wanted to know if the file could be
decoded.
Expectations expected a valid result:
Report failure. The decode should have matched the result.
Empty expectations:
Print a message that the expectation was missing, but still
return success from skimage, since this is a newly added file
(i.e. it has no expectation yet).
Ignore failure:
Return success from skimage, since it is a known failure.
Update the self tests to ensure these behaviors.
R=epoger@google.com
Review URL: https://codereview.chromium.org/22293006
git-svn-id: http://skia.googlecode.com/svn/trunk@11790 2bbb7eff-a529-9590-31e7-b0007b416f81
diff --git a/tools/skimage_main.cpp b/tools/skimage_main.cpp
index 6ff5978..5b8f095 100644
--- a/tools/skimage_main.cpp
+++ b/tools/skimage_main.cpp
@@ -93,6 +93,21 @@
// Store the names of the filenames to report later which ones failed, succeeded, and were
// invalid.
+// FIXME: Add more arrays, for more specific types of errors, and make the output simpler.
+// If each array holds one type of error, the output can change from:
+//
+// Failures:
+// <image> failed for such and such reason
+// <image> failed for some different reason
+//
+// to:
+//
+// Such and such failures:
+// <image>
+//
+// Different reason failures:
+// <image>
+//
static SkTArray<SkString, false> gInvalidStreams;
static SkTArray<SkString, false> gMissingCodecs;
static SkTArray<SkString, false> gDecodeFailures;
@@ -104,6 +119,9 @@
// the bots will not turn red with each new image test.
static SkTArray<SkString, false> gMissingExpectations;
static SkTArray<SkString, false> gMissingSubsetExpectations;
+// For files that are expected to fail.
+static SkTArray<SkString, false> gKnownFailures;
+static SkTArray<SkString, false> gKnownSubsetFailures;
static SkBitmap::Config gPrefConfig(SkBitmap::kNo_Config);
@@ -188,24 +206,15 @@
}
/**
- * Return true if this filename is a known failure, and therefore a failure
- * to decode should be ignored.
- */
-static bool expect_to_fail(const char* filename) {
- if (NULL == gJsonExpectations.get()) {
- return false;
- }
- skiagm::Expectations jsExpectations = gJsonExpectations->get(filename);
- return jsExpectations.ignoreFailure();
-}
-
-/**
* Compare against an expectation for this filename, if there is one.
* @param digest GmResultDigest, computed from the decoded bitmap, to compare to the
- * expectation.
+ * expectation.
* @param filename String used to find the expected value.
* @param failureArray Array to add a failure message to on failure.
- * @param missingArray Array to add missing expectation to on failure.
+ * @param missingArray Array to add failure message to when missing image
+ * expectation.
+ * @param ignoreArray Array to add failure message to when the image does not match
+ * the expectation, but this is a failure we can ignore.
* @return bool True in any of these cases:
* - the bitmap matches the expectation.
* False in any of these cases:
@@ -217,7 +226,8 @@
static bool compare_to_expectations_if_necessary(const skiagm::GmResultDigest& digest,
const char* filename,
SkTArray<SkString, false>* failureArray,
- SkTArray<SkString, false>* missingArray) {
+ SkTArray<SkString, false>* missingArray,
+ SkTArray<SkString, false>* ignoreArray) {
if (!digest.isValid()) {
if (failureArray != NULL) {
failureArray->push_back().printf("decoded %s, but could not create a GmResultDigest.",
@@ -243,7 +253,10 @@
return true;
}
- if (failureArray != NULL) {
+ if (jsExpectation.ignoreFailure()) {
+ ignoreArray->push_back().printf("%s does not match expectation, but this is known.",
+ filename);
+ } else if (failureArray != NULL) {
failureArray->push_back().printf("decoded %s, but the result does not match "
"expectations.",
filename);
@@ -410,12 +423,26 @@
if (!codec->decode(&stream, &bitmap, gPrefConfig,
SkImageDecoder::kDecodePixels_Mode)) {
- if (expect_to_fail(filename)) {
- gSuccessfulDecodes.push_back().appendf(
- "failed to decode %s, which is a known failure.", srcPath);
- } else {
- gDecodeFailures.push_back().set(srcPath);
+ if (NULL != gJsonExpectations.get()) {
+ skiagm::Expectations jsExpectations = gJsonExpectations->get(filename);
+ if (jsExpectations.ignoreFailure()) {
+ // This is a known failure.
+ gKnownFailures.push_back().appendf(
+ "failed to decode %s, which is a known failure.", srcPath);
+ return;
+ }
+ if (jsExpectations.empty()) {
+ // This is a failure, but it is a new file. Mark it as missing, with
+ // a note that it should be marked failing.
+ gMissingExpectations.push_back().appendf(
+ "new file %s (with no expectations) FAILED to decode.", srcPath);
+ return;
+ }
}
+
+ // If there was a failure, and either there was no expectations file, or
+ // the expectations file listed a valid expectation, report the failure.
+ gDecodeFailures.push_back().set(srcPath);
return;
}
@@ -438,7 +465,8 @@
skiagm::GmResultDigest digest(bitmap);
if (compare_to_expectations_if_necessary(digest, filename,
&gDecodeFailures,
- &gMissingExpectations)) {
+ &gMissingExpectations,
+ &gKnownFailures)) {
gSuccessfulDecodes.push_back().printf("%s [%d %d]", srcPath, bitmap.width(),
bitmap.height());
} else if (!FLAGS_mismatchPath.isEmpty()) {
@@ -491,7 +519,8 @@
if (compare_to_expectations_if_necessary(subsetDigest,
subsetName.c_str(),
&gFailedSubsetDecodes,
- &gMissingSubsetExpectations)) {
+ &gMissingSubsetExpectations,
+ &gKnownSubsetFailures)) {
gSuccessfulSubsetDecodes.push_back().printf("Decoded subset %s from %s",
subsetDim.c_str(), srcPath);
} else if (!FLAGS_mismatchPath.isEmpty()) {
@@ -714,8 +743,11 @@
failed |= print_strings("Failed subset decodes", gFailedSubsetDecodes);
print_strings("Decoded subsets", gSuccessfulSubsetDecodes);
print_strings("Missing subset expectations", gMissingSubsetExpectations);
+ print_strings("Known subset failures", gKnownSubsetFailures);
}
+ print_strings("Known failures", gKnownFailures);
+
return failed ? -1 : 0;
}