skdiff: add --failonmismatches and --listfilename options, plus cleanup
These changes are needed in order to switch the buildbots from using "gm -r" to "skdiff" to compare gm results, and should be generally good for humans too.
Review URL: https://codereview.appspot.com/6392054
git-svn-id: http://skia.googlecode.com/svn/trunk@4579 2bbb7eff-a529-9590-31e7-b0007b416f81
diff --git a/tools/tests/run.sh b/tools/tests/run.sh
index f6dbe27..25baf19 100755
--- a/tools/tests/run.sh
+++ b/tools/tests/run.sh
@@ -42,19 +42,20 @@
# Run skdiff over a variety of file pair types: identical bits, identical
# pixels, missing from baseDir, etc.
-# TODO: In the near future, skdiff will return a nonzero exit code in this case.
skdiff_test "$SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/test1"
-# Same as above but without generating HTML output files.
-# TODO: In the near future, skdiff will return a nonzero exit code in this case.
-skdiff_test "--nodiffs $SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/test2"
+# Same as above, except:
+# - return the number of mismatching file pairs
+# - list filenames with each result type to stdout
+# - don't generate HTML output files
+skdiff_test "--failonmismatches --listfilenames --nodiffs $SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/test2"
# Run skdiff over just the files that have identical bits, to validate any
# behavior/return value differences in this case.
-skdiff_test "--nodiffs --match identical-bits $SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/identical-bits"
+skdiff_test "--failonmismatches --nodiffs --match identical-bits $SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/identical-bits"
# Run skdiff over just the files that have identical bits or identical pixels,
# to validate any behavior/return value differences in this case.
-skdiff_test "--nodiffs --match identical-bits --match identical-pixels $SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/identical-bits-or-pixels"
+skdiff_test "--failonmismatches --nodiffs --match identical-bits --match identical-pixels $SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/identical-bits-or-pixels"
echo "All tests passed."
diff --git a/tools/tests/skdiff/identical-bits-or-pixels/output-expected/command_line b/tools/tests/skdiff/identical-bits-or-pixels/output-expected/command_line
index 8a20667..eed0759 100644
--- a/tools/tests/skdiff/identical-bits-or-pixels/output-expected/command_line
+++ b/tools/tests/skdiff/identical-bits-or-pixels/output-expected/command_line
@@ -1 +1 @@
-out/Debug/skdiff --nodiffs --match identical-bits --match identical-pixels tools/tests/skdiff/baseDir tools/tests/skdiff/comparisonDir tools/tests/skdiff/identical-bits-or-pixels/output-actual
+out/Debug/skdiff --failonmismatches --nodiffs --match identical-bits --match identical-pixels tools/tests/skdiff/baseDir tools/tests/skdiff/comparisonDir tools/tests/skdiff/identical-bits-or-pixels/output-actual
diff --git a/tools/tests/skdiff/identical-bits-or-pixels/output-expected/stdout b/tools/tests/skdiff/identical-bits-or-pixels/output-expected/stdout
index 937f8c7..c5d12ec 100644
--- a/tools/tests/skdiff/identical-bits-or-pixels/output-expected/stdout
+++ b/tools/tests/skdiff/identical-bits-or-pixels/output-expected/stdout
@@ -1,4 +1,6 @@
baseDir is [tools/tests/skdiff/baseDir/]
comparisonDir is [tools/tests/skdiff/comparisonDir/]
not writing any diffs to outputDir [tools/tests/skdiff/identical-bits-or-pixels/output-actual/]
-3 of 3 images matched.
+compared 3 file pairs:
+ 2 file pairs contain exactly the same bits
+ 1 file pairs contain the same pixel values, but not the same bits
diff --git a/tools/tests/skdiff/identical-bits/output-expected/command_line b/tools/tests/skdiff/identical-bits/output-expected/command_line
index 49f9ad5..5c8c27b 100644
--- a/tools/tests/skdiff/identical-bits/output-expected/command_line
+++ b/tools/tests/skdiff/identical-bits/output-expected/command_line
@@ -1 +1 @@
-out/Debug/skdiff --nodiffs --match identical-bits tools/tests/skdiff/baseDir tools/tests/skdiff/comparisonDir tools/tests/skdiff/identical-bits/output-actual
+out/Debug/skdiff --failonmismatches --nodiffs --match identical-bits tools/tests/skdiff/baseDir tools/tests/skdiff/comparisonDir tools/tests/skdiff/identical-bits/output-actual
diff --git a/tools/tests/skdiff/identical-bits/output-expected/stdout b/tools/tests/skdiff/identical-bits/output-expected/stdout
index ccb24f8..c40ae33 100644
--- a/tools/tests/skdiff/identical-bits/output-expected/stdout
+++ b/tools/tests/skdiff/identical-bits/output-expected/stdout
@@ -1,4 +1,5 @@
baseDir is [tools/tests/skdiff/baseDir/]
comparisonDir is [tools/tests/skdiff/comparisonDir/]
not writing any diffs to outputDir [tools/tests/skdiff/identical-bits/output-actual/]
-2 of 2 images matched.
+compared 2 file pairs:
+ 2 file pairs contain exactly the same bits
diff --git a/tools/tests/skdiff/test1/output-expected/index.html b/tools/tests/skdiff/test1/output-expected/index.html
index 9f3ce45..88b64ed 100644
--- a/tools/tests/skdiff/test1/output-expected/index.html
+++ b/tools/tests/skdiff/test1/output-expected/index.html
@@ -4,8 +4,8 @@
<tr><th>3 of 12 images matched exactly.<br></th>
<th>every different pixel shown in white</th>
<th>color difference at each pixel</th>
-<th>tools/tests/skdiff/baseDir/</th>
-<th>tools/tests/skdiff/comparisonDir/</th>
+<th>baseDir: tools/tests/skdiff/baseDir/</th>
+<th>comparisonDir: tools/tests/skdiff/comparisonDir/</th>
</tr>
<tr>
<td><b>missing-from-baseDir.png</b><br>Missing from baseDir</td><td>N/A</td><td>N/A</td><td>N/A</td><td><a href="../../../../../tools/tests/skdiff/comparisonDir/missing-from-baseDir.png"><img src="../../../../../tools/tests/skdiff/comparisonDir/missing-from-baseDir.png" height="240px"></a></td></tr>
diff --git a/tools/tests/skdiff/test1/output-expected/return_value b/tools/tests/skdiff/test1/output-expected/return_value
index ec63514..573541a 100644
--- a/tools/tests/skdiff/test1/output-expected/return_value
+++ b/tools/tests/skdiff/test1/output-expected/return_value
@@ -1 +1 @@
-9
+0
diff --git a/tools/tests/skdiff/test1/output-expected/stdout b/tools/tests/skdiff/test1/output-expected/stdout
index 3d337d3..142093f 100644
--- a/tools/tests/skdiff/test1/output-expected/stdout
+++ b/tools/tests/skdiff/test1/output-expected/stdout
@@ -6,12 +6,13 @@
baseDir is [tools/tests/skdiff/baseDir/]
comparisonDir is [tools/tests/skdiff/comparisonDir/]
writing diffs to outputDir is [tools/tests/skdiff/test1/output-actual/]
-Missing in baseDir:
- missing-from-baseDir.png
- missing-from-baseDir.xyz
-Missing in comparisonDir:
- missing-from-comparisonDir.png
- missing-from-comparisonDir.xyz
-3 of 12 images matched.
+compared 12 file pairs:
+ 2 file pairs contain exactly the same bits
+ 1 file pairs contain the same pixel values, but not the same bits
+ 2 file pairs have identical dimensions but some differing pixels
+ 2 file pairs have differing dimensions
+ 1 file pairs contain different bits and are not parsable images
+ 2 file pairs missing from comparisonDir
+ 2 file pairs missing from baseDir
Maximum pixel intensity mismatch 239
Largest area mismatch was 97.99% of pixels
diff --git a/tools/tests/skdiff/test2/output-expected/command_line b/tools/tests/skdiff/test2/output-expected/command_line
index b0bdd62..22dcf74 100644
--- a/tools/tests/skdiff/test2/output-expected/command_line
+++ b/tools/tests/skdiff/test2/output-expected/command_line
@@ -1 +1 @@
-out/Debug/skdiff --nodiffs tools/tests/skdiff/baseDir tools/tests/skdiff/comparisonDir tools/tests/skdiff/test2/output-actual
+out/Debug/skdiff --failonmismatches --listfilenames --nodiffs tools/tests/skdiff/baseDir tools/tests/skdiff/comparisonDir tools/tests/skdiff/test2/output-actual
diff --git a/tools/tests/skdiff/test2/output-expected/stdout b/tools/tests/skdiff/test2/output-expected/stdout
index eae392f..855deae 100644
--- a/tools/tests/skdiff/test2/output-expected/stdout
+++ b/tools/tests/skdiff/test2/output-expected/stdout
@@ -2,12 +2,13 @@
baseDir is [tools/tests/skdiff/baseDir/]
comparisonDir is [tools/tests/skdiff/comparisonDir/]
not writing any diffs to outputDir [tools/tests/skdiff/test2/output-actual/]
-Missing in baseDir:
- missing-from-baseDir.png
- missing-from-baseDir.xyz
-Missing in comparisonDir:
- missing-from-comparisonDir.png
- missing-from-comparisonDir.xyz
-3 of 12 images matched.
+compared 12 file pairs:
+ 2 file pairs contain exactly the same bits: identical-bits-unknown-format.xyz identical-bits.png
+ 1 file pairs contain the same pixel values, but not the same bits: different-bits-identical-pixels.png
+ 2 file pairs have identical dimensions but some differing pixels: slightly-different-pixels-same-size.png very-different-pixels-same-size.png
+ 2 file pairs have differing dimensions: slightly-different-sizes.png very-different-sizes.png
+ 1 file pairs contain different bits and are not parsable images: different-bits-unknown-format.xyz
+ 2 file pairs missing from comparisonDir: missing-from-comparisonDir.png missing-from-comparisonDir.xyz
+ 2 file pairs missing from baseDir: missing-from-baseDir.png missing-from-baseDir.xyz
Maximum pixel intensity mismatch 239
Largest area mismatch was 97.99% of pixels