Make gm's json results include checksums even if run without --readPath
Review URL: https://codereview.appspot.com/6979043
git-svn-id: http://skia.googlecode.com/svn/trunk@6917 2bbb7eff-a529-9590-31e7-b0007b416f81
diff --git a/gm/gmmain.cpp b/gm/gmmain.cpp
index 690ad0d..314df85 100644
--- a/gm/gmmain.cpp
+++ b/gm/gmmain.cpp
@@ -80,6 +80,7 @@
const static char kJsonKey_ActualResults[] = "actual-results";
const static char kJsonKey_ActualResults_Failed[] = "failed";
const static char kJsonKey_ActualResults_FailureIgnored[]= "failure-ignored";
+const static char kJsonKey_ActualResults_NoComparison[] = "no-comparison";
const static char kJsonKey_ActualResults_Succeeded[] = "succeeded";
const static char kJsonKey_ActualResults_AnyStatus_Checksum[] = "checksum";
@@ -623,7 +624,7 @@
fJsonActualResults_Failed[name.c_str()] = actualResults;
}
} else {
- fJsonActualResults_FailureIgnored[name.c_str()] = actualResults;
+ fJsonActualResults_NoComparison[name.c_str()] = actualResults;
}
// Add this test to the JSON collection of expected results.
@@ -659,6 +660,17 @@
retval |= compare_to_reference_image_on_disk(readPath, name, bitmap,
diffPath,
renderModeDescriptor);
+ } else if (NULL == referenceBitmap) {
+ // If we are running without "--readPath", we still want to
+ // record the actual results.
+ //
+ // For now, though, we don't record results of comparisons against
+ // different in-memory representations (hence the referenceBitmap
+ // NULL check).
+ Json::Value actualResults;
+ actualResults[kJsonKey_ActualResults_AnyStatus_Checksum] =
+ Json::UInt64(SkBitmapChecksummer::Compute64(bitmap));
+ fJsonActualResults_NoComparison[name.c_str()] = actualResults;
}
if (writePath && (gRec.fFlags & kWrite_ConfigFlag)) {
retval |= write_reference_image(gRec, writePath,
@@ -845,6 +857,7 @@
Json::Value fJsonExpectedResults;
Json::Value fJsonActualResults_Failed;
Json::Value fJsonActualResults_FailureIgnored;
+ Json::Value fJsonActualResults_NoComparison;
Json::Value fJsonActualResults_Succeeded;
}; // end of GMMain class definition
@@ -1480,6 +1493,8 @@
gmmain.fJsonActualResults_Failed;
actualResults[kJsonKey_ActualResults_FailureIgnored] =
gmmain.fJsonActualResults_FailureIgnored;
+ actualResults[kJsonKey_ActualResults_NoComparison] =
+ gmmain.fJsonActualResults_NoComparison;
actualResults[kJsonKey_ActualResults_Succeeded] =
gmmain.fJsonActualResults_Succeeded;
Json::Value root;
diff --git a/gm/tests/outputs/compared-against-different-pixels/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-different-pixels/output-expected/json-summary.txt
index 179c60c..180b9ef 100644
--- a/gm/tests/outputs/compared-against-different-pixels/output-expected/json-summary.txt
+++ b/gm/tests/outputs/compared-against-different-pixels/output-expected/json-summary.txt
@@ -6,6 +6,7 @@
}
},
"failure-ignored" : null,
+ "no-comparison" : null,
"succeeded" : null
},
"expected-results" : {
diff --git a/gm/tests/outputs/compared-against-empty-dir/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-empty-dir/output-expected/json-summary.txt
index 935ba50..bdc8f8a 100644
--- a/gm/tests/outputs/compared-against-empty-dir/output-expected/json-summary.txt
+++ b/gm/tests/outputs/compared-against-empty-dir/output-expected/json-summary.txt
@@ -1,7 +1,8 @@
{
"actual-results" : {
"failed" : null,
- "failure-ignored" : {
+ "failure-ignored" : null,
+ "no-comparison" : {
"8888/dashing2" : {
"checksum" : 2675870163990933333
}
diff --git a/gm/tests/outputs/compared-against-identical-bytes/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-identical-bytes/output-expected/json-summary.txt
index cbb28e1..dcfa9f6 100644
--- a/gm/tests/outputs/compared-against-identical-bytes/output-expected/json-summary.txt
+++ b/gm/tests/outputs/compared-against-identical-bytes/output-expected/json-summary.txt
@@ -2,6 +2,7 @@
"actual-results" : {
"failed" : null,
"failure-ignored" : null,
+ "no-comparison" : null,
"succeeded" : {
"8888/dashing2" : {
"checksum" : 2675870163990933333
diff --git a/gm/tests/outputs/compared-against-identical-pixels/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-identical-pixels/output-expected/json-summary.txt
index cbb28e1..dcfa9f6 100644
--- a/gm/tests/outputs/compared-against-identical-pixels/output-expected/json-summary.txt
+++ b/gm/tests/outputs/compared-against-identical-pixels/output-expected/json-summary.txt
@@ -2,6 +2,7 @@
"actual-results" : {
"failed" : null,
"failure-ignored" : null,
+ "no-comparison" : null,
"succeeded" : {
"8888/dashing2" : {
"checksum" : 2675870163990933333
diff --git a/gm/tests/outputs/no-readpath/output-expected/command_line b/gm/tests/outputs/no-readpath/output-expected/command_line
new file mode 100644
index 0000000..8eab263
--- /dev/null
+++ b/gm/tests/outputs/no-readpath/output-expected/command_line
@@ -0,0 +1 @@
+out/Debug/gm --hierarchy --match dashing2 --config 8888 --writeJsonSummary gm/tests/outputs/no-readpath/output-actual/json-summary.txt -w gm/tests/outputs/no-readpath/output-actual/images
diff --git a/gm/tests/outputs/no-readpath/output-expected/images/8888/dashing2.png b/gm/tests/outputs/no-readpath/output-expected/images/8888/dashing2.png
new file mode 100644
index 0000000..465c019
--- /dev/null
+++ b/gm/tests/outputs/no-readpath/output-expected/images/8888/dashing2.png
Binary files differ
diff --git a/gm/tests/outputs/no-readpath/output-expected/json-summary.txt b/gm/tests/outputs/no-readpath/output-expected/json-summary.txt
new file mode 100644
index 0000000..09f095c
--- /dev/null
+++ b/gm/tests/outputs/no-readpath/output-expected/json-summary.txt
@@ -0,0 +1,13 @@
+{
+ "actual-results" : {
+ "failed" : null,
+ "failure-ignored" : null,
+ "no-comparison" : {
+ "8888/dashing2" : {
+ "checksum" : 2675870163990933333
+ }
+ },
+ "succeeded" : null
+ },
+ "expected-results" : null
+}
diff --git a/gm/tests/outputs/no-readpath/output-expected/return_value b/gm/tests/outputs/no-readpath/output-expected/return_value
new file mode 100644
index 0000000..573541a
--- /dev/null
+++ b/gm/tests/outputs/no-readpath/output-expected/return_value
@@ -0,0 +1 @@
+0
diff --git a/gm/tests/outputs/no-readpath/output-expected/stdout b/gm/tests/outputs/no-readpath/output-expected/stdout
new file mode 100644
index 0000000..ca55645
--- /dev/null
+++ b/gm/tests/outputs/no-readpath/output-expected/stdout
@@ -0,0 +1,3 @@
+writing to gm/tests/outputs/no-readpath/output-actual/images
+drawing... dashing2 [640 480]
+Ran 1 tests: 1 passed, 0 failed, 0 missing reference images
diff --git a/gm/tests/rebaseline.sh b/gm/tests/rebaseline.sh
index 77f8711..be8ccf2 100755
--- a/gm/tests/rebaseline.sh
+++ b/gm/tests/rebaseline.sh
@@ -5,34 +5,64 @@
# Use with caution: are you sure the new results are actually correct?
#
# YOU MUST RE-RUN THIS UNTIL THE SELF-TESTS SUCCEED!
-# (It takes one run for each call to gm_test in run.sh)
+# (It takes one run for each failing call to gm_test in run.sh)
+
+function replace_expected_with_actual {
+ # Delete all the expected output files
+ EXPECTED_FILES=$(find outputs/*/output-expected -type f | grep -v /\.svn/)
+ for EXPECTED_FILE in $EXPECTED_FILES; do
+ rm $EXPECTED_FILE
+ done
+
+ # Copy all the actual output files into the "expected" directories,
+ # creating new subdirs as we go.
+ ACTUAL_FILES=$(find outputs/*/output-actual -type f | grep -v /\.svn/)
+ for ACTUAL_FILE in $ACTUAL_FILES; do
+ EXPECTED_FILE=${ACTUAL_FILE//actual/expected}
+ mkdir -p $(dirname $EXPECTED_FILE)
+ cp $ACTUAL_FILE $EXPECTED_FILE
+ done
+}
+
+function svn_add_new_files {
+ # Delete all the "actual" directories, so we can svn-add any new "expected"
+ # directories without adding the "actual" ones.
+ rm -rf outputs/*/output-actual
+ FILES=$(svn stat outputs/* | grep ^\? | awk '{print $2}')
+ for FILE in $FILES; do
+ svn add $FILE
+ done
+ FILES=$(svn stat outputs/*/output-expected | grep ^\? | awk '{print $2}')
+ for FILE in $FILES; do
+ svn add $FILE
+ done
+}
+
+function svn_delete_old_files {
+ FILES=$(svn stat outputs/*/output-expected | grep ^\! | awk '{print $2}')
+ for FILE in $FILES; do
+ svn rm $FILE
+ done
+ FILES=$(svn stat outputs/* | grep ^\! | awk '{print $2}')
+ for FILE in $FILES; do
+ svn rm $FILE
+ done
+}
+
# cd into the gm self-test dir
cd $(dirname $0)
./run.sh
+SELFTEST_RESULT=$?
+echo
+if [ "$SELFTEST_RESULT" != "0" ]; then
+ replace_expected_with_actual
+ echo "Self-tests still failing, you should probably run this again..."
+else
+ svn_add_new_files
+ svn_delete_old_files
+ echo "Self-tests succeeded this time, you should be done!"
+fi
+exit $SELFTEST_RESULT
-# Delete all the expected output files
-EXPECTED_FILES=$(find outputs/*/output-expected -type f | grep -v /\.svn/)
-for EXPECTED_FILE in $EXPECTED_FILES; do
- rm $EXPECTED_FILE
-done
-
-# Copy all the actual output files into the "expected" directories,
-# creating new subdirs as we go.
-ACTUAL_FILES=$(find outputs/*/output-actual -type f | grep -v /\.svn/)
-for ACTUAL_FILE in $ACTUAL_FILES; do
- EXPECTED_FILE=${ACTUAL_FILE//actual/expected}
- mkdir -p $(dirname $EXPECTED_FILE)
- cp $ACTUAL_FILE $EXPECTED_FILE
-done
-
-# "svn add" any newly expected files/dirs, and "svn rm" any that are gone now
-FILES=$(svn stat outputs/*/output-expected | grep ^\? | awk '{print $2}')
-for FILE in $FILES; do
- svn add $FILE
-done
-FILES=$(svn stat outputs/*/output-expected | grep ^\! | awk '{print $2}')
-for FILE in $FILES; do
- svn rm $FILE
-done
diff --git a/gm/tests/run.sh b/gm/tests/run.sh
index 2bb9441..46c8929 100755
--- a/gm/tests/run.sh
+++ b/gm/tests/run.sh
@@ -76,4 +76,9 @@
# Compare generated image against an empty "expected image" dir.
gm_test "--hierarchy --match dashing2 --config 8888 -r $GM_INPUTS/empty-dir" "$GM_OUTPUTS/compared-against-empty-dir"
+# If run without "-r", the JSON's "actual-results" section should contain
+# actual checksums marked as "failure-ignored", but the "expected-results"
+# section should be empty.
+gm_test "--hierarchy --match dashing2 --config 8888" "$GM_OUTPUTS/no-readpath"
+
echo "All tests passed."
