Add --writeJsonSummary option to gm
This writes out a JSON file containing expected and actual checksum values for every test.
Next step: allow the same checksum file to be compared against, instead of PNG files.
Review URL: https://codereview.appspot.com/6940053
git-svn-id: http://skia.googlecode.com/svn/trunk@6843 2bbb7eff-a529-9590-31e7-b0007b416f81
diff --git a/gm/gmmain.cpp b/gm/gmmain.cpp
index b2cc8d8..2f18aaa 100644
--- a/gm/gmmain.cpp
+++ b/gm/gmmain.cpp
@@ -15,6 +15,7 @@
#include "gm.h"
#include "system_preferences.h"
+#include "SkBitmapChecksummer.h"
#include "SkColorPriv.h"
#include "SkData.h"
#include "SkDeferredCanvas.h"
@@ -32,6 +33,8 @@
#include "SkTileGridPicture.h"
#include "SamplePipeControllers.h"
+#include "json/value.h"
+
#if SK_SUPPORT_GPU
#include "GrContextFactory.h"
#include "GrRenderTarget.h"
@@ -74,6 +77,16 @@
const static ErrorBitfield ERROR_READING_REFERENCE_IMAGE = 0x08;
const static ErrorBitfield ERROR_WRITING_REFERENCE_IMAGE = 0x10;
+const static char kJsonKey_ActualResults[] = "actual-results";
+const static char kJsonKey_ActualResults_Failed[] = "failed";
+const static char kJsonKey_ActualResults_FailureIgnored[]= "failure-ignored";
+const static char kJsonKey_ActualResults_Succeeded[] = "succeeded";
+const static char kJsonKey_ActualResults_AnyStatus_Checksum[] = "checksum";
+
+const static char kJsonKey_ExpectedResults[] = "expected-results";
+const static char kJsonKey_ExpectedResults_Checksums[] = "checksums";
+const static char kJsonKey_ExpectedResults_IgnoreFailure[] = "ignore-failure";
+
using namespace skiagm;
/*
@@ -570,24 +583,57 @@
ErrorBitfield compare_to_reference_image_on_disk(
const char readPath [], const SkString& name, SkBitmap &bitmap,
const char diffPath [], const char renderModeDescriptor []) {
+ ErrorBitfield retval;
SkString path = make_filename(readPath, "", name, "png");
SkBitmap referenceBitmap;
- if (SkImageDecoder::DecodeFile(path.c_str(), &referenceBitmap,
+ Json::Value expectedChecksumsArray;
+
+ bool decodedReferenceBitmap =
+ SkImageDecoder::DecodeFile(path.c_str(), &referenceBitmap,
SkBitmap::kARGB_8888_Config,
SkImageDecoder::kDecodePixels_Mode,
- NULL)) {
- return compare_to_reference_image_in_memory(name, bitmap,
- referenceBitmap,
- diffPath,
- renderModeDescriptor);
+ NULL);
+ if (decodedReferenceBitmap) {
+ expectedChecksumsArray.append(Json::UInt64(
+ SkBitmapChecksummer::Compute64(referenceBitmap)));
+ retval = compare_to_reference_image_in_memory(name, bitmap,
+ referenceBitmap,
+ diffPath,
+ renderModeDescriptor);
} else {
if (fNotifyMissingReadReference) {
fprintf(stderr, "FAILED to read %s\n", path.c_str());
}
RecordError(ERROR_READING_REFERENCE_IMAGE, name,
renderModeDescriptor);
- return ERROR_READING_REFERENCE_IMAGE;
+ retval = ERROR_READING_REFERENCE_IMAGE;
}
+
+ // Add this result to the appropriate JSON collection of actual results,
+ // depending on status.
+ Json::Value actualResults;
+ actualResults[kJsonKey_ActualResults_AnyStatus_Checksum] = Json::UInt64(
+ SkBitmapChecksummer::Compute64(bitmap));
+ if (decodedReferenceBitmap) {
+ if (ERROR_NONE == retval) {
+ fJsonActualResults_Succeeded[name.c_str()] = actualResults;
+ } else {
+ fJsonActualResults_Failed[name.c_str()] = actualResults;
+ }
+ } else {
+ fJsonActualResults_FailureIgnored[name.c_str()] = actualResults;
+ }
+
+ // Add this test to the JSON collection of expected results.
+ // For now, we assume that this collection starts out empty and we
+ // just fill it in as we go; once gm accepts a JSON file as input,
+ // we'll have to change that.
+ Json::Value expectedResults;
+ expectedResults[kJsonKey_ExpectedResults_Checksums] = expectedChecksumsArray;
+ expectedResults[kJsonKey_ExpectedResults_IgnoreFailure] = !decodedReferenceBitmap;
+ fJsonExpectedResults[name.c_str()] = expectedResults;
+
+ return retval;
}
// NOTE: As far as I can tell, this function is NEVER called with a
@@ -791,6 +837,11 @@
// information about all failed tests we have encountered so far
SkTArray<FailRec> fFailedTests;
+ Json::Value fJsonExpectedResults;
+ Json::Value fJsonActualResults_Failed;
+ Json::Value fJsonActualResults_FailureIgnored;
+ Json::Value fJsonActualResults_Succeeded;
+
}; // end of GMMain class definition
#if SK_SUPPORT_GPU
@@ -874,6 +925,7 @@
" [--notexturecache]: disable the gpu texture cache\n"
" [--tiledPipe]: Exercise tiled SkGPipe replay\n"
" [--tileGrid]: use a tileGrid structure for SkPicture testing\n"
+" [--writeJsonSummary <path>]: write a JSON-formatted result summary to this file\n"
" [--writePath|-w <path>]: write rendered images into this directory\n"
" [--writePicturePath|-wp <path>]: write .skp files into this directory\n"
);
@@ -957,6 +1009,7 @@
setSystemPreferences();
GMMain gmmain;
+ const char* writeJsonSummaryPath = NULL;// if non-null, where we write the JSON summary
const char* writePath = NULL; // if non-null, where we write the originals
const char* writePicturePath = NULL; // if non-null, where we write serialized pictures
const char* readPath = NULL; // if non-null, were we read from to compare
@@ -1080,6 +1133,11 @@
if (argv < stop && **argv) {
writePath = *argv;
}
+ } else if (0 == strcmp(*argv, "--writeJsonSummary")) {
+ argv++;
+ if (argv < stop && **argv) {
+ writeJsonSummaryPath = *argv;
+ }
} else if ((0 == strcmp(*argv, "--writePicturePath")) ||
(0 == strcmp(*argv, "-wp"))) {
argv++;
@@ -1351,6 +1409,22 @@
testsRun, testsPassed, testsFailed, testsMissingReferenceImages);
gmmain.ListErrors();
+ if (NULL != writeJsonSummaryPath) {
+ Json::Value actualResults;
+ actualResults[kJsonKey_ActualResults_Failed] =
+ gmmain.fJsonActualResults_Failed;
+ actualResults[kJsonKey_ActualResults_FailureIgnored] =
+ gmmain.fJsonActualResults_FailureIgnored;
+ actualResults[kJsonKey_ActualResults_Succeeded] =
+ gmmain.fJsonActualResults_Succeeded;
+ Json::Value root;
+ root[kJsonKey_ActualResults] = actualResults;
+ root[kJsonKey_ExpectedResults] = gmmain.fJsonExpectedResults;
+ std::string jsonStdString = root.toStyledString();
+ SkFILEWStream stream(writeJsonSummaryPath);
+ stream.write(jsonStdString.c_str(), jsonStdString.length());
+ }
+
#if SK_SUPPORT_GPU
#if GR_CACHE_STATS
diff --git a/gm/tests/inputs/empty-dir/README b/gm/tests/inputs/empty-dir/README
new file mode 100644
index 0000000..4d39134
--- /dev/null
+++ b/gm/tests/inputs/empty-dir/README
@@ -0,0 +1 @@
+This directory intentionally left empty. Except for this file.
diff --git a/gm/tests/outputs/compared-against-different-pixels/output-expected/command_line b/gm/tests/outputs/compared-against-different-pixels/output-expected/command_line
index ef8c5af..4253e73 100644
--- a/gm/tests/outputs/compared-against-different-pixels/output-expected/command_line
+++ b/gm/tests/outputs/compared-against-different-pixels/output-expected/command_line
@@ -1 +1 @@
-out/Debug/gm --hierarchy --match dashing2 --config 8888 -r gm/tests/inputs/different-pixels -w gm/tests/outputs/compared-against-different-pixels/output-actual/images
+out/Debug/gm --hierarchy --match dashing2 --config 8888 -r gm/tests/inputs/different-pixels --writeJsonSummary gm/tests/outputs/compared-against-different-pixels/output-actual/json-summary.txt -w gm/tests/outputs/compared-against-different-pixels/output-actual/images
diff --git a/gm/tests/outputs/compared-against-different-pixels/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-different-pixels/output-expected/json-summary.txt
new file mode 100644
index 0000000..179c60c
--- /dev/null
+++ b/gm/tests/outputs/compared-against-different-pixels/output-expected/json-summary.txt
@@ -0,0 +1,17 @@
+{
+ "actual-results" : {
+ "failed" : {
+ "8888/dashing2" : {
+ "checksum" : 2675870163990933333
+ }
+ },
+ "failure-ignored" : null,
+ "succeeded" : null
+ },
+ "expected-results" : {
+ "8888/dashing2" : {
+ "checksums" : [ 15161495552186645995 ],
+ "ignore-failure" : false
+ }
+ }
+}
diff --git a/gm/tests/outputs/compared-against-empty-dir/output-expected/command_line b/gm/tests/outputs/compared-against-empty-dir/output-expected/command_line
new file mode 100644
index 0000000..0ee96be
--- /dev/null
+++ b/gm/tests/outputs/compared-against-empty-dir/output-expected/command_line
@@ -0,0 +1 @@
+out/Debug/gm --hierarchy --match dashing2 --config 8888 -r gm/tests/inputs/empty-dir --writeJsonSummary gm/tests/outputs/compared-against-empty-dir/output-actual/json-summary.txt -w gm/tests/outputs/compared-against-empty-dir/output-actual/images
diff --git a/gm/tests/outputs/compared-against-empty-dir/output-expected/images/8888/dashing2.png b/gm/tests/outputs/compared-against-empty-dir/output-expected/images/8888/dashing2.png
new file mode 100644
index 0000000..465c019
--- /dev/null
+++ b/gm/tests/outputs/compared-against-empty-dir/output-expected/images/8888/dashing2.png
Binary files differ
diff --git a/gm/tests/outputs/compared-against-empty-dir/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-empty-dir/output-expected/json-summary.txt
new file mode 100644
index 0000000..935ba50
--- /dev/null
+++ b/gm/tests/outputs/compared-against-empty-dir/output-expected/json-summary.txt
@@ -0,0 +1,17 @@
+{
+ "actual-results" : {
+ "failed" : null,
+ "failure-ignored" : {
+ "8888/dashing2" : {
+ "checksum" : 2675870163990933333
+ }
+ },
+ "succeeded" : null
+ },
+ "expected-results" : {
+ "8888/dashing2" : {
+ "checksums" : null,
+ "ignore-failure" : true
+ }
+ }
+}
diff --git a/gm/tests/outputs/compared-against-empty-dir/output-expected/return_value b/gm/tests/outputs/compared-against-empty-dir/output-expected/return_value
new file mode 100644
index 0000000..573541a
--- /dev/null
+++ b/gm/tests/outputs/compared-against-empty-dir/output-expected/return_value
@@ -0,0 +1 @@
+0
diff --git a/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout b/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout
new file mode 100644
index 0000000..436554d
--- /dev/null
+++ b/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout
@@ -0,0 +1,5 @@
+reading from gm/tests/inputs/empty-dir
+writing to gm/tests/outputs/compared-against-empty-dir/output-actual/images
+drawing... dashing2 [640 480]
+FAILED to read gm/tests/inputs/empty-dir/8888/dashing2.png
+Ran 1 tests: 0 passed, 0 failed, 1 missing reference images
diff --git a/gm/tests/outputs/compared-against-identical-bytes/output-expected/command_line b/gm/tests/outputs/compared-against-identical-bytes/output-expected/command_line
index 6e89999..556498a 100644
--- a/gm/tests/outputs/compared-against-identical-bytes/output-expected/command_line
+++ b/gm/tests/outputs/compared-against-identical-bytes/output-expected/command_line
@@ -1 +1 @@
-out/Debug/gm --hierarchy --match dashing2 --config 8888 -r gm/tests/inputs/identical-bytes -w gm/tests/outputs/compared-against-identical-bytes/output-actual/images
+out/Debug/gm --hierarchy --match dashing2 --config 8888 -r gm/tests/inputs/identical-bytes --writeJsonSummary gm/tests/outputs/compared-against-identical-bytes/output-actual/json-summary.txt -w gm/tests/outputs/compared-against-identical-bytes/output-actual/images
diff --git a/gm/tests/outputs/compared-against-identical-bytes/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-identical-bytes/output-expected/json-summary.txt
new file mode 100644
index 0000000..cbb28e1
--- /dev/null
+++ b/gm/tests/outputs/compared-against-identical-bytes/output-expected/json-summary.txt
@@ -0,0 +1,17 @@
+{
+ "actual-results" : {
+ "failed" : null,
+ "failure-ignored" : null,
+ "succeeded" : {
+ "8888/dashing2" : {
+ "checksum" : 2675870163990933333
+ }
+ }
+ },
+ "expected-results" : {
+ "8888/dashing2" : {
+ "checksums" : [ 2675870163990933333 ],
+ "ignore-failure" : false
+ }
+ }
+}
diff --git a/gm/tests/outputs/compared-against-identical-pixels/output-expected/command_line b/gm/tests/outputs/compared-against-identical-pixels/output-expected/command_line
index 285cef2..ecdef27 100644
--- a/gm/tests/outputs/compared-against-identical-pixels/output-expected/command_line
+++ b/gm/tests/outputs/compared-against-identical-pixels/output-expected/command_line
@@ -1 +1 @@
-out/Debug/gm --hierarchy --match dashing2 --config 8888 -r gm/tests/inputs/identical-pixels -w gm/tests/outputs/compared-against-identical-pixels/output-actual/images
+out/Debug/gm --hierarchy --match dashing2 --config 8888 -r gm/tests/inputs/identical-pixels --writeJsonSummary gm/tests/outputs/compared-against-identical-pixels/output-actual/json-summary.txt -w gm/tests/outputs/compared-against-identical-pixels/output-actual/images
diff --git a/gm/tests/outputs/compared-against-identical-pixels/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-identical-pixels/output-expected/json-summary.txt
new file mode 100644
index 0000000..cbb28e1
--- /dev/null
+++ b/gm/tests/outputs/compared-against-identical-pixels/output-expected/json-summary.txt
@@ -0,0 +1,17 @@
+{
+ "actual-results" : {
+ "failed" : null,
+ "failure-ignored" : null,
+ "succeeded" : {
+ "8888/dashing2" : {
+ "checksum" : 2675870163990933333
+ }
+ }
+ },
+ "expected-results" : {
+ "8888/dashing2" : {
+ "checksums" : [ 2675870163990933333 ],
+ "ignore-failure" : false
+ }
+ }
+}
diff --git a/gm/tests/run.sh b/gm/tests/run.sh
index b97b289..2bb9441 100755
--- a/gm/tests/run.sh
+++ b/gm/tests/run.sh
@@ -38,6 +38,7 @@
# - with the arguments in $1
# - writing resulting images into $2/output-actual/images
# - writing stdout into $2/output-actual/stdout
+# - writing json summary into $2/output-actual/json-summary.txt
# - writing return value into $2/output-actual/return_value
# Then compare all of those against $2/output-expected .
function gm_test {
@@ -51,7 +52,7 @@
rm -rf $ACTUAL_OUTPUT_DIR
mkdir -p $ACTUAL_OUTPUT_DIR
- COMMAND="$GM_BINARY $GM_ARGS -w $ACTUAL_OUTPUT_DIR/images"
+ COMMAND="$GM_BINARY $GM_ARGS --writeJsonSummary $ACTUAL_OUTPUT_DIR/json-summary.txt -w $ACTUAL_OUTPUT_DIR/images"
echo "$COMMAND" >$ACTUAL_OUTPUT_DIR/command_line
$COMMAND &>$ACTUAL_OUTPUT_DIR/stdout
echo $? >$ACTUAL_OUTPUT_DIR/return_value
@@ -72,4 +73,7 @@
# Compare generated image against an input image file with different pixels.
gm_test "--hierarchy --match dashing2 --config 8888 -r $GM_INPUTS/different-pixels" "$GM_OUTPUTS/compared-against-different-pixels"
+# Compare generated image against an empty "expected image" dir.
+gm_test "--hierarchy --match dashing2 --config 8888 -r $GM_INPUTS/empty-dir" "$GM_OUTPUTS/compared-against-empty-dir"
+
echo "All tests passed."
diff --git a/gyp/gm.gyp b/gyp/gm.gyp
index 6f0fc3d..2dd9aa3 100644
--- a/gyp/gm.gyp
+++ b/gyp/gm.gyp
@@ -10,6 +10,7 @@
'include_dirs' : [
'../src/core',
'../src/pipe/utils/',
+ '../src/utils/',
],
'includes': [
'gmslides.gypi',
@@ -26,6 +27,8 @@
'effects.gyp:effects',
'images.gyp:images',
'pdf.gyp:pdf',
+ 'utils.gyp:utils',
+ '../third_party/externals/jsoncpp/jsoncpp.gyp:jsoncpp',
],
'conditions': [
['skia_os == "mac"', {
