gm self-tests: generate JSON expectations, in addition to image expectations
Review URL: https://codereview.appspot.com/7310050
git-svn-id: http://skia.googlecode.com/svn/trunk@7624 2bbb7eff-a529-9590-31e7-b0007b416f81
diff --git a/gm/tests/outputs/compared-against-different-pixels-images/output-expected/command_line b/gm/tests/outputs/compared-against-different-pixels-images/output-expected/command_line
new file mode 100644
index 0000000..494eb85
--- /dev/null
+++ b/gm/tests/outputs/compared-against-different-pixels-images/output-expected/command_line
@@ -0,0 +1 @@
+out/Debug/gm --hierarchy --match dashing2 --config 8888 --config 565 -r gm/tests/inputs/images/different-pixels --writeJsonSummary gm/tests/outputs/compared-against-different-pixels-images/output-actual/json-summary.txt
diff --git a/gm/tests/outputs/compared-against-different-pixels/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-different-pixels-images/output-expected/json-summary.txt
similarity index 100%
rename from gm/tests/outputs/compared-against-different-pixels/output-expected/json-summary.txt
rename to gm/tests/outputs/compared-against-different-pixels-images/output-expected/json-summary.txt
diff --git a/gm/tests/outputs/compared-against-different-pixels/output-expected/return_value b/gm/tests/outputs/compared-against-different-pixels-images/output-expected/return_value
similarity index 100%
rename from gm/tests/outputs/compared-against-different-pixels/output-expected/return_value
rename to gm/tests/outputs/compared-against-different-pixels-images/output-expected/return_value
diff --git a/gm/tests/outputs/compared-against-different-pixels/output-expected/stdout b/gm/tests/outputs/compared-against-different-pixels-images/output-expected/stdout
similarity index 63%
rename from gm/tests/outputs/compared-against-different-pixels/output-expected/stdout
rename to gm/tests/outputs/compared-against-different-pixels-images/output-expected/stdout
index 885dfd7..a001fc6 100644
--- a/gm/tests/outputs/compared-against-different-pixels/output-expected/stdout
+++ b/gm/tests/outputs/compared-against-different-pixels-images/output-expected/stdout
@@ -1,3 +1,3 @@
-reading from gm/tests/inputs/different-pixels
+reading from gm/tests/inputs/images/different-pixels
drawing... dashing2 [640 480]
Ran 1 tests: 0 passed, 1 failed, 0 missing reference images
diff --git a/gm/tests/outputs/compared-against-different-pixels/output-expected/command_line b/gm/tests/outputs/compared-against-different-pixels/output-expected/command_line
deleted file mode 100644
index d887baa..0000000
--- a/gm/tests/outputs/compared-against-different-pixels/output-expected/command_line
+++ /dev/null
@@ -1 +0,0 @@
-out/Debug/gm --hierarchy --match dashing2 --config 8888 --config 565 -r gm/tests/inputs/different-pixels --writeJsonSummary gm/tests/outputs/compared-against-different-pixels/output-actual/json-summary.txt
diff --git a/gm/tests/outputs/compared-against-empty-dir/output-expected/command_line b/gm/tests/outputs/compared-against-empty-dir/output-expected/command_line
index d00ec60..637b397 100644
--- a/gm/tests/outputs/compared-against-empty-dir/output-expected/command_line
+++ b/gm/tests/outputs/compared-against-empty-dir/output-expected/command_line
@@ -1 +1 @@
-out/Debug/gm --hierarchy --match dashing2 --config 8888 --config 565 -r gm/tests/inputs/empty-dir --writeJsonSummary gm/tests/outputs/compared-against-empty-dir/output-actual/json-summary.txt
+out/Debug/gm --hierarchy --match dashing2 --config 8888 --config 565 -r gm/tests/inputs/images/empty-dir --writeJsonSummary gm/tests/outputs/compared-against-empty-dir/output-actual/json-summary.txt
diff --git a/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout b/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout
index 246c05b..ee432a1 100644
--- a/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout
+++ b/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout
@@ -1,5 +1,5 @@
-reading from gm/tests/inputs/empty-dir
+reading from gm/tests/inputs/images/empty-dir
drawing... dashing2 [640 480]
-FAILED to read gm/tests/inputs/empty-dir/8888/dashing2.png
-FAILED to read gm/tests/inputs/empty-dir/565/dashing2.png
+FAILED to read gm/tests/inputs/images/empty-dir/8888/dashing2.png
+FAILED to read gm/tests/inputs/images/empty-dir/565/dashing2.png
Ran 1 tests: 0 passed, 0 failed, 1 missing reference images
diff --git a/gm/tests/outputs/compared-against-identical-bytes-images/output-expected/command_line b/gm/tests/outputs/compared-against-identical-bytes-images/output-expected/command_line
new file mode 100644
index 0000000..382655a
--- /dev/null
+++ b/gm/tests/outputs/compared-against-identical-bytes-images/output-expected/command_line
@@ -0,0 +1 @@
+out/Debug/gm --hierarchy --match dashing2 --config 8888 --config 565 -r gm/tests/inputs/images/identical-bytes --writeJsonSummary gm/tests/outputs/compared-against-identical-bytes-images/output-actual/json-summary.txt
diff --git a/gm/tests/outputs/compared-against-identical-bytes/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-identical-bytes-images/output-expected/json-summary.txt
similarity index 100%
rename from gm/tests/outputs/compared-against-identical-bytes/output-expected/json-summary.txt
rename to gm/tests/outputs/compared-against-identical-bytes-images/output-expected/json-summary.txt
diff --git a/gm/tests/outputs/compared-against-identical-bytes/output-expected/return_value b/gm/tests/outputs/compared-against-identical-bytes-images/output-expected/return_value
similarity index 100%
rename from gm/tests/outputs/compared-against-identical-bytes/output-expected/return_value
rename to gm/tests/outputs/compared-against-identical-bytes-images/output-expected/return_value
diff --git a/gm/tests/outputs/compared-against-identical-pixels/output-expected/stdout b/gm/tests/outputs/compared-against-identical-bytes-images/output-expected/stdout
similarity index 63%
copy from gm/tests/outputs/compared-against-identical-pixels/output-expected/stdout
copy to gm/tests/outputs/compared-against-identical-bytes-images/output-expected/stdout
index a60b4ef..8055992 100644
--- a/gm/tests/outputs/compared-against-identical-pixels/output-expected/stdout
+++ b/gm/tests/outputs/compared-against-identical-bytes-images/output-expected/stdout
@@ -1,3 +1,3 @@
-reading from gm/tests/inputs/identical-pixels
+reading from gm/tests/inputs/images/identical-bytes
drawing... dashing2 [640 480]
Ran 1 tests: 1 passed, 0 failed, 0 missing reference images
diff --git a/gm/tests/outputs/compared-against-identical-bytes/output-expected/command_line b/gm/tests/outputs/compared-against-identical-bytes/output-expected/command_line
deleted file mode 100644
index d065b13..0000000
--- a/gm/tests/outputs/compared-against-identical-bytes/output-expected/command_line
+++ /dev/null
@@ -1 +0,0 @@
-out/Debug/gm --hierarchy --match dashing2 --config 8888 --config 565 -r gm/tests/inputs/identical-bytes --writeJsonSummary gm/tests/outputs/compared-against-identical-bytes/output-actual/json-summary.txt
diff --git a/gm/tests/outputs/compared-against-identical-bytes/output-expected/stdout b/gm/tests/outputs/compared-against-identical-bytes/output-expected/stdout
deleted file mode 100644
index 03153e6..0000000
--- a/gm/tests/outputs/compared-against-identical-bytes/output-expected/stdout
+++ /dev/null
@@ -1,3 +0,0 @@
-reading from gm/tests/inputs/identical-bytes
-drawing... dashing2 [640 480]
-Ran 1 tests: 1 passed, 0 failed, 0 missing reference images
diff --git a/gm/tests/outputs/compared-against-identical-pixels-images/output-expected/command_line b/gm/tests/outputs/compared-against-identical-pixels-images/output-expected/command_line
new file mode 100644
index 0000000..0b0ee91
--- /dev/null
+++ b/gm/tests/outputs/compared-against-identical-pixels-images/output-expected/command_line
@@ -0,0 +1 @@
+out/Debug/gm --hierarchy --match dashing2 --config 8888 --config 565 -r gm/tests/inputs/images/identical-pixels --writeJsonSummary gm/tests/outputs/compared-against-identical-pixels-images/output-actual/json-summary.txt
diff --git a/gm/tests/outputs/compared-against-identical-pixels/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-identical-pixels-images/output-expected/json-summary.txt
similarity index 100%
rename from gm/tests/outputs/compared-against-identical-pixels/output-expected/json-summary.txt
rename to gm/tests/outputs/compared-against-identical-pixels-images/output-expected/json-summary.txt
diff --git a/gm/tests/outputs/compared-against-identical-pixels/output-expected/return_value b/gm/tests/outputs/compared-against-identical-pixels-images/output-expected/return_value
similarity index 100%
rename from gm/tests/outputs/compared-against-identical-pixels/output-expected/return_value
rename to gm/tests/outputs/compared-against-identical-pixels-images/output-expected/return_value
diff --git a/gm/tests/outputs/compared-against-identical-pixels/output-expected/stdout b/gm/tests/outputs/compared-against-identical-pixels-images/output-expected/stdout
similarity index 63%
rename from gm/tests/outputs/compared-against-identical-pixels/output-expected/stdout
rename to gm/tests/outputs/compared-against-identical-pixels-images/output-expected/stdout
index a60b4ef..2f8fa86 100644
--- a/gm/tests/outputs/compared-against-identical-pixels/output-expected/stdout
+++ b/gm/tests/outputs/compared-against-identical-pixels-images/output-expected/stdout
@@ -1,3 +1,3 @@
-reading from gm/tests/inputs/identical-pixels
+reading from gm/tests/inputs/images/identical-pixels
drawing... dashing2 [640 480]
Ran 1 tests: 1 passed, 0 failed, 0 missing reference images
diff --git a/gm/tests/outputs/compared-against-identical-pixels/output-expected/command_line b/gm/tests/outputs/compared-against-identical-pixels/output-expected/command_line
deleted file mode 100644
index 086f984..0000000
--- a/gm/tests/outputs/compared-against-identical-pixels/output-expected/command_line
+++ /dev/null
@@ -1 +0,0 @@
-out/Debug/gm --hierarchy --match dashing2 --config 8888 --config 565 -r gm/tests/inputs/identical-pixels --writeJsonSummary gm/tests/outputs/compared-against-identical-pixels/output-actual/json-summary.txt
diff --git a/gm/tests/run.sh b/gm/tests/run.sh
index 717262e..e1b6ce7 100755
--- a/gm/tests/run.sh
+++ b/gm/tests/run.sh
@@ -85,8 +85,8 @@
compare_directories $EXPECTED_OUTPUT_DIR $ACTUAL_OUTPUT_DIR
}
-# Create input dir (at path $1) with images that match or mismatch
-# as appropriate.
+# Create input dir (at path $1) with expectations (both image and json)
+# that gm will match or mismatch as appropriate.
#
# We used to check these files into SVN, but then we needed to rebasline them
# when our drawing changed at all... so, as proposed in
@@ -98,29 +98,42 @@
exit 1
fi
INPUTS_DIR="$1"
- mkdir -p $INPUTS_DIR
+ IMAGES_DIR=$INPUTS_DIR/images
+ JSON_DIR=$INPUTS_DIR/json
+ mkdir -p $IMAGES_DIR $JSON_DIR
- mkdir -p $INPUTS_DIR/identical-bytes
+ mkdir -p $IMAGES_DIR/identical-bytes
+ # Run GM to write out the images actually generated.
$GM_BINARY --hierarchy --match dashing2 $CONFIGS \
- -w $INPUTS_DIR/identical-bytes
-
- mkdir -p $INPUTS_DIR/identical-pixels
+ -w $IMAGES_DIR/identical-bytes
+ # Run GM again to read in those images and write them out as a JSON summary.
$GM_BINARY --hierarchy --match dashing2 $CONFIGS \
- -w $INPUTS_DIR/identical-pixels
- echo "more bytes that do not change the image pixels" \
- >> $INPUTS_DIR/identical-pixels/8888/dashing2.png
- echo "more bytes that do not change the image pixels" \
- >> $INPUTS_DIR/identical-pixels/565/dashing2.png
+ -r $IMAGES_DIR/identical-bytes \
+ --writeJsonSummary $JSON_DIR/identical-bytes.json
- mkdir -p $INPUTS_DIR/different-pixels
+ mkdir -p $IMAGES_DIR/identical-pixels
+ $GM_BINARY --hierarchy --match dashing2 $CONFIGS \
+ -w $IMAGES_DIR/identical-pixels
+ echo "more bytes that do not change the image pixels" \
+ >> $IMAGES_DIR/identical-pixels/8888/dashing2.png
+ echo "more bytes that do not change the image pixels" \
+ >> $IMAGES_DIR/identical-pixels/565/dashing2.png
+ $GM_BINARY --hierarchy --match dashing2 $CONFIGS \
+ -r $IMAGES_DIR/identical-pixels \
+ --writeJsonSummary $JSON_DIR/identical-pixels.json
+
+ mkdir -p $IMAGES_DIR/different-pixels
$GM_BINARY --hierarchy --match dashing3 $CONFIGS \
- -w $INPUTS_DIR/different-pixels
- mv $INPUTS_DIR/different-pixels/8888/dashing3.png \
- $INPUTS_DIR/different-pixels/8888/dashing2.png
- mv $INPUTS_DIR/different-pixels/565/dashing3.png \
- $INPUTS_DIR/different-pixels/565/dashing2.png
+ -w $IMAGES_DIR/different-pixels
+ mv $IMAGES_DIR/different-pixels/8888/dashing3.png \
+ $IMAGES_DIR/different-pixels/8888/dashing2.png
+ mv $IMAGES_DIR/different-pixels/565/dashing3.png \
+ $IMAGES_DIR/different-pixels/565/dashing2.png
+ $GM_BINARY --hierarchy --match dashing2 $CONFIGS \
+ -r $IMAGES_DIR/different-pixels \
+ --writeJsonSummary $JSON_DIR/different-pixels.json
- mkdir -p $INPUTS_DIR/empty-dir
+ mkdir -p $IMAGES_DIR/empty-dir
}
GM_TESTDIR=gm/tests
@@ -131,16 +144,16 @@
create_inputs_dir $GM_INPUTS
# Compare generated image against an input image file with identical bytes.
-gm_test "--hierarchy --match dashing2 $CONFIGS -r $GM_INPUTS/identical-bytes" "$GM_OUTPUTS/compared-against-identical-bytes"
+gm_test "--hierarchy --match dashing2 $CONFIGS -r $GM_INPUTS/images/identical-bytes" "$GM_OUTPUTS/compared-against-identical-bytes-images"
# Compare generated image against an input image file with identical pixels but different PNG encoding.
-gm_test "--hierarchy --match dashing2 $CONFIGS -r $GM_INPUTS/identical-pixels" "$GM_OUTPUTS/compared-against-identical-pixels"
+gm_test "--hierarchy --match dashing2 $CONFIGS -r $GM_INPUTS/images/identical-pixels" "$GM_OUTPUTS/compared-against-identical-pixels-images"
# Compare generated image against an input image file with different pixels.
-gm_test "--hierarchy --match dashing2 $CONFIGS -r $GM_INPUTS/different-pixels" "$GM_OUTPUTS/compared-against-different-pixels"
+gm_test "--hierarchy --match dashing2 $CONFIGS -r $GM_INPUTS/images/different-pixels" "$GM_OUTPUTS/compared-against-different-pixels-images"
# Compare generated image against an empty "expected image" dir.
-gm_test "--hierarchy --match dashing2 $CONFIGS -r $GM_INPUTS/empty-dir" "$GM_OUTPUTS/compared-against-empty-dir"
+gm_test "--hierarchy --match dashing2 $CONFIGS -r $GM_INPUTS/images/empty-dir" "$GM_OUTPUTS/compared-against-empty-dir"
# If run without "-r", the JSON's "actual-results" section should contain
# actual checksums marked as "failure-ignored", but the "expected-results"