make gm self-test more robust
BUG=http://code.google.com/p/skia/issues/detail?id=1068
Review URL: https://codereview.appspot.com/7164043
git-svn-id: http://skia.googlecode.com/svn/trunk@7284 2bbb7eff-a529-9590-31e7-b0007b416f81
diff --git a/gm/tests/inputs/different-pixels/8888/dashing2.png b/gm/tests/inputs/different-pixels/8888/dashing2.png
deleted file mode 100644
index 3a0bc2e..0000000
--- a/gm/tests/inputs/different-pixels/8888/dashing2.png
+++ /dev/null
Binary files differ
diff --git a/gm/tests/inputs/empty-dir/README b/gm/tests/inputs/empty-dir/README
deleted file mode 100644
index 4d39134..0000000
--- a/gm/tests/inputs/empty-dir/README
+++ /dev/null
@@ -1 +0,0 @@
-This directory intentionally left empty. Except for this file.
diff --git a/gm/tests/inputs/identical-bytes/8888/dashing2.png b/gm/tests/inputs/identical-bytes/8888/dashing2.png
deleted file mode 100644
index cb858d7..0000000
--- a/gm/tests/inputs/identical-bytes/8888/dashing2.png
+++ /dev/null
Binary files differ
diff --git a/gm/tests/inputs/identical-pixels/8888/dashing2.png b/gm/tests/inputs/identical-pixels/8888/dashing2.png
deleted file mode 100644
index 86d5bce..0000000
--- a/gm/tests/inputs/identical-pixels/8888/dashing2.png
+++ /dev/null
Binary files differ
diff --git a/gm/tests/outputs/compared-against-different-pixels/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-different-pixels/output-expected/json-summary.txt
index 182dd0e..d6c59c0 100644
--- a/gm/tests/outputs/compared-against-different-pixels/output-expected/json-summary.txt
+++ b/gm/tests/outputs/compared-against-different-pixels/output-expected/json-summary.txt
@@ -2,7 +2,7 @@
"actual-results" : {
"failed" : {
"8888/dashing2" : {
- "checksum" : 16595425910463109922
+ "checksum" : FAKE
}
},
"failure-ignored" : null,
@@ -11,7 +11,7 @@
},
"expected-results" : {
"8888/dashing2" : {
- "checksums" : [ 15161495552186645995 ],
+ "checksums" : [ FAKE ],
"ignore-failure" : false
}
}
diff --git a/gm/tests/outputs/compared-against-different-pixels/output-expected/stdout b/gm/tests/outputs/compared-against-different-pixels/output-expected/stdout
index e4dbd8c..4286eb5 100644
--- a/gm/tests/outputs/compared-against-different-pixels/output-expected/stdout
+++ b/gm/tests/outputs/compared-against-different-pixels/output-expected/stdout
@@ -1,6 +1,4 @@
-Non-default runtime configuration options:
reading from gm/tests/inputs/different-pixels
writing to gm/tests/outputs/compared-against-different-pixels/output-actual/images
drawing... dashing2 [640 480]
Ran 1 tests: 0 passed, 1 failed, 0 missing reference images
- 8888/dashing2 pixel_error
diff --git a/gm/tests/outputs/compared-against-empty-dir/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-empty-dir/output-expected/json-summary.txt
index 226b087..73ff348 100644
--- a/gm/tests/outputs/compared-against-empty-dir/output-expected/json-summary.txt
+++ b/gm/tests/outputs/compared-against-empty-dir/output-expected/json-summary.txt
@@ -4,7 +4,7 @@
"failure-ignored" : null,
"no-comparison" : {
"8888/dashing2" : {
- "checksum" : 16595425910463109922
+ "checksum" : FAKE
}
},
"succeeded" : null
diff --git a/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout b/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout
index 8621668..436554d 100644
--- a/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout
+++ b/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout
@@ -1,4 +1,3 @@
-Non-default runtime configuration options:
reading from gm/tests/inputs/empty-dir
writing to gm/tests/outputs/compared-against-empty-dir/output-actual/images
drawing... dashing2 [640 480]
diff --git a/gm/tests/outputs/compared-against-identical-bytes/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-identical-bytes/output-expected/json-summary.txt
index 6f812e0..9156fb4 100644
--- a/gm/tests/outputs/compared-against-identical-bytes/output-expected/json-summary.txt
+++ b/gm/tests/outputs/compared-against-identical-bytes/output-expected/json-summary.txt
@@ -5,13 +5,13 @@
"no-comparison" : null,
"succeeded" : {
"8888/dashing2" : {
- "checksum" : 16595425910463109922
+ "checksum" : FAKE
}
}
},
"expected-results" : {
"8888/dashing2" : {
- "checksums" : [ 16595425910463109922 ],
+ "checksums" : [ FAKE ],
"ignore-failure" : false
}
}
diff --git a/gm/tests/outputs/compared-against-identical-bytes/output-expected/stdout b/gm/tests/outputs/compared-against-identical-bytes/output-expected/stdout
index 8e67783..301f17f 100644
--- a/gm/tests/outputs/compared-against-identical-bytes/output-expected/stdout
+++ b/gm/tests/outputs/compared-against-identical-bytes/output-expected/stdout
@@ -1,4 +1,3 @@
-Non-default runtime configuration options:
reading from gm/tests/inputs/identical-bytes
writing to gm/tests/outputs/compared-against-identical-bytes/output-actual/images
drawing... dashing2 [640 480]
diff --git a/gm/tests/outputs/compared-against-identical-pixels/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-identical-pixels/output-expected/json-summary.txt
index 6f812e0..9156fb4 100644
--- a/gm/tests/outputs/compared-against-identical-pixels/output-expected/json-summary.txt
+++ b/gm/tests/outputs/compared-against-identical-pixels/output-expected/json-summary.txt
@@ -5,13 +5,13 @@
"no-comparison" : null,
"succeeded" : {
"8888/dashing2" : {
- "checksum" : 16595425910463109922
+ "checksum" : FAKE
}
}
},
"expected-results" : {
"8888/dashing2" : {
- "checksums" : [ 16595425910463109922 ],
+ "checksums" : [ FAKE ],
"ignore-failure" : false
}
}
diff --git a/gm/tests/outputs/compared-against-identical-pixels/output-expected/stdout b/gm/tests/outputs/compared-against-identical-pixels/output-expected/stdout
index 8829ef5..00868ac 100644
--- a/gm/tests/outputs/compared-against-identical-pixels/output-expected/stdout
+++ b/gm/tests/outputs/compared-against-identical-pixels/output-expected/stdout
@@ -1,4 +1,3 @@
-Non-default runtime configuration options:
reading from gm/tests/inputs/identical-pixels
writing to gm/tests/outputs/compared-against-identical-pixels/output-actual/images
drawing... dashing2 [640 480]
diff --git a/gm/tests/outputs/no-readpath/output-expected/json-summary.txt b/gm/tests/outputs/no-readpath/output-expected/json-summary.txt
index 3b93856..04a042c 100644
--- a/gm/tests/outputs/no-readpath/output-expected/json-summary.txt
+++ b/gm/tests/outputs/no-readpath/output-expected/json-summary.txt
@@ -4,7 +4,7 @@
"failure-ignored" : null,
"no-comparison" : {
"8888/dashing2" : {
- "checksum" : 16595425910463109922
+ "checksum" : FAKE
}
},
"succeeded" : null
diff --git a/gm/tests/outputs/no-readpath/output-expected/stdout b/gm/tests/outputs/no-readpath/output-expected/stdout
index 61b3131..4781221 100644
--- a/gm/tests/outputs/no-readpath/output-expected/stdout
+++ b/gm/tests/outputs/no-readpath/output-expected/stdout
@@ -1,4 +1,3 @@
-Non-default runtime configuration options:
writing to gm/tests/outputs/no-readpath/output-actual/images
drawing... dashing2 [640 480]
Ran 1 tests: 0 passed, 0 failed, 1 missing reference images
diff --git a/gm/tests/run.sh b/gm/tests/run.sh
index 2c9b719..2bcff4e 100755
--- a/gm/tests/run.sh
+++ b/gm/tests/run.sh
@@ -20,6 +20,9 @@
# TODO(epoger): make it look in Release and/or Debug
GM_BINARY=out/Debug/gm
+OUTPUT_ACTUAL_SUBDIR=output-actual
+OUTPUT_EXPECTED_SUBDIR=output-expected
+
# Compare contents of all files within directories $1 and $2,
# EXCEPT for any dotfiles.
# If there are any differences, a description is written to stdout and
@@ -39,40 +42,89 @@
# Run gm...
# - with the arguments in $1
-# - writing resulting images into $2/output-actual/images
-# - writing stdout into $2/output-actual/stdout
-# - writing json summary into $2/output-actual/json-summary.txt
-# - writing return value into $2/output-actual/return_value
-# Then compare all of those against $2/output-expected .
+# - writing resulting images into $2/$OUTPUT_ACTUAL_SUBDIR/images
+# - writing stdout into $2/$OUTPUT_ACTUAL_SUBDIR/stdout
+# - writing json summary into $2/$OUTPUT_ACTUAL_SUBDIR/json-summary.txt
+# - writing return value into $2/$OUTPUT_ACTUAL_SUBDIR/return_value
+# Then compare all of those against $2/$OUTPUT_EXPECTED_SUBDIR .
function gm_test {
if [ $# != 2 ]; then
echo "gm_test requires exactly 2 parameters, got $#"
exit 1
fi
GM_ARGS="$1"
- ACTUAL_OUTPUT_DIR="$2/output-actual"
- EXPECTED_OUTPUT_DIR="$2/output-expected"
+ ACTUAL_OUTPUT_DIR="$2/$OUTPUT_ACTUAL_SUBDIR"
+ EXPECTED_OUTPUT_DIR="$2/$OUTPUT_EXPECTED_SUBDIR"
+ JSON_SUMMARY_FILE="$ACTUAL_OUTPUT_DIR/json-summary.txt"
rm -rf $ACTUAL_OUTPUT_DIR
mkdir -p $ACTUAL_OUTPUT_DIR
- COMMAND="$GM_BINARY $GM_ARGS --writeJsonSummary $ACTUAL_OUTPUT_DIR/json-summary.txt -w $ACTUAL_OUTPUT_DIR/images"
+ COMMAND="$GM_BINARY $GM_ARGS --writeJsonSummary $JSON_SUMMARY_FILE -w $ACTUAL_OUTPUT_DIR/images"
echo "$COMMAND" >$ACTUAL_OUTPUT_DIR/command_line
$COMMAND &>$ACTUAL_OUTPUT_DIR/stdout
echo $? >$ACTUAL_OUTPUT_DIR/return_value
- # Temporary hack to alleviate
- # http://code.google.com/p/skia/issues/detail?id=1068 ('GM self-test failures')
- # (Remove instance-counting lines from the Debug output)
- grep -v Leaked $ACTUAL_OUTPUT_DIR/stdout >$ACTUAL_OUTPUT_DIR/stdout-tmp
+ # Only compare selected lines in the output, to ignore any spurious lines
+ # as noted in http://code.google.com/p/skia/issues/detail?id=1068 .
+ #
+ # TODO(epoger): This is still hacky... we need to rewrite this script in
+ # Python soon, and make stuff like this more maintainable.
+ grep --regexp=^reading --regexp=^writing --regexp=^drawing \
+ --regexp=^FAILED --regexp=^Ran $ACTUAL_OUTPUT_DIR/stdout \
+ >$ACTUAL_OUTPUT_DIR/stdout-tmp
mv $ACTUAL_OUTPUT_DIR/stdout-tmp $ACTUAL_OUTPUT_DIR/stdout
+ # Replace particular checksums in json output with a placeholder, so
+ # we don't need to rebaseline these json files when our drawing routines
+ # change.
+ sed -e 's/"checksum" : [0-9]*/"checksum" : FAKE/g' \
+ --in-place $JSON_SUMMARY_FILE
+ sed -e 's/"checksums" : \[ [0-9]* \]/"checksums" : [ FAKE ]/g' \
+ --in-place $JSON_SUMMARY_FILE
+
compare_directories $EXPECTED_OUTPUT_DIR $ACTUAL_OUTPUT_DIR
}
+# Create input dir (at path $1) with images that match or mismatch
+# as appropriate.
+#
+# We used to check these files into SVN, but then we needed to rebasline them
+# when our drawing changed at all... so, as proposed in
+# http://code.google.com/p/skia/issues/detail?id=1068 , we generate them
+# new each time.
+function create_inputs_dir {
+ if [ $# != 1 ]; then
+ echo "create_inputs_dir requires exactly 1 parameter, got $#"
+ exit 1
+ fi
+ INPUTS_DIR="$1"
+ mkdir -p $INPUTS_DIR
+
+ mkdir -p $INPUTS_DIR/identical-bytes
+ $GM_BINARY --hierarchy --match dashing2 --config 8888 \
+ -w $INPUTS_DIR/identical-bytes
+
+ mkdir -p $INPUTS_DIR/identical-pixels
+ $GM_BINARY --hierarchy --match dashing2 --config 8888 \
+ -w $INPUTS_DIR/identical-pixels
+ echo "more bytes that do not change the image pixels" \
+ >> $INPUTS_DIR/identical-pixels/8888/dashing2.png
+
+ mkdir -p $INPUTS_DIR/different-pixels
+ $GM_BINARY --hierarchy --match dashing3 --config 8888 \
+ -w $INPUTS_DIR/different-pixels
+ mv $INPUTS_DIR/different-pixels/8888/dashing3.png \
+ $INPUTS_DIR/different-pixels/8888/dashing2.png
+
+ mkdir -p $INPUTS_DIR/empty-dir
+}
+
GM_TESTDIR=gm/tests
GM_INPUTS=$GM_TESTDIR/inputs
GM_OUTPUTS=$GM_TESTDIR/outputs
+create_inputs_dir $GM_INPUTS
+
# Compare generated image against an input image file with identical bytes.
gm_test "--hierarchy --match dashing2 --config 8888 -r $GM_INPUTS/identical-bytes" "$GM_OUTPUTS/compared-against-identical-bytes"
