gm: hide list of tests with ignored errors, by default
Review URL: https://codereview.chromium.org/13811031
git-svn-id: http://skia.googlecode.com/svn/trunk@8591 2bbb7eff-a529-9590-31e7-b0007b416f81
diff --git a/gm/gmmain.cpp b/gm/gmmain.cpp
index cf0ab41..5b0f662 100644
--- a/gm/gmmain.cpp
+++ b/gm/gmmain.cpp
@@ -299,9 +299,40 @@
}
/**
- * List contents of fFailedTests to stdout.
+ * Display the summary of results with this ErrorType.
+ *
+ * @param type which ErrorType
+ * @param verbose whether to be all verbose about it
*/
- void ListErrors() {
+ void DisplayResultTypeSummary(ErrorType type, bool verbose) {
+ bool isIgnorableType = fIgnorableErrorCombination.includes(type);
+
+ SkString line;
+ if (isIgnorableType) {
+ line.append("[ ] ");
+ } else {
+ line.append("[*] ");
+ }
+
+ SkTArray<SkString> *failedTestsOfThisType = &fFailedTests[type];
+ int count = failedTestsOfThisType->count();
+ line.appendf("%d %s", count, getErrorTypeName(type));
+ if (!isIgnorableType || verbose) {
+ line.append(":");
+ for (int i = 0; i < count; ++i) {
+ line.append(" ");
+ line.append((*failedTestsOfThisType)[i]);
+ }
+ }
+ gm_fprintf(stdout, "%s\n", line.c_str());
+ }
+
+ /**
+ * List contents of fFailedTests to stdout.
+ *
+ * @param verbose whether to be all verbose about it
+ */
+ void ListErrors(bool verbose) {
// First, print a single summary line.
SkString summary;
summary.appendf("Ran %d tests:", fTestsRun);
@@ -313,22 +344,7 @@
// Now, for each failure type, list the tests that failed that way.
for (int typeInt = 0; typeInt <= kLast_ErrorType; typeInt++) {
- SkString line;
- ErrorType type = static_cast<ErrorType>(typeInt);
- if (fIgnorableErrorCombination.includes(type)) {
- line.append("[ ] ");
- } else {
- line.append("[*] ");
- }
-
- SkTArray<SkString> *failedTestsOfThisType = &fFailedTests[type];
- int count = failedTestsOfThisType->count();
- line.appendf("%d %s:", count, getErrorTypeName(type));
- for (int i = 0; i < count; ++i) {
- line.append(" ");
- line.append((*failedTestsOfThisType)[i]);
- }
- gm_fprintf(stdout, "%s\n", line.c_str());
+ this->DisplayResultTypeSummary(static_cast<ErrorType>(typeInt), verbose);
}
gm_fprintf(stdout, "(results marked with [*] will cause nonzero return value)\n");
}
@@ -1686,15 +1702,6 @@
GrContextFactory* grFactory = NULL;
#endif
- if (FLAGS_verbose) {
- SkString str;
- str.printf("%d configs:", configs.count());
- for (int i = 0; i < configs.count(); ++i) {
- str.appendf(" %s", gRec[configs[i]].fName);
- }
- gm_fprintf(stderr, "%s\n", str.c_str());
- }
-
if (FLAGS_resourcePath.count() == 1) {
GM::SetResourcePath(FLAGS_resourcePath[0]);
}
@@ -1798,15 +1805,17 @@
if (gmmain.NumSignificantErrors() > 0) {
reportError = true;
}
+ int expectedNumberOfTests = gmsRun * (configs.count() + modes.count());
// Output summary to stdout.
- gm_fprintf(stdout, "Ran %d GMs\n", gmsRun);
- gm_fprintf(stdout, "... over %2d configs [%s]\n", configs.count(),
- list_all_config_names(configs).c_str());
- gm_fprintf(stdout, "... and %2d modes [%s]\n", modes.count(), list_all(modes).c_str());
- int expectedNumberOfTests = gmsRun * (configs.count() + modes.count());
- gm_fprintf(stdout, "... so there should be a total of %d tests.\n", expectedNumberOfTests);
- gmmain.ListErrors();
+ if (FLAGS_verbose) {
+ gm_fprintf(stdout, "Ran %d GMs\n", gmsRun);
+ gm_fprintf(stdout, "... over %2d configs [%s]\n", configs.count(),
+ list_all_config_names(configs).c_str());
+ gm_fprintf(stdout, "... and %2d modes [%s]\n", modes.count(), list_all(modes).c_str());
+ gm_fprintf(stdout, "... so there should be a total of %d tests.\n", expectedNumberOfTests);
+ }
+ gmmain.ListErrors(FLAGS_verbose);
// TODO(epoger): in a standalone CL, enable this new check.
#if 0
diff --git a/gm/tests/outputs/compared-against-different-pixels-images/output-expected/command_line b/gm/tests/outputs/compared-against-different-pixels-images/output-expected/command_line
index fcf9744..8b4bf99 100644
--- a/gm/tests/outputs/compared-against-different-pixels-images/output-expected/command_line
+++ b/gm/tests/outputs/compared-against-different-pixels-images/output-expected/command_line
@@ -1 +1 @@
-out/Debug/gm --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/images/different-pixels --writeJsonSummaryPath gm/tests/outputs/compared-against-different-pixels-images/output-actual/json-summary.txt
+out/Debug/gm --verbose --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/images/different-pixels --writeJsonSummaryPath gm/tests/outputs/compared-against-different-pixels-images/output-actual/json-summary.txt
diff --git a/gm/tests/outputs/compared-against-different-pixels-json/output-expected/command_line b/gm/tests/outputs/compared-against-different-pixels-json/output-expected/command_line
index be84052..73227a9 100644
--- a/gm/tests/outputs/compared-against-different-pixels-json/output-expected/command_line
+++ b/gm/tests/outputs/compared-against-different-pixels-json/output-expected/command_line
@@ -1 +1 @@
-out/Debug/gm --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/json/different-pixels.json --writeJsonSummaryPath gm/tests/outputs/compared-against-different-pixels-json/output-actual/json-summary.txt
+out/Debug/gm --verbose --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/json/different-pixels.json --writeJsonSummaryPath gm/tests/outputs/compared-against-different-pixels-json/output-actual/json-summary.txt
diff --git a/gm/tests/outputs/compared-against-empty-dir/output-expected/command_line b/gm/tests/outputs/compared-against-empty-dir/output-expected/command_line
index ad626bc..4151af3 100644
--- a/gm/tests/outputs/compared-against-empty-dir/output-expected/command_line
+++ b/gm/tests/outputs/compared-against-empty-dir/output-expected/command_line
@@ -1 +1 @@
-out/Debug/gm --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/images/empty-dir --writeJsonSummaryPath gm/tests/outputs/compared-against-empty-dir/output-actual/json-summary.txt
+out/Debug/gm --verbose --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/images/empty-dir --writeJsonSummaryPath gm/tests/outputs/compared-against-empty-dir/output-actual/json-summary.txt
diff --git a/gm/tests/outputs/compared-against-identical-bytes-images/output-expected/command_line b/gm/tests/outputs/compared-against-identical-bytes-images/output-expected/command_line
index 614cb2b..ab8bac6 100644
--- a/gm/tests/outputs/compared-against-identical-bytes-images/output-expected/command_line
+++ b/gm/tests/outputs/compared-against-identical-bytes-images/output-expected/command_line
@@ -1 +1 @@
-out/Debug/gm --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/images/identical-bytes --writeJsonSummaryPath gm/tests/outputs/compared-against-identical-bytes-images/output-actual/json-summary.txt
+out/Debug/gm --verbose --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/images/identical-bytes --writeJsonSummaryPath gm/tests/outputs/compared-against-identical-bytes-images/output-actual/json-summary.txt
diff --git a/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/command_line b/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/command_line
index aae0b60..8f34c11 100644
--- a/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/command_line
+++ b/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/command_line
@@ -1 +1 @@
-out/Debug/gm --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/json/identical-bytes.json --writeJsonSummaryPath gm/tests/outputs/compared-against-identical-bytes-json/output-actual/json-summary.txt
+out/Debug/gm --verbose --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/json/identical-bytes.json --writeJsonSummaryPath gm/tests/outputs/compared-against-identical-bytes-json/output-actual/json-summary.txt
diff --git a/gm/tests/outputs/compared-against-identical-pixels-images/output-expected/command_line b/gm/tests/outputs/compared-against-identical-pixels-images/output-expected/command_line
index 34fcc98..23f498e 100644
--- a/gm/tests/outputs/compared-against-identical-pixels-images/output-expected/command_line
+++ b/gm/tests/outputs/compared-against-identical-pixels-images/output-expected/command_line
@@ -1 +1 @@
-out/Debug/gm --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/images/identical-pixels --writeJsonSummaryPath gm/tests/outputs/compared-against-identical-pixels-images/output-actual/json-summary.txt
+out/Debug/gm --verbose --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/images/identical-pixels --writeJsonSummaryPath gm/tests/outputs/compared-against-identical-pixels-images/output-actual/json-summary.txt
diff --git a/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/command_line b/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/command_line
index a7a9c82..9fe0158 100644
--- a/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/command_line
+++ b/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/command_line
@@ -1 +1 @@
-out/Debug/gm --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/json/identical-pixels.json --writeJsonSummaryPath gm/tests/outputs/compared-against-identical-pixels-json/output-actual/json-summary.txt
+out/Debug/gm --verbose --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/json/identical-pixels.json --writeJsonSummaryPath gm/tests/outputs/compared-against-identical-pixels-json/output-actual/json-summary.txt
diff --git a/gm/tests/outputs/intentionally-skipped-tests/output-expected/command_line b/gm/tests/outputs/intentionally-skipped-tests/output-expected/command_line
index c80a9a2..361cb52 100644
--- a/gm/tests/outputs/intentionally-skipped-tests/output-expected/command_line
+++ b/gm/tests/outputs/intentionally-skipped-tests/output-expected/command_line
@@ -1 +1 @@
-out/Debug/gm --hierarchy --match selftest1 selftest2 --config 8888 565 --writeJsonSummaryPath gm/tests/outputs/intentionally-skipped-tests/output-actual/json-summary.txt
+out/Debug/gm --verbose --hierarchy --match selftest1 selftest2 --config 8888 565 --writeJsonSummaryPath gm/tests/outputs/intentionally-skipped-tests/output-actual/json-summary.txt
diff --git a/gm/tests/outputs/no-readpath/output-expected/command_line b/gm/tests/outputs/no-readpath/output-expected/command_line
index a75c735..3192ddb 100644
--- a/gm/tests/outputs/no-readpath/output-expected/command_line
+++ b/gm/tests/outputs/no-readpath/output-expected/command_line
@@ -1 +1 @@
-out/Debug/gm --hierarchy --match selftest1 --config 8888 565 --writeJsonSummaryPath gm/tests/outputs/no-readpath/output-actual/json-summary.txt
+out/Debug/gm --verbose --hierarchy --match selftest1 --config 8888 565 --writeJsonSummaryPath gm/tests/outputs/no-readpath/output-actual/json-summary.txt
diff --git a/gm/tests/outputs/nonverbose/output-expected/command_line b/gm/tests/outputs/nonverbose/output-expected/command_line
new file mode 100644
index 0000000..7c71a72
--- /dev/null
+++ b/gm/tests/outputs/nonverbose/output-expected/command_line
@@ -0,0 +1 @@
+out/Debug/gm --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/images/empty-dir --writeJsonSummaryPath gm/tests/outputs/nonverbose/output-actual/json-summary.txt
diff --git a/gm/tests/outputs/nonverbose/output-expected/json-summary.txt b/gm/tests/outputs/nonverbose/output-expected/json-summary.txt
new file mode 100644
index 0000000..69ef127
--- /dev/null
+++ b/gm/tests/outputs/nonverbose/output-expected/json-summary.txt
@@ -0,0 +1,25 @@
+{
+ "actual-results" : {
+ "failed" : null,
+ "failure-ignored" : null,
+ "no-comparison" : {
+ "565/selftest1" : {
+ "checksum" : 9512553915271796906
+ },
+ "8888/selftest1" : {
+ "checksum" : 14022967492765711532
+ }
+ },
+ "succeeded" : null
+ },
+ "expected-results" : {
+ "565/selftest1" : {
+ "checksums" : null,
+ "ignore-failure" : false
+ },
+ "8888/selftest1" : {
+ "checksums" : null,
+ "ignore-failure" : false
+ }
+ }
+}
diff --git a/gm/tests/outputs/nonverbose/output-expected/return_value b/gm/tests/outputs/nonverbose/output-expected/return_value
new file mode 100644
index 0000000..573541a
--- /dev/null
+++ b/gm/tests/outputs/nonverbose/output-expected/return_value
@@ -0,0 +1 @@
+0
diff --git a/gm/tests/outputs/nonverbose/output-expected/stderr b/gm/tests/outputs/nonverbose/output-expected/stderr
new file mode 100644
index 0000000..2e951cf
--- /dev/null
+++ b/gm/tests/outputs/nonverbose/output-expected/stderr
@@ -0,0 +1,2 @@
+GM: FAILED to read gm/tests/inputs/images/empty-dir/8888/selftest1.png
+GM: FAILED to read gm/tests/inputs/images/empty-dir/565/selftest1.png
diff --git a/gm/tests/outputs/nonverbose/output-expected/stdout b/gm/tests/outputs/nonverbose/output-expected/stdout
new file mode 100644
index 0000000..6139fdb
--- /dev/null
+++ b/gm/tests/outputs/nonverbose/output-expected/stdout
@@ -0,0 +1,10 @@
+GM: reading from gm/tests/inputs/images/empty-dir
+GM: drawing... selftest1 [300 200]
+GM: Ran 9 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 ExpectationsMismatch=0 MissingExpectations=2 WritingReferenceImage=0
+GM: [*] 0 NoGpuContext:
+GM: [ ] 0 IntentionallySkipped
+GM: [*] 0 RenderModeMismatch:
+GM: [*] 0 ExpectationsMismatch:
+GM: [ ] 2 MissingExpectations
+GM: [*] 0 WritingReferenceImage:
+GM: (results marked with [*] will cause nonzero return value)
diff --git a/gm/tests/outputs/pipe-playback-failure/output-expected/command_line b/gm/tests/outputs/pipe-playback-failure/output-expected/command_line
index 950339d..28c5a67 100644
--- a/gm/tests/outputs/pipe-playback-failure/output-expected/command_line
+++ b/gm/tests/outputs/pipe-playback-failure/output-expected/command_line
@@ -1 +1 @@
-out/Debug/gm --simulatePipePlaybackFailure --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/json/identical-pixels.json --writeJsonSummaryPath gm/tests/outputs/pipe-playback-failure/output-actual/json-summary.txt
+out/Debug/gm --simulatePipePlaybackFailure --verbose --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/json/identical-pixels.json --writeJsonSummaryPath gm/tests/outputs/pipe-playback-failure/output-actual/json-summary.txt
diff --git a/gm/tests/run.sh b/gm/tests/run.sh
index 8348cce..844294a 100755
--- a/gm/tests/run.sh
+++ b/gm/tests/run.sh
@@ -137,29 +137,32 @@
create_inputs_dir $GM_INPUTS
# Compare generated image against an input image file with identical bytes.
-gm_test "--hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/identical-bytes" "$GM_OUTPUTS/compared-against-identical-bytes-images"
-gm_test "--hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/identical-bytes.json" "$GM_OUTPUTS/compared-against-identical-bytes-json"
+gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/identical-bytes" "$GM_OUTPUTS/compared-against-identical-bytes-images"
+gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/identical-bytes.json" "$GM_OUTPUTS/compared-against-identical-bytes-json"
# Compare generated image against an input image file with identical pixels but different PNG encoding.
-gm_test "--hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/identical-pixels" "$GM_OUTPUTS/compared-against-identical-pixels-images"
-gm_test "--hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/identical-pixels.json" "$GM_OUTPUTS/compared-against-identical-pixels-json"
+gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/identical-pixels" "$GM_OUTPUTS/compared-against-identical-pixels-images"
+gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/identical-pixels.json" "$GM_OUTPUTS/compared-against-identical-pixels-json"
# Compare generated image against an input image file with different pixels.
-gm_test "--hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/different-pixels" "$GM_OUTPUTS/compared-against-different-pixels-images"
-gm_test "--hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels.json" "$GM_OUTPUTS/compared-against-different-pixels-json"
+gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/different-pixels" "$GM_OUTPUTS/compared-against-different-pixels-images"
+gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels.json" "$GM_OUTPUTS/compared-against-different-pixels-json"
# Compare generated image against an empty "expected image" dir.
-gm_test "--hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/empty-dir" "$GM_OUTPUTS/compared-against-empty-dir"
+gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/empty-dir" "$GM_OUTPUTS/compared-against-empty-dir"
+
+# Compare generated image against an empty "expected image" dir, but NOT in verbose mode.
+gm_test "--hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/empty-dir" "$GM_OUTPUTS/nonverbose"
# If run without "-r", the JSON's "actual-results" section should contain
# actual checksums marked as "failure-ignored", but the "expected-results"
# section should be empty.
-gm_test "--hierarchy --match selftest1 $CONFIGS" "$GM_OUTPUTS/no-readpath"
+gm_test "--verbose --hierarchy --match selftest1 $CONFIGS" "$GM_OUTPUTS/no-readpath"
# Test what happens if a subset of the renderModes fail (e.g. pipe)
-gm_test "--simulatePipePlaybackFailure --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/identical-pixels.json" "$GM_OUTPUTS/pipe-playback-failure"
+gm_test "--simulatePipePlaybackFailure --verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/identical-pixels.json" "$GM_OUTPUTS/pipe-playback-failure"
# Confirm that IntentionallySkipped tests are recorded as such.
-gm_test "--hierarchy --match selftest1 selftest2 $CONFIGS" "$GM_OUTPUTS/intentionally-skipped-tests"
+gm_test "--verbose --hierarchy --match selftest1 selftest2 $CONFIGS" "$GM_OUTPUTS/intentionally-skipped-tests"
echo "All tests passed."