Improve tool that analyzes gm JSON summary
BUG=https://code.google.com/p/skia/issues/detail?id=1300
R=borenet@google.com
Review URL: https://codereview.chromium.org/15298010
git-svn-id: http://skia.googlecode.com/svn/trunk@9217 2bbb7eff-a529-9590-31e7-b0007b416f81
diff --git a/gm/confirm_no_failures_in_json.py b/gm/confirm_no_failures_in_json.py
deleted file mode 100644
index c849926..0000000
--- a/gm/confirm_no_failures_in_json.py
+++ /dev/null
@@ -1,47 +0,0 @@
-#!/usr/bin/env python
-# Copyright (c) 2013 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Utility to confirm that a JSON summary written by GM contains no failures.
-
-Usage:
- python confirm_no_failures_in_json.py <filename>
-"""
-
-__author__ = 'Elliot Poger'
-
-
-import json
-import sys
-
-
-# These constants must be kept in sync with the kJsonKey_ constants in
-# gm_expectations.cpp !
-JSONKEY_ACTUALRESULTS = 'actual-results'
-JSONKEY_ACTUALRESULTS_FAILED = 'failed'
-
-# This is the same indent level as used by jsoncpp, just for consistency.
-JSON_INDENTLEVEL = 3
-
-
-def Assert(filepath):
- """Raises an exception if the JSON summary at filepath contains any failed
- tests, or if we were unable to read the JSON summary."""
- failed_tests = GetFailedTests(filepath)
- if failed_tests:
- raise Exception('JSON file %s contained these test failures...\n%s' % (
- filepath, json.dumps(failed_tests, indent=JSON_INDENTLEVEL)))
-
-
-def GetFailedTests(filepath):
- """Returns the dictionary of failed tests from the JSON file at filepath."""
- json_dict = json.load(open(filepath))
- actual_results = json_dict[JSONKEY_ACTUALRESULTS]
- return actual_results[JSONKEY_ACTUALRESULTS_FAILED]
-
-
-if '__main__' == __name__:
- if len(sys.argv) != 2:
- raise Exception('usage: %s <input-json-filepath>' % sys.argv[0])
- Assert(sys.argv[1])
diff --git a/gm/display_json_results.py b/gm/display_json_results.py
new file mode 100644
index 0000000..00c0c0e
--- /dev/null
+++ b/gm/display_json_results.py
@@ -0,0 +1,120 @@
+#!/usr/bin/env python
+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Utility to display a summary of JSON-format GM results, and exit with
+a nonzero errorcode if there were non-ignored failures in the GM results.
+
+Usage:
+ python display_json_results.py <filename>
+
+TODO(epoger): We may want to add flags to set the following:
+- which error types cause a nonzero return code
+- maximum number of tests to list for any one ResultAccumulator
+ (to keep the output reasonably short)
+"""
+
+__author__ = 'Elliot Poger'
+
+
+import json
+import sys
+
+
+# These constants must be kept in sync with the kJsonKey_ constants in
+# gm_expectations.cpp !
+JSONKEY_ACTUALRESULTS = 'actual-results'
+JSONKEY_ACTUALRESULTS_FAILED = 'failed'
+JSONKEY_ACTUALRESULTS_FAILUREIGNORED = 'failure-ignored'
+JSONKEY_ACTUALRESULTS_NOCOMPARISON = 'no-comparison'
+JSONKEY_ACTUALRESULTS_SUCCEEDED = 'succeeded'
+
+
+class ResultAccumulator(object):
+ """Object that accumulates results of a given type, and can generate a
+ summary upon request."""
+
+ def __init__(self, name, do_list, do_fail):
+ """name: name of the category this result type falls into
+ do_list: whether to list all of the tests with this results type
+ do_fail: whether to return with nonzero exit code if there are any
+ results of this type
+ """
+ self._name = name
+ self._do_list = do_list
+ self._do_fail = do_fail
+ self._testnames = []
+
+ def AddResult(self, testname):
+ """Adds a result of this particular type.
+ testname: (string) name of the test"""
+ self._testnames.append(testname)
+
+ def ShouldSignalFailure(self):
+ """Returns true if this result type is serious (self._do_fail is True)
+ and there were any results of this type."""
+ if self._do_fail and self._testnames:
+ return True
+ else:
+ return False
+
+ def GetSummaryLine(self):
+ """Returns a single-line string summary of all results added to this
+ accumulator so far."""
+ summary = ''
+ if self._do_fail:
+ summary += '[*] '
+ else:
+ summary += '[ ] '
+ summary += str(len(self._testnames))
+ summary += ' '
+ summary += self._name
+ if self._do_list:
+ summary += ': '
+ for testname in self._testnames:
+ summary += testname
+ summary += ' '
+ return summary
+
+
+def Display(filepath):
+ """Displays a summary of the results in a JSON file.
+ Returns True if the results are free of any significant failures.
+ filepath: (string) path to JSON file"""
+
+ # Map labels within the JSON file to the ResultAccumulator for each label.
+ results_map = {
+ JSONKEY_ACTUALRESULTS_FAILED:
+ ResultAccumulator(name='ExpectationsMismatch',
+ do_list=True, do_fail=True),
+ JSONKEY_ACTUALRESULTS_FAILUREIGNORED:
+ ResultAccumulator(name='IgnoredExpectationsMismatch',
+ do_list=True, do_fail=False),
+ JSONKEY_ACTUALRESULTS_NOCOMPARISON:
+ ResultAccumulator(name='MissingExpectations',
+ do_list=False, do_fail=False),
+ JSONKEY_ACTUALRESULTS_SUCCEEDED:
+ ResultAccumulator(name='Passed',
+ do_list=False, do_fail=False),
+ }
+
+ success = True
+ json_dict = json.load(open(filepath))
+ actual_results = json_dict[JSONKEY_ACTUALRESULTS]
+ for label, accumulator in results_map.iteritems():
+ results = actual_results[label]
+ if results:
+ for result in results:
+ accumulator.AddResult(result)
+ print accumulator.GetSummaryLine()
+ if accumulator.ShouldSignalFailure():
+ success = False
+ print '(results marked with [*] will cause nonzero return value)'
+ return success
+
+
+if '__main__' == __name__:
+ if len(sys.argv) != 2:
+ raise Exception('usage: %s <input-json-filepath>' % sys.argv[0])
+ sys.exit(0 if Display(sys.argv[1]) else 1)
diff --git a/gm/gm_expectations.cpp b/gm/gm_expectations.cpp
index 19e5de6..71311d5 100644
--- a/gm/gm_expectations.cpp
+++ b/gm/gm_expectations.cpp
@@ -12,7 +12,7 @@
#define DEBUGFAIL_SEE_STDERR SkDEBUGFAIL("see stderr for message")
// These constants must be kept in sync with the JSONKEY_ constants in
-// confirm_no_failures_in_json.py !
+// display_json_results.py !
const static char kJsonKey_ActualResults[] = "actual-results";
const static char kJsonKey_ActualResults_Failed[] = "failed";
const static char kJsonKey_ActualResults_FailureIgnored[]= "failure-ignored";
diff --git a/gm/tests/run.sh b/gm/tests/run.sh
index 2038c49..809c7d0 100755
--- a/gm/tests/run.sh
+++ b/gm/tests/run.sh
@@ -216,14 +216,14 @@
# Test non-hierarchical mode.
gm_test "--verbose --match selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels-no-hierarchy.json" "$GM_OUTPUTS/no-hierarchy"
-# Exercise confirm_no_failures_in_json.py
+# Exercise display_json_results.py
PASSING_CASES="compared-against-identical-bytes-json compared-against-identical-pixels-json"
FAILING_CASES="compared-against-different-pixels-json"
for CASE in $PASSING_CASES; do
- assert_passes "python gm/confirm_no_failures_in_json.py $GM_OUTPUTS/$CASE/$OUTPUT_EXPECTED_SUBDIR/json-summary.txt"
+ assert_passes "python gm/display_json_results.py $GM_OUTPUTS/$CASE/$OUTPUT_EXPECTED_SUBDIR/json-summary.txt"
done
for CASE in $FAILING_CASES; do
- assert_fails "python gm/confirm_no_failures_in_json.py $GM_OUTPUTS/$CASE/$OUTPUT_EXPECTED_SUBDIR/json-summary.txt"
+ assert_fails "python gm/display_json_results.py $GM_OUTPUTS/$CASE/$OUTPUT_EXPECTED_SUBDIR/json-summary.txt"
done
if [ $ENCOUNTERED_ANY_ERRORS == 0 ]; then