rebaseline_server: add --compare-configs option
This allows us to compare GMs between configs across all builders, so we can see the largest deviations between raster and GPU renderings.
BUG=skia:1919
NOTREECHECKS=True
NOTRY=True
R=rmistry@google.com
Author: epoger@google.com
Review URL: https://codereview.chromium.org/215503002
git-svn-id: http://skia.googlecode.com/svn/trunk@13991 2bbb7eff-a529-9590-31e7-b0007b416f81
diff --git a/gm/rebaseline_server/results.py b/gm/rebaseline_server/results.py
index aadb6a7..7cf5745 100755
--- a/gm/rebaseline_server/results.py
+++ b/gm/rebaseline_server/results.py
@@ -10,6 +10,7 @@
"""
# System-level imports
+import fnmatch
import os
import re
import sys
@@ -26,6 +27,7 @@
if GM_DIRECTORY not in sys.path:
sys.path.append(GM_DIRECTORY)
import gm_json
+import imagepairset
# Keys used to link an image to a particular GM test.
# NOTE: Keep these in sync with static/constants.js
@@ -56,3 +58,142 @@
IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
IMAGE_FILENAME_FORMATTER = '%s_%s.png' # pass in (testname, config)
+
+# Ignore expectations/actuals for builders matching any of these patterns.
+# This allows us to ignore builders for which we don't maintain expectations
+# (trybots, Valgrind, ASAN, TSAN), and avoid problems like
+# https://code.google.com/p/skia/issues/detail?id=2036 ('rebaseline_server
+# produces error when trying to add baselines for ASAN/TSAN builders')
+SKIP_BUILDERS_PATTERN_LIST = [re.compile(p) for p in [
+ '.*-Trybot', '.*Valgrind.*', '.*TSAN.*', '.*ASAN.*']]
+
+DEFAULT_ACTUALS_DIR = '.gm-actuals'
+DEFAULT_GENERATED_IMAGES_ROOT = os.path.join(
+ PARENT_DIRECTORY, '.generated-images')
+
+
+class BaseComparisons(object):
+ """Base class for generating summary of comparisons between two image sets.
+ """
+
+ def __init__(self):
+ raise NotImplementedError('cannot instantiate the abstract base class')
+
+ def get_results_of_type(self, results_type):
+ """Return results of some/all tests (depending on 'results_type' parameter).
+
+ Args:
+ results_type: string describing which types of results to include; must
+ be one of the RESULTS_* constants
+
+ Results are returned in a dictionary as output by ImagePairSet.as_dict().
+ """
+ return self._results[results_type]
+
+ def get_packaged_results_of_type(self, results_type, reload_seconds=None,
+ is_editable=False, is_exported=True):
+ """Package the results of some/all tests as a complete response_dict.
+
+ Args:
+ results_type: string indicating which set of results to return;
+ must be one of the RESULTS_* constants
+ reload_seconds: if specified, note that new results may be available once
+ these results are reload_seconds old
+ is_editable: whether clients are allowed to submit new baselines
+ is_exported: whether these results are being made available to other
+ network hosts
+ """
+ response_dict = self._results[results_type]
+ time_updated = self.get_timestamp()
+ response_dict[KEY__HEADER] = {
+ KEY__HEADER__SCHEMA_VERSION: (
+ REBASELINE_SERVER_SCHEMA_VERSION_NUMBER),
+
+ # Timestamps:
+ # 1. when this data was last updated
+ # 2. when the caller should check back for new data (if ever)
+ KEY__HEADER__TIME_UPDATED: time_updated,
+ KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE: (
+ (time_updated+reload_seconds) if reload_seconds else None),
+
+ # The type we passed to get_results_of_type()
+ KEY__HEADER__TYPE: results_type,
+
+ # Hash of dataset, which the client must return with any edits--
+ # this ensures that the edits were made to a particular dataset.
+ KEY__HEADER__DATAHASH: str(hash(repr(
+ response_dict[imagepairset.KEY__IMAGEPAIRS]))),
+
+ # Whether the server will accept edits back.
+ KEY__HEADER__IS_EDITABLE: is_editable,
+
+ # Whether the service is accessible from other hosts.
+ KEY__HEADER__IS_EXPORTED: is_exported,
+ }
+ return response_dict
+
+ def get_timestamp(self):
+ """Return the time at which this object was created, in seconds past epoch
+ (UTC).
+ """
+ return self._timestamp
+
+ @staticmethod
+ def _ignore_builder(builder):
+ """Returns True if this builder matches any of SKIP_BUILDERS_PATTERN_LIST.
+
+ Args:
+ builder: name of this builder, as a string
+
+ Returns:
+ True if we should ignore expectations and actuals for this builder.
+ """
+ for pattern in SKIP_BUILDERS_PATTERN_LIST:
+ if pattern.match(builder):
+ return True
+ return False
+
+ @staticmethod
+ def _read_dicts_from_root(root, pattern='*.json'):
+ """Read all JSON dictionaries within a directory tree.
+
+ Args:
+ root: path to root of directory tree
+ pattern: which files to read within root (fnmatch-style pattern)
+
+ Returns:
+ A meta-dictionary containing all the JSON dictionaries found within
+ the directory tree, keyed by the builder name of each dictionary.
+
+ Raises:
+ IOError if root does not refer to an existing directory
+ """
+ if not os.path.isdir(root):
+ raise IOError('no directory found at path %s' % root)
+ meta_dict = {}
+ for dirpath, dirnames, filenames in os.walk(root):
+ for matching_filename in fnmatch.filter(filenames, pattern):
+ builder = os.path.basename(dirpath)
+ if BaseComparisons._ignore_builder(builder):
+ continue
+ fullpath = os.path.join(dirpath, matching_filename)
+ meta_dict[builder] = gm_json.LoadFromFile(fullpath)
+ return meta_dict
+
+ @staticmethod
+ def _create_relative_url(hashtype_and_digest, test_name):
+ """Returns the URL for this image, relative to GM_ACTUALS_ROOT_HTTP_URL.
+
+ If we don't have a record of this image, returns None.
+
+ Args:
+ hashtype_and_digest: (hash_type, hash_digest) tuple, or None if we
+ don't have a record of this image
+ test_name: string; name of the GM test that created this image
+ """
+ if not hashtype_and_digest:
+ return None
+ return gm_json.CreateGmRelativeUrl(
+ test_name=test_name,
+ hash_type=hashtype_and_digest[0],
+ hash_digest=hashtype_and_digest[1])