rebaseline_server: add --compare-configs option
This allows us to compare GMs between configs across all builders, so we can see the largest deviations between raster and GPU renderings.
BUG=skia:1919
NOTREECHECKS=True
NOTRY=True
R=rmistry@google.com
Author: epoger@google.com
Review URL: https://codereview.chromium.org/215503002
git-svn-id: http://skia.googlecode.com/svn/trunk@13991 2bbb7eff-a529-9590-31e7-b0007b416f81
diff --git a/gm/rebaseline_server/compare_configs.py b/gm/rebaseline_server/compare_configs.py
new file mode 100755
index 0000000..8f92551
--- /dev/null
+++ b/gm/rebaseline_server/compare_configs.py
@@ -0,0 +1,219 @@
+#!/usr/bin/python
+
+"""
+Copyright 2014 Google Inc.
+
+Use of this source code is governed by a BSD-style license that can be
+found in the LICENSE file.
+
+Compare GM results for two configs, across all builders.
+"""
+
+# System-level imports
+import argparse
+import fnmatch
+import json
+import logging
+import os
+import re
+import sys
+import time
+
+# Imports from within Skia
+#
+# TODO(epoger): Once we move the create_filepath_url() function out of
+# download_actuals into a shared utility module, we won't need to import
+# download_actuals anymore.
+#
+# We need to add the 'gm' directory, so that we can import gm_json.py within
+# that directory. That script allows us to parse the actual-results.json file
+# written out by the GM tool.
+# Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end*
+# so any dirs that are already in the PYTHONPATH will be preferred.
+PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
+GM_DIRECTORY = os.path.dirname(PARENT_DIRECTORY)
+TRUNK_DIRECTORY = os.path.dirname(GM_DIRECTORY)
+if GM_DIRECTORY not in sys.path:
+ sys.path.append(GM_DIRECTORY)
+import download_actuals
+import gm_json
+import imagediffdb
+import imagepair
+import imagepairset
+import results
+
+
+class ConfigComparisons(results.BaseComparisons):
+ """Loads results from two different configurations into an ImagePairSet.
+
+ Loads actual and expected results from all builders, except for those skipped
+ by BaseComparisons._ignore_builder().
+ """
+
+ def __init__(self, configs, actuals_root=results.DEFAULT_ACTUALS_DIR,
+ generated_images_root=results.DEFAULT_GENERATED_IMAGES_ROOT,
+ diff_base_url=None):
+ """
+ Args:
+ configs: (string, string) tuple; pair of configs to compare
+ actuals_root: root directory containing all actual-results.json files
+ generated_images_root: directory within which to create all pixel diffs;
+ if this directory does not yet exist, it will be created
+ diff_base_url: base URL within which the client should look for diff
+ images; if not specified, defaults to a "file:///" URL representation
+ of generated_images_root
+ """
+ time_start = int(time.time())
+ self._image_diff_db = imagediffdb.ImageDiffDB(generated_images_root)
+ self._diff_base_url = (
+ diff_base_url or
+ download_actuals.create_filepath_url(generated_images_root))
+ self._actuals_root = actuals_root
+ self._load_config_pairs(configs)
+ self._timestamp = int(time.time())
+ logging.info('Results complete; took %d seconds.' %
+ (self._timestamp - time_start))
+
+ def _load_config_pairs(self, configs):
+ """Loads the results of all tests, across all builders (based on the
+ files within self._actuals_root), compares them across two configs,
+ and stores the summary in self._results.
+
+ Args:
+ configs: tuple of strings; pair of configs to compare
+ """
+ logging.info('Reading actual-results JSON files from %s...' %
+ self._actuals_root)
+ actual_builder_dicts = ConfigComparisons._read_dicts_from_root(
+ self._actuals_root)
+ configA, configB = configs
+ logging.info('Comparing configs %s and %s...' % (configA, configB))
+
+ all_image_pairs = imagepairset.ImagePairSet(
+ descriptions=configs,
+ diff_base_url=self._diff_base_url)
+ failing_image_pairs = imagepairset.ImagePairSet(
+ descriptions=configs,
+ diff_base_url=self._diff_base_url)
+
+ all_image_pairs.ensure_extra_column_values_in_summary(
+ column_id=results.KEY__EXTRACOLUMN__RESULT_TYPE, values=[
+ results.KEY__RESULT_TYPE__FAILED,
+ results.KEY__RESULT_TYPE__NOCOMPARISON,
+ results.KEY__RESULT_TYPE__SUCCEEDED,
+ ])
+ failing_image_pairs.ensure_extra_column_values_in_summary(
+ column_id=results.KEY__EXTRACOLUMN__RESULT_TYPE, values=[
+ results.KEY__RESULT_TYPE__FAILED,
+ results.KEY__RESULT_TYPE__NOCOMPARISON,
+ ])
+
+ builders = sorted(actual_builder_dicts.keys())
+ num_builders = len(builders)
+ builder_num = 0
+ for builder in builders:
+ builder_num += 1
+ logging.info('Generating pixel diffs for builder #%d of %d, "%s"...' %
+ (builder_num, num_builders, builder))
+ actual_results_for_this_builder = (
+ actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
+ for result_type in sorted(actual_results_for_this_builder.keys()):
+ results_of_this_type = actual_results_for_this_builder[result_type]
+ if not results_of_this_type:
+ continue
+
+ tests_found = set()
+ for image_name in sorted(results_of_this_type.keys()):
+ (test, config) = results.IMAGE_FILENAME_RE.match(image_name).groups()
+ tests_found.add(test)
+
+ for test in tests_found:
+ # Get image_relative_url (or None) for each of configA, configB
+ image_name_A = results.IMAGE_FILENAME_FORMATTER % (test, configA)
+ configA_image_relative_url = ConfigComparisons._create_relative_url(
+ hashtype_and_digest=results_of_this_type.get(image_name_A),
+ test_name=test)
+ image_name_B = results.IMAGE_FILENAME_FORMATTER % (test, configB)
+ configB_image_relative_url = ConfigComparisons._create_relative_url(
+ hashtype_and_digest=results_of_this_type.get(image_name_B),
+ test_name=test)
+
+ # If we have images for at least one of these two configs,
+ # add them to our list.
+ if configA_image_relative_url or configB_image_relative_url:
+ if configA_image_relative_url == configB_image_relative_url:
+ result_type = results.KEY__RESULT_TYPE__SUCCEEDED
+ elif not configA_image_relative_url:
+ result_type = results.KEY__RESULT_TYPE__NOCOMPARISON
+ elif not configB_image_relative_url:
+ result_type = results.KEY__RESULT_TYPE__NOCOMPARISON
+ else:
+ result_type = results.KEY__RESULT_TYPE__FAILED
+
+ extra_columns_dict = {
+ results.KEY__EXTRACOLUMN__RESULT_TYPE: result_type,
+ results.KEY__EXTRACOLUMN__BUILDER: builder,
+ results.KEY__EXTRACOLUMN__TEST: test,
+ # TODO(epoger): Right now, the client UI crashes if it receives
+ # results that do not include a 'config' column.
+ # Until we fix that, keep the client happy.
+ results.KEY__EXTRACOLUMN__CONFIG: 'TODO',
+ }
+
+ try:
+ image_pair = imagepair.ImagePair(
+ image_diff_db=self._image_diff_db,
+ base_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL,
+ imageA_relative_url=configA_image_relative_url,
+ imageB_relative_url=configB_image_relative_url,
+ extra_columns=extra_columns_dict)
+ all_image_pairs.add_image_pair(image_pair)
+ if result_type != results.KEY__RESULT_TYPE__SUCCEEDED:
+ failing_image_pairs.add_image_pair(image_pair)
+ except (KeyError, TypeError):
+ logging.exception(
+ 'got exception while creating ImagePair for image_name '
+ '"%s", builder "%s"' % (image_name, builder))
+
+ self._results = {
+ results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(),
+ results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(),
+ }
+
+
+def main():
+ logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
+ datefmt='%m/%d/%Y %H:%M:%S',
+ level=logging.INFO)
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ '--actuals', default=results.DEFAULT_ACTUALS_DIR,
+ help='Directory containing all actual-result JSON files; defaults to '
+ '\'%(default)s\' .')
+ parser.add_argument(
+ 'config', nargs=2,
+ help='Two configurations to compare (8888, gpu, etc.).')
+ parser.add_argument(
+ '--outfile', required=True,
+ help='File to write result summary into, in JSON format.')
+ parser.add_argument(
+ '--results', default=results.KEY__HEADER__RESULTS_FAILURES,
+ help='Which result types to include. Defaults to \'%(default)s\'; '
+ 'must be one of ' +
+ str([results.KEY__HEADER__RESULTS_FAILURES,
+ results.KEY__HEADER__RESULTS_ALL]))
+ parser.add_argument(
+ '--workdir', default=results.DEFAULT_GENERATED_IMAGES_ROOT,
+ help='Directory within which to download images and generate diffs; '
+ 'defaults to \'%(default)s\' .')
+ args = parser.parse_args()
+ results_obj = ConfigComparisons(configs=args.config,
+ actuals_root=args.actuals,
+ generated_images_root=args.workdir)
+ gm_json.WriteToFile(
+ results_obj.get_packaged_results_of_type(results_type=args.results),
+ args.outfile)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/gm/rebaseline_server/compare_configs_test.py b/gm/rebaseline_server/compare_configs_test.py
new file mode 100755
index 0000000..756af66
--- /dev/null
+++ b/gm/rebaseline_server/compare_configs_test.py
@@ -0,0 +1,57 @@
+#!/usr/bin/python
+
+"""
+Copyright 2014 Google Inc.
+
+Use of this source code is governed by a BSD-style license that can be
+found in the LICENSE file.
+
+Test compare_configs.py
+
+TODO(epoger): Create a command to update the expected results (in
+self._output_dir_expected) when appropriate. For now, you should:
+1. examine the results in self._output_dir_actual and make sure they are ok
+2. rm -rf self._output_dir_expected
+3. mv self._output_dir_actual self._output_dir_expected
+Although, if you're using an SVN checkout, this will blow away .svn directories
+within self._output_dir_expected, which wouldn't be good...
+
+"""
+
+import os
+import sys
+
+# Imports from within Skia
+import base_unittest
+import compare_configs
+import results
+import gm_json # must import results first, so that gm_json will be in sys.path
+
+
+class CompareConfigsTest(base_unittest.TestCase):
+
+ def test_gm(self):
+ """Process results of a GM run with the ConfigComparisons object."""
+ results_obj = compare_configs.ConfigComparisons(
+ configs=('8888', 'gpu'),
+ actuals_root=os.path.join(self._input_dir, 'gm-actuals'),
+ generated_images_root=self._temp_dir,
+ diff_base_url='/static/generated-images')
+ results_obj.get_timestamp = mock_get_timestamp
+ gm_json.WriteToFile(
+ results_obj.get_packaged_results_of_type(
+ results.KEY__HEADER__RESULTS_ALL),
+ os.path.join(self._output_dir_actual, 'gm.json'))
+
+
+def mock_get_timestamp():
+ """Mock version of BaseComparisons.get_timestamp() for testing."""
+ return 12345678
+
+
+def main():
+ base_unittest.main(CompareConfigsTest)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/gm/rebaseline_server/compare_to_expectations.py b/gm/rebaseline_server/compare_to_expectations.py
index e0fa23f..c8510d6 100755
--- a/gm/rebaseline_server/compare_to_expectations.py
+++ b/gm/rebaseline_server/compare_to_expectations.py
@@ -47,28 +47,24 @@
results.KEY__EXPECTATIONS__IGNOREFAILURE,
results.KEY__EXPECTATIONS__REVIEWED,
]
+DEFAULT_EXPECTATIONS_DIR = os.path.join(TRUNK_DIRECTORY, 'expectations', 'gm')
IMAGEPAIR_SET_DESCRIPTIONS = ('expected image', 'actual image')
-DEFAULT_ACTUALS_DIR = '.gm-actuals'
-DEFAULT_EXPECTATIONS_DIR = os.path.join(TRUNK_DIRECTORY, 'expectations', 'gm')
-DEFAULT_GENERATED_IMAGES_ROOT = os.path.join(
- PARENT_DIRECTORY, '.generated-images')
-
-class Results(object):
- """ Loads actual and expected GM results into an ImagePairSet.
+class ExpectationComparisons(results.BaseComparisons):
+ """Loads actual and expected GM results into an ImagePairSet.
Loads actual and expected results from all builders, except for those skipped
by _ignore_builder().
Once this object has been constructed, the results (in self._results[])
are immutable. If you want to update the results based on updated JSON
- file contents, you will need to create a new Results object."""
+ file contents, you will need to create a new ExpectationComparisons object."""
- def __init__(self, actuals_root=DEFAULT_ACTUALS_DIR,
+ def __init__(self, actuals_root=results.DEFAULT_ACTUALS_DIR,
expected_root=DEFAULT_EXPECTATIONS_DIR,
- generated_images_root=DEFAULT_GENERATED_IMAGES_ROOT,
+ generated_images_root=results.DEFAULT_GENERATED_IMAGES_ROOT,
diff_base_url=None):
"""
Args:
@@ -92,19 +88,13 @@
logging.info('Results complete; took %d seconds.' %
(self._timestamp - time_start))
- def get_timestamp(self):
- """Return the time at which this object was created, in seconds past epoch
- (UTC).
- """
- return self._timestamp
-
def edit_expectations(self, modifications):
"""Edit the expectations stored within this object and write them back
to disk.
Note that this will NOT update the results stored in self._results[] ;
- in order to see those updates, you must instantiate a new Results object
- based on the (now updated) files on disk.
+ in order to see those updates, you must instantiate a new
+ ExpectationComparisons object based on the (now updated) files on disk.
Args:
modifications: a list of dictionaries, one for each expectation to update:
@@ -127,7 +117,8 @@
]
"""
- expected_builder_dicts = Results._read_dicts_from_root(self._expected_root)
+ expected_builder_dicts = ExpectationComparisons._read_dicts_from_root(
+ self._expected_root)
for mod in modifications:
image_name = results.IMAGE_FILENAME_FORMATTER % (
mod[imagepair.KEY__EXTRA_COLUMN_VALUES]
@@ -152,125 +143,8 @@
builder_expectations = {}
builder_dict[gm_json.JSONKEY_EXPECTEDRESULTS] = builder_expectations
builder_expectations[image_name] = new_expectations
- Results._write_dicts_to_root(expected_builder_dicts, self._expected_root)
-
- def get_results_of_type(self, results_type):
- """Return results of some/all tests (depending on 'results_type' parameter).
-
- Args:
- results_type: string describing which types of results to include; must
- be one of the RESULTS_* constants
-
- Results are returned in a dictionary as output by ImagePairSet.as_dict().
- """
- return self._results[results_type]
-
- def get_packaged_results_of_type(self, results_type, reload_seconds=None,
- is_editable=False, is_exported=True):
- """ Package the results of some/all tests as a complete response_dict.
-
- Args:
- results_type: string indicating which set of results to return;
- must be one of the RESULTS_* constants
- reload_seconds: if specified, note that new results may be available once
- these results are reload_seconds old
- is_editable: whether clients are allowed to submit new baselines
- is_exported: whether these results are being made available to other
- network hosts
- """
- response_dict = self._results[results_type]
- time_updated = self.get_timestamp()
- response_dict[results.KEY__HEADER] = {
- results.KEY__HEADER__SCHEMA_VERSION: (
- results.REBASELINE_SERVER_SCHEMA_VERSION_NUMBER),
-
- # Timestamps:
- # 1. when this data was last updated
- # 2. when the caller should check back for new data (if ever)
- results.KEY__HEADER__TIME_UPDATED: time_updated,
- results.KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE: (
- (time_updated+reload_seconds) if reload_seconds else None),
-
- # The type we passed to get_results_of_type()
- results.KEY__HEADER__TYPE: results_type,
-
- # Hash of dataset, which the client must return with any edits--
- # this ensures that the edits were made to a particular dataset.
- results.KEY__HEADER__DATAHASH: str(hash(repr(
- response_dict[imagepairset.KEY__IMAGEPAIRS]))),
-
- # Whether the server will accept edits back.
- results.KEY__HEADER__IS_EDITABLE: is_editable,
-
- # Whether the service is accessible from other hosts.
- results.KEY__HEADER__IS_EXPORTED: is_exported,
- }
- return response_dict
-
- @staticmethod
- def _ignore_builder(builder):
- """Returns True if we should ignore expectations and actuals for a builder.
-
- This allows us to ignore builders for which we don't maintain expectations
- (trybots, Valgrind, ASAN, TSAN), and avoid problems like
- https://code.google.com/p/skia/issues/detail?id=2036 ('rebaseline_server
- produces error when trying to add baselines for ASAN/TSAN builders')
-
- Args:
- builder: name of this builder, as a string
-
- Returns:
- True if we should ignore expectations and actuals for this builder.
- """
- return (builder.endswith('-Trybot') or
- ('Valgrind' in builder) or
- ('TSAN' in builder) or
- ('ASAN' in builder))
-
- @staticmethod
- def _read_dicts_from_root(root, pattern='*.json'):
- """Read all JSON dictionaries within a directory tree.
-
- Args:
- root: path to root of directory tree
- pattern: which files to read within root (fnmatch-style pattern)
-
- Returns:
- A meta-dictionary containing all the JSON dictionaries found within
- the directory tree, keyed by the builder name of each dictionary.
-
- Raises:
- IOError if root does not refer to an existing directory
- """
- if not os.path.isdir(root):
- raise IOError('no directory found at path %s' % root)
- meta_dict = {}
- for dirpath, dirnames, filenames in os.walk(root):
- for matching_filename in fnmatch.filter(filenames, pattern):
- builder = os.path.basename(dirpath)
- if Results._ignore_builder(builder):
- continue
- fullpath = os.path.join(dirpath, matching_filename)
- meta_dict[builder] = gm_json.LoadFromFile(fullpath)
- return meta_dict
-
- @staticmethod
- def _create_relative_url(hashtype_and_digest, test_name):
- """Returns the URL for this image, relative to GM_ACTUALS_ROOT_HTTP_URL.
-
- If we don't have a record of this image, returns None.
-
- Args:
- hashtype_and_digest: (hash_type, hash_digest) tuple, or None if we
- don't have a record of this image
- test_name: string; name of the GM test that created this image
- """
- if not hashtype_and_digest:
- return None
- return gm_json.CreateGmRelativeUrl(
- test_name=test_name,
- hash_type=hashtype_and_digest[0],
- hash_digest=hashtype_and_digest[1])
+ ExpectationComparisons._write_dicts_to_root(
+ expected_builder_dicts, self._expected_root)
@staticmethod
def _write_dicts_to_root(meta_dict, root, pattern='*.json'):
@@ -300,7 +174,7 @@
for dirpath, dirnames, filenames in os.walk(root):
for matching_filename in fnmatch.filter(filenames, pattern):
builder = os.path.basename(dirpath)
- if Results._ignore_builder(builder):
+ if ExpectationComparisons._ignore_builder(builder):
continue
per_builder_dict = meta_dict.get(builder)
if per_builder_dict is not None:
@@ -325,10 +199,12 @@
"""
logging.info('Reading actual-results JSON files from %s...' %
self._actuals_root)
- actual_builder_dicts = Results._read_dicts_from_root(self._actuals_root)
+ actual_builder_dicts = ExpectationComparisons._read_dicts_from_root(
+ self._actuals_root)
logging.info('Reading expected-results JSON files from %s...' %
self._expected_root)
- expected_builder_dicts = Results._read_dicts_from_root(self._expected_root)
+ expected_builder_dicts = ExpectationComparisons._read_dicts_from_root(
+ self._expected_root)
all_image_pairs = imagepairset.ImagePairSet(
descriptions=IMAGEPAIR_SET_DESCRIPTIONS,
@@ -366,9 +242,10 @@
continue
for image_name in sorted(results_of_this_type.keys()):
(test, config) = results.IMAGE_FILENAME_RE.match(image_name).groups()
- actual_image_relative_url = Results._create_relative_url(
- hashtype_and_digest=results_of_this_type[image_name],
- test_name=test)
+ actual_image_relative_url = (
+ ExpectationComparisons._create_relative_url(
+ hashtype_and_digest=results_of_this_type[image_name],
+ test_name=test))
# Default empty expectations; overwrite these if we find any real ones
expectations_per_test = None
@@ -383,9 +260,10 @@
expected_image_hashtype_and_digest = (
expectations_per_test
[gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS][0])
- expected_image_relative_url = Results._create_relative_url(
- hashtype_and_digest=expected_image_hashtype_and_digest,
- test_name=test)
+ expected_image_relative_url = (
+ ExpectationComparisons._create_relative_url(
+ hashtype_and_digest=expected_image_hashtype_and_digest,
+ test_name=test))
expectations_dict = {}
for field in EXPECTATION_FIELDS_PASSED_THRU_VERBATIM:
expectations_dict[field] = expectations_per_test.get(field)
@@ -465,7 +343,7 @@
level=logging.INFO)
parser = argparse.ArgumentParser()
parser.add_argument(
- '--actuals', default=DEFAULT_ACTUALS_DIR,
+ '--actuals', default=results.DEFAULT_ACTUALS_DIR,
help='Directory containing all actual-result JSON files; defaults to '
'\'%(default)s\' .')
parser.add_argument(
@@ -482,13 +360,13 @@
str([results.KEY__HEADER__RESULTS_FAILURES,
results.KEY__HEADER__RESULTS_ALL]))
parser.add_argument(
- '--workdir', default=DEFAULT_GENERATED_IMAGES_ROOT,
+ '--workdir', default=results.DEFAULT_GENERATED_IMAGES_ROOT,
help='Directory within which to download images and generate diffs; '
'defaults to \'%(default)s\' .')
args = parser.parse_args()
- results_obj = Results(actuals_root=args.actuals,
- expected_root=args.expectations,
- generated_images_root=args.workdir)
+ results_obj = ExpectationComparisons(actuals_root=args.actuals,
+ expected_root=args.expectations,
+ generated_images_root=args.workdir)
gm_json.WriteToFile(
results_obj.get_packaged_results_of_type(results_type=args.results),
args.outfile)
diff --git a/gm/rebaseline_server/compare_to_expectations_test.py b/gm/rebaseline_server/compare_to_expectations_test.py
index c0bf190..76e4a7b 100755
--- a/gm/rebaseline_server/compare_to_expectations_test.py
+++ b/gm/rebaseline_server/compare_to_expectations_test.py
@@ -31,8 +31,8 @@
class CompareToExpectationsTest(base_unittest.TestCase):
def test_gm(self):
- """Process results of a GM run with the Results object."""
- results_obj = compare_to_expectations.Results(
+ """Process results of a GM run with the ExpectationComparisons object."""
+ results_obj = compare_to_expectations.ExpectationComparisons(
actuals_root=os.path.join(self._input_dir, 'gm-actuals'),
expected_root=os.path.join(self._input_dir, 'gm-expectations'),
generated_images_root=self._temp_dir,
@@ -45,7 +45,7 @@
def mock_get_timestamp():
- """Mock version of Results.get_timestamp() for testing."""
+ """Mock version of BaseComparisons.get_timestamp() for testing."""
return 12345678
diff --git a/gm/rebaseline_server/results.py b/gm/rebaseline_server/results.py
index aadb6a7..7cf5745 100755
--- a/gm/rebaseline_server/results.py
+++ b/gm/rebaseline_server/results.py
@@ -10,6 +10,7 @@
"""
# System-level imports
+import fnmatch
import os
import re
import sys
@@ -26,6 +27,7 @@
if GM_DIRECTORY not in sys.path:
sys.path.append(GM_DIRECTORY)
import gm_json
+import imagepairset
# Keys used to link an image to a particular GM test.
# NOTE: Keep these in sync with static/constants.js
@@ -56,3 +58,142 @@
IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
IMAGE_FILENAME_FORMATTER = '%s_%s.png' # pass in (testname, config)
+
+# Ignore expectations/actuals for builders matching any of these patterns.
+# This allows us to ignore builders for which we don't maintain expectations
+# (trybots, Valgrind, ASAN, TSAN), and avoid problems like
+# https://code.google.com/p/skia/issues/detail?id=2036 ('rebaseline_server
+# produces error when trying to add baselines for ASAN/TSAN builders')
+SKIP_BUILDERS_PATTERN_LIST = [re.compile(p) for p in [
+ '.*-Trybot', '.*Valgrind.*', '.*TSAN.*', '.*ASAN.*']]
+
+DEFAULT_ACTUALS_DIR = '.gm-actuals'
+DEFAULT_GENERATED_IMAGES_ROOT = os.path.join(
+ PARENT_DIRECTORY, '.generated-images')
+
+
+class BaseComparisons(object):
+ """Base class for generating summary of comparisons between two image sets.
+ """
+
+ def __init__(self):
+ raise NotImplementedError('cannot instantiate the abstract base class')
+
+ def get_results_of_type(self, results_type):
+ """Return results of some/all tests (depending on 'results_type' parameter).
+
+ Args:
+ results_type: string describing which types of results to include; must
+ be one of the RESULTS_* constants
+
+ Results are returned in a dictionary as output by ImagePairSet.as_dict().
+ """
+ return self._results[results_type]
+
+ def get_packaged_results_of_type(self, results_type, reload_seconds=None,
+ is_editable=False, is_exported=True):
+ """Package the results of some/all tests as a complete response_dict.
+
+ Args:
+ results_type: string indicating which set of results to return;
+ must be one of the RESULTS_* constants
+ reload_seconds: if specified, note that new results may be available once
+ these results are reload_seconds old
+ is_editable: whether clients are allowed to submit new baselines
+ is_exported: whether these results are being made available to other
+ network hosts
+ """
+ response_dict = self._results[results_type]
+ time_updated = self.get_timestamp()
+ response_dict[KEY__HEADER] = {
+ KEY__HEADER__SCHEMA_VERSION: (
+ REBASELINE_SERVER_SCHEMA_VERSION_NUMBER),
+
+ # Timestamps:
+ # 1. when this data was last updated
+ # 2. when the caller should check back for new data (if ever)
+ KEY__HEADER__TIME_UPDATED: time_updated,
+ KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE: (
+ (time_updated+reload_seconds) if reload_seconds else None),
+
+ # The type we passed to get_results_of_type()
+ KEY__HEADER__TYPE: results_type,
+
+ # Hash of dataset, which the client must return with any edits--
+ # this ensures that the edits were made to a particular dataset.
+ KEY__HEADER__DATAHASH: str(hash(repr(
+ response_dict[imagepairset.KEY__IMAGEPAIRS]))),
+
+ # Whether the server will accept edits back.
+ KEY__HEADER__IS_EDITABLE: is_editable,
+
+ # Whether the service is accessible from other hosts.
+ KEY__HEADER__IS_EXPORTED: is_exported,
+ }
+ return response_dict
+
+ def get_timestamp(self):
+ """Return the time at which this object was created, in seconds past epoch
+ (UTC).
+ """
+ return self._timestamp
+
+ @staticmethod
+ def _ignore_builder(builder):
+ """Returns True if this builder matches any of SKIP_BUILDERS_PATTERN_LIST.
+
+ Args:
+ builder: name of this builder, as a string
+
+ Returns:
+ True if we should ignore expectations and actuals for this builder.
+ """
+ for pattern in SKIP_BUILDERS_PATTERN_LIST:
+ if pattern.match(builder):
+ return True
+ return False
+
+ @staticmethod
+ def _read_dicts_from_root(root, pattern='*.json'):
+ """Read all JSON dictionaries within a directory tree.
+
+ Args:
+ root: path to root of directory tree
+ pattern: which files to read within root (fnmatch-style pattern)
+
+ Returns:
+ A meta-dictionary containing all the JSON dictionaries found within
+ the directory tree, keyed by the builder name of each dictionary.
+
+ Raises:
+ IOError if root does not refer to an existing directory
+ """
+ if not os.path.isdir(root):
+ raise IOError('no directory found at path %s' % root)
+ meta_dict = {}
+ for dirpath, dirnames, filenames in os.walk(root):
+ for matching_filename in fnmatch.filter(filenames, pattern):
+ builder = os.path.basename(dirpath)
+ if BaseComparisons._ignore_builder(builder):
+ continue
+ fullpath = os.path.join(dirpath, matching_filename)
+ meta_dict[builder] = gm_json.LoadFromFile(fullpath)
+ return meta_dict
+
+ @staticmethod
+ def _create_relative_url(hashtype_and_digest, test_name):
+ """Returns the URL for this image, relative to GM_ACTUALS_ROOT_HTTP_URL.
+
+ If we don't have a record of this image, returns None.
+
+ Args:
+ hashtype_and_digest: (hash_type, hash_digest) tuple, or None if we
+ don't have a record of this image
+ test_name: string; name of the GM test that created this image
+ """
+ if not hashtype_and_digest:
+ return None
+ return gm_json.CreateGmRelativeUrl(
+ test_name=test_name,
+ hash_type=hashtype_and_digest[0],
+ hash_digest=hashtype_and_digest[1])
diff --git a/gm/rebaseline_server/server.py b/gm/rebaseline_server/server.py
index dff9504..04620f5 100755
--- a/gm/rebaseline_server/server.py
+++ b/gm/rebaseline_server/server.py
@@ -28,22 +28,28 @@
# Imports from within Skia
#
-# We need to add the 'tools' directory, so that we can import svn.py within
+# We need to add the 'tools' directory for svn.py, and the 'gm' directory for
+# gm_json.py .
# that directory.
# Make sure that the 'tools' dir is in the PYTHONPATH, but add it at the *end*
# so any dirs that are already in the PYTHONPATH will be preferred.
PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
-TRUNK_DIRECTORY = os.path.dirname(os.path.dirname(PARENT_DIRECTORY))
+GM_DIRECTORY = os.path.dirname(PARENT_DIRECTORY)
+TRUNK_DIRECTORY = os.path.dirname(GM_DIRECTORY)
TOOLS_DIRECTORY = os.path.join(TRUNK_DIRECTORY, 'tools')
if TOOLS_DIRECTORY not in sys.path:
sys.path.append(TOOLS_DIRECTORY)
import svn
+if GM_DIRECTORY not in sys.path:
+ sys.path.append(GM_DIRECTORY)
+import gm_json
# Imports from local dir
#
# Note: we import results under a different name, to avoid confusion with the
# Server.results() property. See discussion at
# https://codereview.chromium.org/195943004/diff/1/gm/rebaseline_server/server.py#newcode44
+import compare_configs
import compare_to_expectations
import imagepairset
import results as results_mod
@@ -67,19 +73,33 @@
KEY__EDITS__OLD_RESULTS_HASH = 'oldResultsHash'
KEY__EDITS__OLD_RESULTS_TYPE = 'oldResultsType'
-DEFAULT_ACTUALS_DIR = compare_to_expectations.DEFAULT_ACTUALS_DIR
+DEFAULT_ACTUALS_DIR = results_mod.DEFAULT_ACTUALS_DIR
DEFAULT_ACTUALS_REPO_REVISION = 'HEAD'
DEFAULT_ACTUALS_REPO_URL = 'http://skia-autogen.googlecode.com/svn/gm-actual'
DEFAULT_PORT = 8888
-# Directory within which the server will serve out static files.
-STATIC_CONTENTS_SUBDIR = 'static' # within PARENT_DIR
-GENERATED_IMAGES_SUBDIR = 'generated-images' # within STATIC_CONTENTS_SUBDIR
+# Directory, relative to PARENT_DIRECTORY, within which the server will serve
+# out live results (not static files).
+RESULTS_SUBDIR = 'results'
+# Directory, relative to PARENT_DIRECTORY, within which the server will serve
+# out static files.
+STATIC_CONTENTS_SUBDIR = 'static'
+# All of the GENERATED_*_SUBDIRS are relative to STATIC_CONTENTS_SUBDIR
+GENERATED_HTML_SUBDIR = 'generated-html'
+GENERATED_IMAGES_SUBDIR = 'generated-images'
+GENERATED_JSON_SUBDIR = 'generated-json'
# How often (in seconds) clients should reload while waiting for initial
# results to load.
RELOAD_INTERVAL_UNTIL_READY = 10
+SUMMARY_TYPES = [
+ results_mod.KEY__HEADER__RESULTS_FAILURES,
+ results_mod.KEY__HEADER__RESULTS_ALL,
+]
+# If --compare-configs is specified, compare these configs.
+CONFIG_PAIRS_TO_COMPARE = [('8888', 'gpu')]
+
_HTTP_HEADER_CONTENT_LENGTH = 'Content-Length'
_HTTP_HEADER_CONTENT_TYPE = 'Content-Type'
@@ -137,6 +157,57 @@
return local_checkout
+def _create_index(file_path, config_pairs):
+ """Creates an index file linking to all results available from this server.
+
+ Prior to https://codereview.chromium.org/215503002 , we had a static
+ index.html within our repo. But now that the results may or may not include
+ config comparisons, index.html needs to be generated differently depending
+ on which results are included.
+
+ TODO(epoger): Instead of including raw HTML within the Python code,
+ consider restoring the index.html file as a template and using django (or
+ similar) to fill in dynamic content.
+
+ Args:
+ file_path: path on local disk to write index to; any directory components
+ of this path that do not already exist will be created
+ config_pairs: what pairs of configs (if any) we compare actual results of
+ """
+ dir_path = os.path.dirname(file_path)
+ if not os.path.isdir(dir_path):
+ os.makedirs(dir_path)
+ with open(file_path, 'w') as file_handle:
+ file_handle.write(
+ '<!DOCTYPE html><html>'
+ '<head><title>rebaseline_server</title></head>'
+ '<body><ul>')
+ if SUMMARY_TYPES:
+ file_handle.write('<li>Expectations vs Actuals</li><ul>')
+ for summary_type in SUMMARY_TYPES:
+ file_handle.write(
+ '<li>'
+ '<a href="/%s/view.html#/view.html?resultsToLoad=/%s/%s">'
+ '%s</a></li>' % (
+ STATIC_CONTENTS_SUBDIR, RESULTS_SUBDIR,
+ summary_type, summary_type))
+ file_handle.write('</ul>')
+ if config_pairs:
+ file_handle.write('<li>Comparing configs within actual results</li><ul>')
+ for config_pair in config_pairs:
+ file_handle.write('<li>%s vs %s:' % config_pair)
+ for summary_type in SUMMARY_TYPES:
+ file_handle.write(
+ ' <a href="/%s/view.html#/view.html?'
+ 'resultsToLoad=/%s/%s/%s-vs-%s_%s.json">%s</a>' % (
+ STATIC_CONTENTS_SUBDIR, STATIC_CONTENTS_SUBDIR,
+ GENERATED_JSON_SUBDIR, config_pair[0], config_pair[1],
+ summary_type, summary_type))
+ file_handle.write('</li>')
+ file_handle.write('</ul>')
+ file_handle.write('</ul></body></html>')
+
+
class Server(object):
""" HTTP server for our HTML rebaseline viewer. """
@@ -145,7 +216,7 @@
actuals_repo_revision=DEFAULT_ACTUALS_REPO_REVISION,
actuals_repo_url=DEFAULT_ACTUALS_REPO_URL,
port=DEFAULT_PORT, export=False, editable=True,
- reload_seconds=0):
+ reload_seconds=0, config_pairs=None):
"""
Args:
actuals_dir: directory under which we will check out the latest actual
@@ -159,6 +230,9 @@
editable: whether HTTP clients are allowed to submit new baselines
reload_seconds: polling interval with which to check for new results;
if 0, don't check for new results at all
+ config_pairs: List of (string, string) tuples; for each tuple, compare
+ actual results of these two configs. If None or empty,
+ don't compare configs at all.
"""
self._actuals_dir = actuals_dir
self._actuals_repo_revision = actuals_repo_revision
@@ -167,6 +241,12 @@
self._export = export
self._editable = editable
self._reload_seconds = reload_seconds
+ self._config_pairs = config_pairs or []
+ _create_index(
+ file_path=os.path.join(
+ PARENT_DIRECTORY, STATIC_CONTENTS_SUBDIR, GENERATED_HTML_SUBDIR,
+ "index.html"),
+ config_pairs=config_pairs)
if actuals_repo_url:
self._actuals_repo = _create_svn_checkout(
dir_path=actuals_dir, repo_url=actuals_repo_url)
@@ -243,7 +323,7 @@
compare_to_expectations.DEFAULT_EXPECTATIONS_DIR)
_run_command(['gclient', 'sync'], TRUNK_DIRECTORY)
- self._results = compare_to_expectations.Results(
+ self._results = compare_to_expectations.ExpectationComparisons(
actuals_root=self._actuals_dir,
generated_images_root=os.path.join(
PARENT_DIRECTORY, STATIC_CONTENTS_SUBDIR,
@@ -251,6 +331,28 @@
diff_base_url=posixpath.join(
os.pardir, STATIC_CONTENTS_SUBDIR, GENERATED_IMAGES_SUBDIR))
+ json_dir = os.path.join(
+ PARENT_DIRECTORY, STATIC_CONTENTS_SUBDIR, GENERATED_JSON_SUBDIR)
+ if not os.path.isdir(json_dir):
+ os.makedirs(json_dir)
+
+ for config_pair in self._config_pairs:
+ config_comparisons = compare_configs.ConfigComparisons(
+ configs=config_pair,
+ actuals_root=self._actuals_dir,
+ generated_images_root=os.path.join(
+ PARENT_DIRECTORY, STATIC_CONTENTS_SUBDIR,
+ GENERATED_IMAGES_SUBDIR),
+ diff_base_url=posixpath.join(
+ os.pardir, GENERATED_IMAGES_SUBDIR))
+ for summary_type in SUMMARY_TYPES:
+ gm_json.WriteToFile(
+ config_comparisons.get_packaged_results_of_type(
+ results_type=summary_type),
+ os.path.join(
+ json_dir, '%s-vs-%s_%s.json' % (
+ config_pair[0], config_pair[1], summary_type)))
+
def _result_loader(self, reload_seconds=0):
""" Call self.update_results(), either once or periodically.
@@ -300,7 +402,8 @@
try:
logging.debug('do_GET: path="%s"' % self.path)
if self.path == '' or self.path == '/' or self.path == '/index.html' :
- self.redirect_to('/%s/index.html' % STATIC_CONTENTS_SUBDIR)
+ self.redirect_to('/%s/%s/index.html' % (
+ STATIC_CONTENTS_SUBDIR, GENERATED_HTML_SUBDIR))
return
if self.path == '/favicon.ico' :
self.redirect_to('/%s/favicon.ico' % STATIC_CONTENTS_SUBDIR)
@@ -313,8 +416,8 @@
normpath = posixpath.normpath(self.path)
(dispatcher_name, remainder) = PATHSPLIT_RE.match(normpath).groups()
dispatchers = {
- 'results': self.do_GET_results,
- STATIC_CONTENTS_SUBDIR: self.do_GET_static,
+ RESULTS_SUBDIR: self.do_GET_results,
+ STATIC_CONTENTS_SUBDIR: self.do_GET_static,
}
dispatcher = dispatchers[dispatcher_name]
dispatcher(remainder)
@@ -330,9 +433,9 @@
must be one of the results_mod.RESULTS_* constants
"""
logging.debug('do_GET_results: sending results of type "%s"' % results_type)
- # Since we must make multiple calls to the Results object, grab a
- # reference to it in case it is updated to point at a new Results
- # object within another thread.
+ # Since we must make multiple calls to the ExpectationComparisons object,
+ # grab a reference to it in case it is updated to point at a new
+ # ExpectationComparisons object within another thread.
#
# TODO(epoger): Rather than using a global variable for the handler
# to refer to the Server object, make Server a subclass of
@@ -524,6 +627,11 @@
'argument in conjunction with --editable; you '
'probably only want to edit results at HEAD.'),
default=DEFAULT_ACTUALS_REPO_REVISION)
+ parser.add_argument('--compare-configs', action='store_true',
+ help=('In addition to generating differences between '
+ 'expectations and actuals, also generate '
+ 'differences between these config pairs: '
+ + str(CONFIG_PAIRS_TO_COMPARE)))
parser.add_argument('--editable', action='store_true',
help=('Allow HTTP clients to submit new baselines.'))
parser.add_argument('--export', action='store_true',
@@ -545,12 +653,17 @@
'must restart the server to pick up new data.'),
default=0)
args = parser.parse_args()
+ if args.compare_configs:
+ config_pairs = CONFIG_PAIRS_TO_COMPARE
+ else:
+ config_pairs = None
+
global _SERVER
_SERVER = Server(actuals_dir=args.actuals_dir,
actuals_repo_revision=args.actuals_revision,
actuals_repo_url=args.actuals_repo,
port=args.port, export=args.export, editable=args.editable,
- reload_seconds=args.reload)
+ reload_seconds=args.reload, config_pairs=config_pairs)
_SERVER.run()
diff --git a/gm/rebaseline_server/static/index.html b/gm/rebaseline_server/static/index.html
deleted file mode 100644
index 9cfd41c..0000000
--- a/gm/rebaseline_server/static/index.html
+++ /dev/null
@@ -1,29 +0,0 @@
-<!DOCTYPE html>
-
-<html>
- <head>
- <title>rebaseline_server</title>
- </head>
-
- <body>
- Here are links to the result pages:
- <ul>
- <li>
- <a href="/static/view.html#/view.html?resultsToLoad=/results/failures">
- failures
- </a>
- (includes failed, failure-ignored, and no-comparison)
- </li>
- <li>
- <a href="/static/view.html#/view.html?resultsToLoad=/results/all">
- all results
- </a>
- (includes successful test results also, but takes longer to load)
- </li>
- </ul>
- Instructions, roadmap, etc. are at
- <a href="http://tinyurl.com/SkiaRebaselineServer">
- http://tinyurl.com/SkiaRebaselineServer
- </a>
- </body>
-</html>
diff --git a/gm/rebaseline_server/testdata/outputs/expected/compare_configs_test.CompareConfigsTest.test_gm/gm.json b/gm/rebaseline_server/testdata/outputs/expected/compare_configs_test.CompareConfigsTest.test_gm/gm.json
new file mode 100644
index 0000000..0dcccc9
--- /dev/null
+++ b/gm/rebaseline_server/testdata/outputs/expected/compare_configs_test.CompareConfigsTest.test_gm/gm.json
@@ -0,0 +1,207 @@
+{
+ "extraColumnHeaders": {
+ "builder": {
+ "headerText": "builder",
+ "isFilterable": true,
+ "isSortable": true,
+ "valuesAndCounts": {
+ "Test-Android-GalaxyNexus-SGX540-Arm7-Release": 7,
+ "Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug": 3
+ }
+ },
+ "config": {
+ "headerText": "config",
+ "isFilterable": true,
+ "isSortable": true,
+ "valuesAndCounts": {
+ "TODO": 10
+ }
+ },
+ "resultType": {
+ "headerText": "resultType",
+ "isFilterable": true,
+ "isSortable": true,
+ "valuesAndCounts": {
+ "failed": 2,
+ "no-comparison": 7,
+ "succeeded": 1
+ }
+ },
+ "test": {
+ "headerText": "test",
+ "isFilterable": true,
+ "isSortable": true,
+ "valuesAndCounts": {
+ "3x3bitmaprect": 2,
+ "aaclip": 1,
+ "bigblurs": 2,
+ "bitmapsource": 1,
+ "displacement": 1,
+ "filterbitmap_checkerboard_192_192": 1,
+ "filterbitmap_checkerboard_32_2": 1,
+ "texdata": 1
+ }
+ }
+ },
+ "header": {
+ "dataHash": "-8308658702264135120",
+ "isEditable": false,
+ "isExported": true,
+ "schemaVersion": 2,
+ "timeNextUpdateAvailable": null,
+ "timeUpdated": 12345678,
+ "type": "all"
+ },
+ "imagePairs": [
+ {
+ "extraColumns": {
+ "builder": "Test-Android-GalaxyNexus-SGX540-Arm7-Release",
+ "config": "TODO",
+ "resultType": "no-comparison",
+ "test": "texdata"
+ },
+ "imageAUrl": null,
+ "imageBUrl": "bitmap-64bitMD5/texdata/3695033638604474475.png",
+ "isDifferent": false
+ },
+ {
+ "extraColumns": {
+ "builder": "Test-Android-GalaxyNexus-SGX540-Arm7-Release",
+ "config": "TODO",
+ "resultType": "no-comparison",
+ "test": "filterbitmap_checkerboard_32_2"
+ },
+ "imageAUrl": "bitmap-64bitMD5/filterbitmap_checkerboard_32_2/712827739969462165.png",
+ "imageBUrl": null,
+ "isDifferent": false
+ },
+ {
+ "extraColumns": {
+ "builder": "Test-Android-GalaxyNexus-SGX540-Arm7-Release",
+ "config": "TODO",
+ "resultType": "no-comparison",
+ "test": "filterbitmap_checkerboard_192_192"
+ },
+ "imageAUrl": "bitmap-64bitMD5/filterbitmap_checkerboard_192_192/3154864687054945306.png",
+ "imageBUrl": null,
+ "isDifferent": false
+ },
+ {
+ "extraColumns": {
+ "builder": "Test-Android-GalaxyNexus-SGX540-Arm7-Release",
+ "config": "TODO",
+ "resultType": "no-comparison",
+ "test": "bigblurs"
+ },
+ "imageAUrl": "bitmap-64bitMD5/bigblurs/17309852422285247848.png",
+ "imageBUrl": null,
+ "isDifferent": false
+ },
+ {
+ "extraColumns": {
+ "builder": "Test-Android-GalaxyNexus-SGX540-Arm7-Release",
+ "config": "TODO",
+ "resultType": "no-comparison",
+ "test": "bitmapsource"
+ },
+ "imageAUrl": "bitmap-64bitMD5/bitmapsource/16289727936158057543.png",
+ "imageBUrl": null,
+ "isDifferent": false
+ },
+ {
+ "extraColumns": {
+ "builder": "Test-Android-GalaxyNexus-SGX540-Arm7-Release",
+ "config": "TODO",
+ "resultType": "no-comparison",
+ "test": "3x3bitmaprect"
+ },
+ "imageAUrl": "bitmap-64bitMD5/3x3bitmaprect/2054956815327187963.png",
+ "imageBUrl": null,
+ "isDifferent": false
+ },
+ {
+ "extraColumns": {
+ "builder": "Test-Android-GalaxyNexus-SGX540-Arm7-Release",
+ "config": "TODO",
+ "resultType": "no-comparison",
+ "test": "aaclip"
+ },
+ "imageAUrl": "bitmap-64bitMD5/aaclip/14456211900777561488.png",
+ "imageBUrl": null,
+ "isDifferent": false
+ },
+ {
+ "differenceData": {
+ "maxDiffPerChannel": [
+ 136,
+ 68,
+ 34
+ ],
+ "numDifferingPixels": 6081,
+ "percentDifferingPixels": 2.4324,
+ "perceptualDifference": 1.917199999999994,
+ "weightedDiffMeasure": 0.06644571780084584
+ },
+ "extraColumns": {
+ "builder": "Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug",
+ "config": "TODO",
+ "resultType": "failed",
+ "test": "displacement"
+ },
+ "imageAUrl": "bitmap-64bitMD5/displacement/11401048196735046263.png",
+ "imageBUrl": "bitmap-64bitMD5/displacement/5698561127291561694.png",
+ "isDifferent": true
+ },
+ {
+ "differenceData": {
+ "maxDiffPerChannel": [
+ 255,
+ 221,
+ 221
+ ],
+ "numDifferingPixels": 50097,
+ "percentDifferingPixels": 30.5767822265625,
+ "perceptualDifference": 3.3917,
+ "weightedDiffMeasure": 1.4826590790651433
+ },
+ "extraColumns": {
+ "builder": "Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug",
+ "config": "TODO",
+ "resultType": "failed",
+ "test": "bigblurs"
+ },
+ "imageAUrl": "bitmap-64bitMD5/bigblurs/17309852422285247848.png",
+ "imageBUrl": "bitmap-64bitMD5/bigblurs/1822195599289208664.png",
+ "isDifferent": true
+ },
+ {
+ "extraColumns": {
+ "builder": "Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug",
+ "config": "TODO",
+ "resultType": "succeeded",
+ "test": "3x3bitmaprect"
+ },
+ "imageAUrl": "bitmap-64bitMD5/3x3bitmaprect/2054956815327187963.png",
+ "imageBUrl": "bitmap-64bitMD5/3x3bitmaprect/2054956815327187963.png",
+ "isDifferent": false
+ }
+ ],
+ "imageSets": {
+ "diffs": {
+ "baseUrl": "/static/generated-images/diffs",
+ "description": "color difference per channel"
+ },
+ "imageA": {
+ "baseUrl": "http://chromium-skia-gm.commondatastorage.googleapis.com/gm",
+ "description": "8888"
+ },
+ "imageB": {
+ "baseUrl": "http://chromium-skia-gm.commondatastorage.googleapis.com/gm",
+ "description": "gpu"
+ },
+ "whiteDiffs": {
+ "baseUrl": "/static/generated-images/whitediffs",
+ "description": "differing pixels in white"
+ }
+ }
+}
\ No newline at end of file