rebaseline.py: if expectations dir contains JSON format results, update those instead of image files
Part of Step 3 in https://goto.google.com/ChecksumTransitionDetail
R=senorblanco@chromium.org
Review URL: https://codereview.chromium.org/18348018
git-svn-id: http://skia.googlecode.com/svn/trunk@9910 2bbb7eff-a529-9590-31e7-b0007b416f81
diff --git a/gm/gm_expectations.cpp b/gm/gm_expectations.cpp
index ff6fc2c..9c0c274 100644
--- a/gm/gm_expectations.cpp
+++ b/gm/gm_expectations.cpp
@@ -11,14 +11,13 @@
#define DEBUGFAIL_SEE_STDERR SkDEBUGFAIL("see stderr for message")
-// These constants must be kept in sync with the JSONKEY_ constants in
-// gm_json.py !
+// See gm_json.py for descriptions of each of these JSON keys.
+// These constants must be kept in sync with the ones in that Python file!
const static char kJsonKey_ActualResults[] = "actual-results";
const static char kJsonKey_ActualResults_Failed[] = "failed";
const static char kJsonKey_ActualResults_FailureIgnored[]= "failure-ignored";
const static char kJsonKey_ActualResults_NoComparison[] = "no-comparison";
const static char kJsonKey_ActualResults_Succeeded[] = "succeeded";
-
const static char kJsonKey_ExpectedResults[] = "expected-results";
const static char kJsonKey_ExpectedResults_AllowedDigests[] = "allowed-digests";
const static char kJsonKey_ExpectedResults_IgnoreFailure[] = "ignore-failure";
diff --git a/gm/gm_json.py b/gm/gm_json.py
index cd41984..35b954e 100644
--- a/gm/gm_json.py
+++ b/gm/gm_json.py
@@ -15,18 +15,38 @@
import json
+# Key strings used in GM results JSON files (both expected-results.json and
+# actual-results.json).
+#
# These constants must be kept in sync with the kJsonKey_ constants in
# gm_expectations.cpp !
+
JSONKEY_ACTUALRESULTS = 'actual-results'
+# Tests whose results failed to match expectations.
JSONKEY_ACTUALRESULTS_FAILED = 'failed'
+# Tests whose results failed to match expectations, but IGNOREFAILURE causes
+# us to take them less seriously.
JSONKEY_ACTUALRESULTS_FAILUREIGNORED = 'failure-ignored'
+# Tests for which we do not have any expectations. They may be new tests that
+# we haven't had a chance to check in expectations for yet, or we may have
+# consciously decided to leave them without expectations because we are unhappy
+# with the results (although we should try to move away from that, and instead
+# check in expectations with the IGNOREFAILURE flag set).
JSONKEY_ACTUALRESULTS_NOCOMPARISON = 'no-comparison'
+# Tests whose results matched their expectations.
JSONKEY_ACTUALRESULTS_SUCCEEDED = 'succeeded'
JSONKEY_EXPECTEDRESULTS = 'expected-results'
+# One or more [HashType/DigestValue] pairs representing valid results for this
+# test. Typically, there will just be one pair, but we allow for multiple
+# expectations, and the test will pass if any one of them is matched.
JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS = 'allowed-digests'
+# If IGNOREFAILURE is set to True, a failure of this test will be reported
+# within the FAILUREIGNORED section (thus NOT causing the buildbots to go red)
+# rather than the FAILED section (which WOULD cause the buildbots to go red).
JSONKEY_EXPECTEDRESULTS_IGNOREFAILURE = 'ignore-failure'
+# Allowed hash types for test expectations.
JSONKEY_HASHTYPE_BITMAP_64BITMD5 = 'bitmap-64bitMD5'
def LoadFromString(file_contents):
@@ -45,3 +65,8 @@
above."""
file_contents = open(file_path, 'r').read()
return LoadFromString(file_contents)
+
+def WriteToFile(json_dict, file_path):
+ """Writes the JSON summary in json_dict out to file_path."""
+ with open(file_path, 'w') as outfile:
+ json.dump(json_dict, outfile, sort_keys=True, indent=2)
diff --git a/tools/rebaseline.py b/tools/rebaseline.py
index a1ea699..40d6c61 100755
--- a/tools/rebaseline.py
+++ b/tools/rebaseline.py
@@ -17,7 +17,6 @@
import argparse
import os
import re
-import subprocess
import sys
import urllib2
@@ -41,8 +40,6 @@
sys.path.append(GM_DIRECTORY)
import gm_json
-JSON_EXPECTATIONS_FILENAME='expected-results.json'
-
# Mapping of gm-expectations subdir (under
# https://skia.googlecode.com/svn/gm-expected/ )
# to builder name (see list at http://108.170.217.252:10117/builders )
@@ -78,130 +75,45 @@
pass
# Object that rebaselines a JSON expectations file (not individual image files).
-#
-# TODO(epoger): Most of this is just the code from the old ImageRebaseliner...
-# some of it will need to be updated in order to properly rebaseline JSON files.
-# There is a lot of code duplicated between here and ImageRebaseliner, but
-# that's fine because we will delete ImageRebaseliner soon.
class JsonRebaseliner(object):
# params:
- # expectations_root: root directory of all expectations
- # json_base_url: base URL from which to read json_filename
- # json_filename: filename (under json_base_url) from which to read a
- # summary of results; typically "actual-results.json"
+ # expectations_root: root directory of all expectations JSON files
+ # expectations_filename: filename (under expectations_root) of JSON
+ # expectations file; typically
+ # "expected-results.json"
+ # actuals_base_url: base URL from which to read actual-result JSON files
+ # actuals_filename: filename (under actuals_base_url) from which to read a
+ # summary of results; typically "actual-results.json"
# tests: list of tests to rebaseline, or None if we should rebaseline
# whatever files the JSON results summary file tells us to
# configs: which configs to run for each test; this should only be
# specified if the list of tests was also specified (otherwise,
# the JSON file will give us test names and configs)
- # dry_run: if True, instead of actually downloading files or adding
- # files to checkout, display a list of operations that
- # we would normally perform
# add_new: if True, add expectations for tests which don't have any yet
- # missing_json_is_fatal: whether to halt execution if we cannot read a
- # JSON actual result summary file
- def __init__(self, expectations_root, json_base_url, json_filename,
- tests=None, configs=None, dry_run=False,
- add_new=False, missing_json_is_fatal=False):
- raise ValueError('JsonRebaseliner not yet implemented') # TODO(epoger)
+ def __init__(self, expectations_root, expectations_filename,
+ actuals_base_url, actuals_filename,
+ tests=None, configs=None, add_new=False):
if configs and not tests:
raise ValueError('configs should only be specified if tests ' +
'were specified also')
self._expectations_root = expectations_root
+ self._expectations_filename = expectations_filename
self._tests = tests
self._configs = configs
- self._json_base_url = json_base_url
- self._json_filename = json_filename
- self._dry_run = dry_run
+ self._actuals_base_url = actuals_base_url
+ self._actuals_filename = actuals_filename
self._add_new = add_new
- self._missing_json_is_fatal = missing_json_is_fatal
- self._googlestorage_gm_actuals_root = (
- 'http://chromium-skia-gm.commondatastorage.googleapis.com/gm')
self._testname_pattern = re.compile('(\S+)_(\S+).png')
- self._is_svn_checkout = (
- os.path.exists('.svn') or
- os.path.exists(os.path.join(os.pardir, '.svn')))
- self._is_git_checkout = (
- os.path.exists('.git') or
- os.path.exists(os.path.join(os.pardir, '.git')))
- # If dry_run is False, execute subprocess.call(cmd).
- # If dry_run is True, print the command we would have otherwise run.
- # Raises a CommandFailedException if the command fails.
- def _Call(self, cmd):
- if self._dry_run:
- print '%s' % ' '.join(cmd)
- return
- if subprocess.call(cmd) != 0:
- raise CommandFailedException('error running command: ' +
- ' '.join(cmd))
-
- # Download a single actual result from GoogleStorage, returning True if it
- # succeeded.
- def _DownloadFromGoogleStorage(self, infilename, outfilename, all_results):
- test_name = self._testname_pattern.match(infilename).group(1)
- if not test_name:
- print '# unable to find test_name for infilename %s' % infilename
- return False
- try:
- hash_type, hash_value = all_results[infilename]
- except KeyError:
- print ('# unable to find filename %s in all_results dict' %
- infilename)
- return False
- except ValueError as e:
- print '# ValueError reading filename %s from all_results dict: %s'%(
- infilename, e)
- return False
- url = '%s/%s/%s/%s.png' % (self._googlestorage_gm_actuals_root,
- hash_type, test_name, hash_value)
- try:
- self._DownloadFile(source_url=url, dest_filename=outfilename)
- return True
- except CommandFailedException:
- print '# Couldn\'t fetch gs_url %s' % url
- return False
-
- # Download a single actual result from skia-autogen, returning True if it
- # succeeded.
- def _DownloadFromAutogen(self, infilename, outfilename,
- expectations_subdir, builder_name):
- url = ('http://skia-autogen.googlecode.com/svn/gm-actual/' +
- expectations_subdir + '/' + builder_name + '/' +
- expectations_subdir + '/' + infilename)
- try:
- self._DownloadFile(source_url=url, dest_filename=outfilename)
- return True
- except CommandFailedException:
- print '# Couldn\'t fetch autogen_url %s' % url
- return False
-
- # Download a single file, raising a CommandFailedException if it fails.
- def _DownloadFile(self, source_url, dest_filename):
- # Download into a temporary file and then rename it afterwards,
- # so that we don't corrupt the existing file if it fails midway thru.
- temp_filename = os.path.join(os.path.dirname(dest_filename),
- '.temp-' + os.path.basename(dest_filename))
-
- # TODO(epoger): Replace calls to "curl"/"mv" (which will only work on
- # Unix) with a Python HTTP library (which should work cross-platform)
- self._Call([ 'curl', '--fail', '--silent', source_url,
- '--output', temp_filename ])
- self._Call([ 'mv', temp_filename, dest_filename ])
-
- # Returns the full contents of a URL, as a single string.
- #
- # Unlike standard URL handling, we allow relative "file:" URLs;
- # for example, "file:one/two" resolves to the file ./one/two
- # (relative to current working dir)
- def _GetContentsOfUrl(self, url):
- file_prefix = 'file:'
- if url.startswith(file_prefix):
- filename = url[len(file_prefix):]
- return open(filename, 'r').read()
+ # Returns the full contents of filepath, as a single string.
+ # If filepath looks like a URL, try to read it that way instead of as
+ # a path on local storage.
+ def _GetFileContents(self, filepath):
+ if filepath.startswith('http:') or filepath.startswith('https:'):
+ return urllib2.urlopen(filepath).read()
else:
- return urllib2.urlopen(url).read()
+ return open(filepath, 'r').read()
# Returns a dictionary of actual results from actual-results.json file.
#
@@ -212,10 +124,8 @@
# u'shadertext3_8888.png': [u'bitmap-64bitMD5', 3713708307125704716]
# }
#
- # If the JSON actual result summary file cannot be loaded, the behavior
- # depends on self._missing_json_is_fatal:
- # - if true: execution will halt with an exception
- # - if false: we will log an error message but return an empty dictionary
+ # If the JSON actual result summary file cannot be loaded, raise an
+ # exception.
#
# params:
# json_url: URL pointing to a JSON actual result summary file
@@ -224,16 +134,7 @@
# gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON] ;
# if None, then include ALL sections.
def _GetActualResults(self, json_url, sections=None):
- try:
- json_contents = self._GetContentsOfUrl(json_url)
- except (urllib2.HTTPError, IOError):
- message = 'unable to load JSON summary URL %s' % json_url
- if self._missing_json_is_fatal:
- raise ValueError(message)
- else:
- print '# %s' % message
- return {}
-
+ json_contents = self._GetFileContents(json_url)
json_dict = gm_json.LoadFromString(json_contents)
results_to_return = {}
actual_results = json_dict[gm_json.JSONKEY_ACTUALRESULTS]
@@ -245,110 +146,6 @@
results_to_return.update(section_results)
return results_to_return
- # Returns a list of files that require rebaselining.
- #
- # Note that this returns a list of FILES, like this:
- # ['imageblur_565.png', 'xfermodes_pdf.png']
- # rather than a list of TESTS, like this:
- # ['imageblur', 'xfermodes']
- #
- # params:
- # json_url: URL pointing to a JSON actual result summary file
- # add_new: if True, then return files listed in any of these sections:
- # - JSONKEY_ACTUALRESULTS_FAILED
- # - JSONKEY_ACTUALRESULTS_NOCOMPARISON
- # if False, then return files listed in these sections:
- # - JSONKEY_ACTUALRESULTS_FAILED
- #
- def _GetFilesToRebaseline(self, json_url, add_new):
- if self._dry_run:
- print ''
- print '#'
- print ('# Getting files to rebaseline from JSON summary URL %s ...'
- % json_url)
- sections = [gm_json.JSONKEY_ACTUALRESULTS_FAILED]
- if add_new:
- sections.append(gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON)
- results_to_rebaseline = self._GetActualResults(json_url=json_url,
- sections=sections)
- files_to_rebaseline = results_to_rebaseline.keys()
- files_to_rebaseline.sort()
- print '# ... found files_to_rebaseline %s' % files_to_rebaseline
- if self._dry_run:
- print '#'
- return files_to_rebaseline
-
- # Rebaseline a single file.
- def _RebaselineOneFile(self, expectations_subdir, builder_name,
- infilename, outfilename, all_results):
- if self._dry_run:
- print ''
- print '# ' + infilename
-
- # First try to download this result image from Google Storage.
- # If that fails, try skia-autogen.
- # If that fails too, just go on to the next file.
- #
- # This not treated as a fatal failure because not all
- # platforms generate all configs (e.g., Android does not
- # generate PDF).
- #
- # TODO(epoger): Once we are downloading only files that the
- # actual-results.json file told us to, this should become a
- # fatal error. (If the actual-results.json file told us that
- # the test failed with XXX results, we should be able to download
- # those results every time.)
- if not self._DownloadFromGoogleStorage(infilename=infilename,
- outfilename=outfilename,
- all_results=all_results):
- if not self._DownloadFromAutogen(infilename=infilename,
- outfilename=outfilename,
- expectations_subdir=expectations_subdir,
- builder_name=builder_name):
- print '# Couldn\'t fetch infilename ' + infilename
- return
-
- # Add this file to version control (if appropriate).
- if self._add_new:
- if self._is_svn_checkout:
- cmd = [ 'svn', 'add', '--quiet', outfilename ]
- self._Call(cmd)
- cmd = [ 'svn', 'propset', '--quiet', 'svn:mime-type',
- 'image/png', outfilename ];
- self._Call(cmd)
- elif self._is_git_checkout:
- cmd = [ 'git', 'add', outfilename ]
- self._Call(cmd)
-
- # Rebaseline the given configs for a single test.
- #
- # params:
- # expectations_subdir
- # builder_name
- # test: a single test to rebaseline
- # all_results: a dictionary of all actual results
- def _RebaselineOneTest(self, expectations_subdir, builder_name, test,
- all_results):
- if self._configs:
- configs = self._configs
- else:
- if (expectations_subdir == 'base-shuttle-win7-intel-angle'):
- configs = [ 'angle', 'anglemsaa16' ]
- else:
- configs = [ '565', '8888', 'gpu', 'pdf', 'mesa', 'msaa16',
- 'msaa4' ]
- if self._dry_run:
- print ''
- print '# ' + expectations_subdir + ':'
- for config in configs:
- infilename = test + '_' + config + '.png'
- outfilename = os.path.join(expectations_subdir, infilename);
- self._RebaselineOneFile(expectations_subdir=expectations_subdir,
- builder_name=builder_name,
- infilename=infilename,
- outfilename=outfilename,
- all_results=all_results)
-
# Rebaseline all tests/types we specified in the constructor,
# within this gm-expectations subdir.
#
@@ -356,61 +153,101 @@
# subdir : e.g. 'base-shuttle-win7-intel-float'
# builder : e.g. 'Test-Win7-ShuttleA-HD2000-x86-Release'
def RebaselineSubdir(self, subdir, builder):
- json_url = '/'.join([self._json_base_url,
- subdir, builder, subdir,
- self._json_filename])
- all_results = self._GetActualResults(json_url=json_url)
+ # Read in the actual result summary, and extract all the tests whose
+ # results we need to update.
+ actuals_url = '/'.join([self._actuals_base_url,
+ subdir, builder, subdir,
+ self._actuals_filename])
+ sections = [gm_json.JSONKEY_ACTUALRESULTS_FAILED]
+ if self._add_new:
+ sections.append(gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON)
+ results_to_update = self._GetActualResults(json_url=actuals_url,
+ sections=sections)
- if self._tests:
- for test in self._tests:
- self._RebaselineOneTest(expectations_subdir=subdir,
- builder_name=builder,
- test=test, all_results=all_results)
- else: # get the raw list of files that need rebaselining from JSON
- filenames = self._GetFilesToRebaseline(json_url=json_url,
- add_new=self._add_new)
- for filename in filenames:
- outfilename = os.path.join(subdir, filename);
- self._RebaselineOneFile(expectations_subdir=subdir,
- builder_name=builder,
- infilename=filename,
- outfilename=outfilename,
- all_results=all_results)
+ # Read in current expectations.
+ expectations_json_filepath = os.path.join(
+ self._expectations_root, subdir, self._expectations_filename)
+ expectations_dict = gm_json.LoadFromFile(expectations_json_filepath)
+
+ # Update the expectations in memory, skipping any tests/configs that
+ # the caller asked to exclude.
+ skipped_images = []
+ if results_to_update:
+ for (image_name, image_results) in results_to_update.iteritems():
+ (test, config) = self._testname_pattern.match(image_name).groups()
+ if self._tests:
+ if test not in self._tests:
+ skipped_images.append(image_name)
+ continue
+ if self._configs:
+ if config not in self._configs:
+ skipped_images.append(image_name)
+ continue
+ expectations_dict[gm_json.JSONKEY_EXPECTEDRESULTS] \
+ [image_name] \
+ [gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS] = \
+ [image_results]
+
+ # Write out updated expectations.
+ gm_json.WriteToFile(expectations_dict, expectations_json_filepath)
+
+ if skipped_images:
+ print ('Skipped these tests due to test/config filters: %s' %
+ skipped_images)
+
# main...
parser = argparse.ArgumentParser()
+parser.add_argument('--actuals-base-url',
+ help='base URL from which to read files containing JSON ' +
+ 'summaries of actual GM results; defaults to %(default)s',
+ default='http://skia-autogen.googlecode.com/svn/gm-actual')
+parser.add_argument('--actuals-filename',
+ help='filename (within platform-specific subdirectories ' +
+ 'of ACTUALS_BASE_URL) to read a summary of results from; ' +
+ 'defaults to %(default)s',
+ default='actual-results.json')
+# TODO(epoger): Add test that exercises --add-new argument.
parser.add_argument('--add-new', action='store_true',
help='in addition to the standard behavior of ' +
'updating expectations for failing tests, add ' +
'expectations for tests which don\'t have expectations ' +
'yet.')
+# TODO(epoger): Add test that exercises --configs argument.
+# TODO(epoger): Once we are only rebaselining JSON files, update the helpstring
+# to indicate that this is a *filter* over the config names that
+# actual-results.json tells us need to be rebaselined.
+# You don't need to specify tests also, etc.
parser.add_argument('--configs', metavar='CONFIG', nargs='+',
help='which configurations to rebaseline, e.g. ' +
'"--configs 565 8888"; if unspecified, run a default ' +
'set of configs. This should ONLY be specified if ' +
'--tests has also been specified.')
+# TODO(epoger): The --dry-run argument will no longer be needed once we
+# are only rebaselining JSON files.
parser.add_argument('--dry-run', action='store_true',
help='instead of actually downloading files or adding ' +
'files to checkout, display a list of operations that ' +
'we would normally perform')
+parser.add_argument('--expectations-filename',
+ help='filename (under EXPECTATIONS_ROOT) to read ' +
+ 'current expectations from, and to write new ' +
+ 'expectations into; defaults to %(default)s',
+ default='expected-results.json')
parser.add_argument('--expectations-root',
help='root of expectations directory to update-- should ' +
'contain one or more base-* subdirectories. Defaults to ' +
'%(default)s',
default='.')
-parser.add_argument('--json-base-url',
- help='base URL from which to read JSON_FILENAME ' +
- 'files; defaults to %(default)s',
- default='http://skia-autogen.googlecode.com/svn/gm-actual')
-parser.add_argument('--json-filename',
- help='filename (under JSON_BASE_URL) to read a summary ' +
- 'of results from; defaults to %(default)s',
- default='actual-results.json')
parser.add_argument('--subdirs', metavar='SUBDIR', nargs='+',
help='which platform subdirectories to rebaseline; ' +
'if unspecified, rebaseline all subdirs, same as ' +
'"--subdirs %s"' % ' '.join(sorted(SUBDIR_MAPPING.keys())))
+# TODO(epoger): Add test that exercises --tests argument.
+# TODO(epoger): Once we are only rebaselining JSON files, update the helpstring
+# to indicate that this is a *filter* over the test names that
+# actual-results.json tells us need to be rebaselined.
parser.add_argument('--tests', metavar='TEST', nargs='+',
help='which tests to rebaseline, e.g. ' +
'"--tests aaclip bigmatrix"; if unspecified, then all ' +
@@ -438,25 +275,22 @@
#
# See https://goto.google.com/ChecksumTransitionDetail
expectations_json_file = os.path.join(args.expectations_root, subdir,
- JSON_EXPECTATIONS_FILENAME)
+ args.expectations_filename)
if os.path.isfile(expectations_json_file):
- sys.stderr.write('ERROR: JsonRebaseliner is not implemented yet.\n')
- sys.exit(1)
rebaseliner = JsonRebaseliner(
expectations_root=args.expectations_root,
+ expectations_filename=args.expectations_filename,
tests=args.tests, configs=args.configs,
- dry_run=args.dry_run,
- json_base_url=args.json_base_url,
- json_filename=args.json_filename,
- add_new=args.add_new,
- missing_json_is_fatal=missing_json_is_fatal)
+ actuals_base_url=args.actuals_base_url,
+ actuals_filename=args.actuals_filename,
+ add_new=args.add_new)
else:
rebaseliner = rebaseline_imagefiles.ImageRebaseliner(
expectations_root=args.expectations_root,
tests=args.tests, configs=args.configs,
dry_run=args.dry_run,
- json_base_url=args.json_base_url,
- json_filename=args.json_filename,
+ json_base_url=args.actuals_base_url,
+ json_filename=args.actuals_filename,
add_new=args.add_new,
missing_json_is_fatal=missing_json_is_fatal)
rebaseliner.RebaselineSubdir(subdir=subdir, builder=builder)
diff --git a/tools/tests/rebaseline/output/all/output-expected/command_line b/tools/tests/rebaseline/output/all/output-expected/command_line
index f51a2b2..ac35415 100644
--- a/tools/tests/rebaseline/output/all/output-expected/command_line
+++ b/tools/tests/rebaseline/output/all/output-expected/command_line
@@ -1 +1 @@
-python tools/rebaseline.py --dry-run --json-base-url file:nonexistent-path --tests test1 test2
+python tools/rebaseline.py --dry-run --actuals-base-url file:nonexistent-path --tests test1 test2
diff --git a/tools/tests/rebaseline/output/subset/output-expected/command_line b/tools/tests/rebaseline/output/subset/output-expected/command_line
index ac278d2..d66dc01 100644
--- a/tools/tests/rebaseline/output/subset/output-expected/command_line
+++ b/tools/tests/rebaseline/output/subset/output-expected/command_line
@@ -1 +1 @@
-python tools/rebaseline.py --dry-run --expectations-root fake/expectations/path --json-base-url file:tools/tests/rebaseline/input/json1 --tests test1 test2 --configs 565 8888 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float
+python tools/rebaseline.py --dry-run --expectations-root fake/expectations/path --actuals-base-url file:tools/tests/rebaseline/input/json1 --tests test1 test2 --configs 565 8888 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float
diff --git a/tools/tests/rebaseline/output/using-json1-add-new/output-expected/command_line b/tools/tests/rebaseline/output/using-json1-add-new/output-expected/command_line
index 32bd0ef..9bd9539 100644
--- a/tools/tests/rebaseline/output/using-json1-add-new/output-expected/command_line
+++ b/tools/tests/rebaseline/output/using-json1-add-new/output-expected/command_line
@@ -1 +1 @@
-python tools/rebaseline.py --dry-run --json-base-url file:tools/tests/rebaseline/input/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float --add-new
+python tools/rebaseline.py --dry-run --actuals-base-url file:tools/tests/rebaseline/input/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float --add-new
diff --git a/tools/tests/rebaseline/output/using-json1-expectations/output-expected/command_line b/tools/tests/rebaseline/output/using-json1-expectations/output-expected/command_line
index 7629358..ac6be7d 100644
--- a/tools/tests/rebaseline/output/using-json1-expectations/output-expected/command_line
+++ b/tools/tests/rebaseline/output/using-json1-expectations/output-expected/command_line
@@ -1 +1 @@
-python tools/rebaseline.py --expectations-root tools/tests/rebaseline/output/using-json1-expectations/output-actual/gm-expectations --json-base-url file:tools/tests/rebaseline/input/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float
+python tools/rebaseline.py --expectations-root tools/tests/rebaseline/output/using-json1-expectations/output-actual/gm-expectations --actuals-base-url tools/tests/rebaseline/input/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float
diff --git a/tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/base-android-galaxy-nexus/expected-results.json b/tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/base-android-galaxy-nexus/expected-results.json
index d7608cd..20721f3 100644
--- a/tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/base-android-galaxy-nexus/expected-results.json
+++ b/tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/base-android-galaxy-nexus/expected-results.json
@@ -1,52 +1,70 @@
{
- "expected-results" : {
- "3x3bitmaprect_565.png" : {
- "allowed-digests" : null,
- "ignore-failure" : false
- },
- "3x3bitmaprect_8888.png" : {
- "allowed-digests" : null,
- "ignore-failure" : false
- },
- "aaclip_gpu.png" : {
- "allowed-digests" : [
- [ "bitmap-64bitMD5", 11899819492385205974 ]
- ],
- "ignore-failure" : false
- },
- "aarectmodes_565.png" : {
- "allowed-digests" : [
- [ "bitmap-64bitMD5", 14760033689012826769 ]
- ],
- "ignore-failure" : false
- },
- "imageblur_565.png" : {
- "allowed-digests" : [
- [ "bitmap-64bitMD5", 17796243856503591523 ]
- ],
- "ignore-failure" : false
- },
- "imageblur_8888.png" : {
- "allowed-digests" : [
- [ "bitmap-64bitMD5", 7426416989687670152 ]
- ],
- "ignore-failure" : false
- },
- "shadertext3_8888.png" : {
- "allowed-digests" : [
- [ "bitmap-64bitMD5", 10593797161686785561 ]
- ],
- "ignore-failure" : false
- },
- "xfermodeimagefilter_pdf.png" : {
- "allowed-digests" : null,
- "ignore-failure" : false
- },
- "xfermodes_pdf.png" : {
- "allowed-digests" : [
- [ "bitmap-64bitMD5", 9151974350149210736 ]
- ],
- "ignore-failure" : false
- }
- }
-}
+ "expected-results": {
+ "3x3bitmaprect_565.png": {
+ "allowed-digests": null,
+ "ignore-failure": false
+ },
+ "3x3bitmaprect_8888.png": {
+ "allowed-digests": null,
+ "ignore-failure": false
+ },
+ "aaclip_gpu.png": {
+ "allowed-digests": [
+ [
+ "bitmap-64bitMD5",
+ 11899819492385205974
+ ]
+ ],
+ "ignore-failure": false
+ },
+ "aarectmodes_565.png": {
+ "allowed-digests": [
+ [
+ "bitmap-64bitMD5",
+ 14760033689012826769
+ ]
+ ],
+ "ignore-failure": false
+ },
+ "imageblur_565.png": {
+ "allowed-digests": [
+ [
+ "bitmap-64bitMD5",
+ 3359963596899141322
+ ]
+ ],
+ "ignore-failure": false
+ },
+ "imageblur_8888.png": {
+ "allowed-digests": [
+ [
+ "bitmap-64bitMD5",
+ 4217923806027861152
+ ]
+ ],
+ "ignore-failure": false
+ },
+ "shadertext3_8888.png": {
+ "allowed-digests": [
+ [
+ "bitmap-64bitMD5",
+ 3713708307125704716
+ ]
+ ],
+ "ignore-failure": false
+ },
+ "xfermodeimagefilter_pdf.png": {
+ "allowed-digests": null,
+ "ignore-failure": false
+ },
+ "xfermodes_pdf.png": {
+ "allowed-digests": [
+ [
+ "bitmap-64bitMD5",
+ 9151974350149210736
+ ]
+ ],
+ "ignore-failure": false
+ }
+ }
+}
\ No newline at end of file
diff --git a/tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/base-shuttle-win7-intel-float/expected-results.json b/tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/base-shuttle-win7-intel-float/expected-results.json
index b314ed6..20721f3 100644
--- a/tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/base-shuttle-win7-intel-float/expected-results.json
+++ b/tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/base-shuttle-win7-intel-float/expected-results.json
@@ -1,52 +1,70 @@
{
- "expected-results" : {
- "3x3bitmaprect_565.png" : {
- "allowed-digests" : null,
- "ignore-failure" : false
- },
- "3x3bitmaprect_8888.png" : {
- "allowed-digests" : null,
- "ignore-failure" : false
- },
- "aaclip_gpu.png" : {
- "allowed-digests" : [
- [ "bitmap-64bitMD5", 11899819492385205974 ]
- ],
- "ignore-failure" : false
- },
- "aarectmodes_565.png" : {
- "allowed-digests" : [
- [ "bitmap-64bitMD5", 14760033689012826769 ]
- ],
- "ignore-failure" : false
- },
- "imageblur_565.png" : {
- "allowed-digests" : [
- [ "bitmap-64bitMD5", 3359963596899141322 ]
- ],
- "ignore-failure" : false
- },
- "imageblur_8888.png" : {
- "allowed-digests" : [
- [ "bitmap-64bitMD5", 4217923806027861152 ]
- ],
- "ignore-failure" : false
- },
- "shadertext3_8888.png" : {
- "allowed-digests" : [
- [ "bitmap-64bitMD5", 3713708307125704716 ]
- ],
- "ignore-failure" : false
- },
- "xfermodeimagefilter_pdf.png" : {
- "allowed-digests" : null,
- "ignore-failure" : false
- },
- "xfermodes_pdf.png" : {
- "allowed-digests" : [
- [ "bitmap-64bitMD5", 9151974350149210736 ]
- ],
- "ignore-failure" : false
- }
- }
-}
+ "expected-results": {
+ "3x3bitmaprect_565.png": {
+ "allowed-digests": null,
+ "ignore-failure": false
+ },
+ "3x3bitmaprect_8888.png": {
+ "allowed-digests": null,
+ "ignore-failure": false
+ },
+ "aaclip_gpu.png": {
+ "allowed-digests": [
+ [
+ "bitmap-64bitMD5",
+ 11899819492385205974
+ ]
+ ],
+ "ignore-failure": false
+ },
+ "aarectmodes_565.png": {
+ "allowed-digests": [
+ [
+ "bitmap-64bitMD5",
+ 14760033689012826769
+ ]
+ ],
+ "ignore-failure": false
+ },
+ "imageblur_565.png": {
+ "allowed-digests": [
+ [
+ "bitmap-64bitMD5",
+ 3359963596899141322
+ ]
+ ],
+ "ignore-failure": false
+ },
+ "imageblur_8888.png": {
+ "allowed-digests": [
+ [
+ "bitmap-64bitMD5",
+ 4217923806027861152
+ ]
+ ],
+ "ignore-failure": false
+ },
+ "shadertext3_8888.png": {
+ "allowed-digests": [
+ [
+ "bitmap-64bitMD5",
+ 3713708307125704716
+ ]
+ ],
+ "ignore-failure": false
+ },
+ "xfermodeimagefilter_pdf.png": {
+ "allowed-digests": null,
+ "ignore-failure": false
+ },
+ "xfermodes_pdf.png": {
+ "allowed-digests": [
+ [
+ "bitmap-64bitMD5",
+ 9151974350149210736
+ ]
+ ],
+ "ignore-failure": false
+ }
+ }
+}
\ No newline at end of file
diff --git a/tools/tests/rebaseline/output/using-json1-expectations/output-expected/return_value b/tools/tests/rebaseline/output/using-json1-expectations/output-expected/return_value
index d00491f..573541a 100644
--- a/tools/tests/rebaseline/output/using-json1-expectations/output-expected/return_value
+++ b/tools/tests/rebaseline/output/using-json1-expectations/output-expected/return_value
@@ -1 +1 @@
-1
+0
diff --git a/tools/tests/rebaseline/output/using-json1-expectations/output-expected/stdout b/tools/tests/rebaseline/output/using-json1-expectations/output-expected/stdout
index 43c078e..e69de29 100644
--- a/tools/tests/rebaseline/output/using-json1-expectations/output-expected/stdout
+++ b/tools/tests/rebaseline/output/using-json1-expectations/output-expected/stdout
@@ -1 +0,0 @@
-ERROR: JsonRebaseliner is not implemented yet.
diff --git a/tools/tests/rebaseline/output/using-json1/output-expected/command_line b/tools/tests/rebaseline/output/using-json1/output-expected/command_line
index 14624e1..127846c 100644
--- a/tools/tests/rebaseline/output/using-json1/output-expected/command_line
+++ b/tools/tests/rebaseline/output/using-json1/output-expected/command_line
@@ -1 +1 @@
-python tools/rebaseline.py --dry-run --json-base-url file:tools/tests/rebaseline/input/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float
+python tools/rebaseline.py --dry-run --actuals-base-url file:tools/tests/rebaseline/input/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float
diff --git a/tools/tests/run.sh b/tools/tests/run.sh
index 1a3f40b..bbcfeb9 100755
--- a/tools/tests/run.sh
+++ b/tools/tests/run.sh
@@ -267,13 +267,13 @@
REBASELINE_OUTPUT=tools/tests/rebaseline/output
# These test the old image-file expectations.
-rebaseline_images_test "--expectations-root fake/expectations/path --json-base-url file:$REBASELINE_INPUT/json1 --tests test1 test2 --configs 565 8888 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float" "$REBASELINE_OUTPUT/subset"
-rebaseline_images_test "--json-base-url file:nonexistent-path --tests test1 test2" "$REBASELINE_OUTPUT/all"
-rebaseline_images_test "--json-base-url file:$REBASELINE_INPUT/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float" "$REBASELINE_OUTPUT/using-json1"
-rebaseline_images_test "--json-base-url file:$REBASELINE_INPUT/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float --add-new" "$REBASELINE_OUTPUT/using-json1-add-new"
+rebaseline_images_test "--expectations-root fake/expectations/path --actuals-base-url file:$REBASELINE_INPUT/json1 --tests test1 test2 --configs 565 8888 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float" "$REBASELINE_OUTPUT/subset"
+rebaseline_images_test "--actuals-base-url file:nonexistent-path --tests test1 test2" "$REBASELINE_OUTPUT/all"
+rebaseline_images_test "--actuals-base-url file:$REBASELINE_INPUT/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float" "$REBASELINE_OUTPUT/using-json1"
+rebaseline_images_test "--actuals-base-url file:$REBASELINE_INPUT/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float --add-new" "$REBASELINE_OUTPUT/using-json1-add-new"
# These test the new JSON-format expectations.
-rebaseline_test "$REBASELINE_INPUT/json1" "--json-base-url file:$REBASELINE_INPUT/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float" "$REBASELINE_OUTPUT/using-json1-expectations"
+rebaseline_test "$REBASELINE_INPUT/json1" "--actuals-base-url $REBASELINE_INPUT/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float" "$REBASELINE_OUTPUT/using-json1-expectations"
#
# Test jsondiff.py ...