senorblanco@chromium.org | 782f3b4 | 2012-10-29 18:06:26 +0000 | [diff] [blame] | 1 | #!/usr/bin/python |
| 2 | |
| 3 | ''' |
| 4 | Copyright 2012 Google Inc. |
| 5 | |
| 6 | Use of this source code is governed by a BSD-style license that can be |
| 7 | found in the LICENSE file. |
| 8 | ''' |
| 9 | |
| 10 | ''' |
senorblanco@chromium.org | 123a0b5 | 2012-11-29 21:50:34 +0000 | [diff] [blame] | 11 | Rebaselines the given GM tests, on all bots and all configurations. |
senorblanco@chromium.org | 782f3b4 | 2012-10-29 18:06:26 +0000 | [diff] [blame] | 12 | ''' |
| 13 | |
epoger@google.com | 99ba65a | 2013-06-05 15:43:37 +0000 | [diff] [blame] | 14 | # System-level imports |
epoger@google.com | 9166bf5 | 2013-05-30 15:46:19 +0000 | [diff] [blame] | 15 | import argparse |
epoger@google.com | fd04011 | 2013-08-20 16:21:55 +0000 | [diff] [blame] | 16 | import json |
epoger@google.com | ec3397b | 2013-05-29 17:09:43 +0000 | [diff] [blame] | 17 | import os |
epoger@google.com | e78d207 | 2013-06-12 17:44:14 +0000 | [diff] [blame] | 18 | import re |
epoger@google.com | 27e1c00 | 2013-07-24 15:38:39 +0000 | [diff] [blame] | 19 | import subprocess |
epoger@google.com | ec3397b | 2013-05-29 17:09:43 +0000 | [diff] [blame] | 20 | import sys |
epoger@google.com | 99ba65a | 2013-06-05 15:43:37 +0000 | [diff] [blame] | 21 | import urllib2 |
| 22 | |
| 23 | # Imports from within Skia |
| 24 | # |
epoger@google.com | dad5310 | 2013-06-12 14:25:30 +0000 | [diff] [blame] | 25 | # We need to add the 'gm' directory, so that we can import gm_json.py within |
| 26 | # that directory. That script allows us to parse the actual-results.json file |
| 27 | # written out by the GM tool. |
| 28 | # Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end* |
| 29 | # so any dirs that are already in the PYTHONPATH will be preferred. |
| 30 | # |
| 31 | # This assumes that the 'gm' directory has been checked out as a sibling of |
| 32 | # the 'tools' directory containing this script, which will be the case if |
| 33 | # 'trunk' was checked out as a single unit. |
epoger@google.com | 99ba65a | 2013-06-05 15:43:37 +0000 | [diff] [blame] | 34 | GM_DIRECTORY = os.path.realpath( |
| 35 | os.path.join(os.path.dirname(os.path.dirname(__file__)), 'gm')) |
| 36 | if GM_DIRECTORY not in sys.path: |
epoger@google.com | 2a192a8 | 2013-08-02 20:54:46 +0000 | [diff] [blame] | 37 | sys.path.append(GM_DIRECTORY) |
epoger@google.com | 99ba65a | 2013-06-05 15:43:37 +0000 | [diff] [blame] | 38 | import gm_json |
| 39 | |
epoger@google.com | 44f165f | 2013-08-24 20:45:31 +0000 | [diff] [blame] | 40 | # TODO(epoger): In the long run, we want to build this list automatically, |
| 41 | # but for now we hard-code it until we can properly address |
| 42 | # https://code.google.com/p/skia/issues/detail?id=1544 |
| 43 | # ('live query of builder list makes rebaseline.py slow to start up') |
| 44 | TEST_BUILDERS = [ |
| 45 | 'Test-Android-GalaxyNexus-SGX540-Arm7-Debug', |
| 46 | 'Test-Android-GalaxyNexus-SGX540-Arm7-Release', |
| 47 | 'Test-Android-IntelRhb-SGX544-x86-Debug', |
| 48 | 'Test-Android-IntelRhb-SGX544-x86-Release', |
| 49 | 'Test-Android-Nexus10-MaliT604-Arm7-Debug', |
| 50 | 'Test-Android-Nexus10-MaliT604-Arm7-Release', |
| 51 | 'Test-Android-Nexus4-Adreno320-Arm7-Debug', |
| 52 | 'Test-Android-Nexus4-Adreno320-Arm7-Release', |
| 53 | 'Test-Android-Nexus7-Tegra3-Arm7-Debug', |
| 54 | 'Test-Android-Nexus7-Tegra3-Arm7-Release', |
| 55 | 'Test-Android-NexusS-SGX540-Arm7-Debug', |
| 56 | 'Test-Android-NexusS-SGX540-Arm7-Release', |
| 57 | 'Test-Android-Xoom-Tegra2-Arm7-Debug', |
| 58 | 'Test-Android-Xoom-Tegra2-Arm7-Release', |
| 59 | 'Test-ChromeOS-Alex-GMA3150-x86-Debug', |
| 60 | 'Test-ChromeOS-Alex-GMA3150-x86-Release', |
| 61 | 'Test-ChromeOS-Daisy-MaliT604-Arm7-Debug', |
| 62 | 'Test-ChromeOS-Daisy-MaliT604-Arm7-Release', |
| 63 | 'Test-ChromeOS-Link-HD4000-x86_64-Debug', |
| 64 | 'Test-ChromeOS-Link-HD4000-x86_64-Release', |
| 65 | 'Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug', |
| 66 | 'Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release', |
| 67 | 'Test-Mac10.6-MacMini4.1-GeForce320M-x86_64-Debug', |
| 68 | 'Test-Mac10.6-MacMini4.1-GeForce320M-x86_64-Release', |
| 69 | 'Test-Mac10.7-MacMini4.1-GeForce320M-x86-Debug', |
| 70 | 'Test-Mac10.7-MacMini4.1-GeForce320M-x86-Release', |
| 71 | 'Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug', |
| 72 | 'Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Release', |
| 73 | 'Test-Mac10.8-MacMini4.1-GeForce320M-x86-Debug', |
| 74 | 'Test-Mac10.8-MacMini4.1-GeForce320M-x86-Release', |
| 75 | 'Test-Mac10.8-MacMini4.1-GeForce320M-x86_64-Debug', |
| 76 | 'Test-Mac10.8-MacMini4.1-GeForce320M-x86_64-Release', |
| 77 | 'Test-Ubuntu12-ShuttleA-ATI5770-x86-Debug', |
| 78 | 'Test-Ubuntu12-ShuttleA-ATI5770-x86-Release', |
| 79 | 'Test-Ubuntu12-ShuttleA-ATI5770-x86_64-Debug', |
| 80 | 'Test-Ubuntu12-ShuttleA-ATI5770-x86_64-Release', |
| 81 | 'Test-Ubuntu12-ShuttleA-HD2000-x86_64-Release-Valgrind', |
| 82 | 'Test-Ubuntu12-ShuttleA-NoGPU-x86_64-Debug', |
| 83 | 'Test-Ubuntu13-ShuttleA-HD2000-x86_64-Debug-ASAN', |
| 84 | 'Test-Win7-ShuttleA-HD2000-x86-Debug', |
| 85 | 'Test-Win7-ShuttleA-HD2000-x86-Debug-ANGLE', |
| 86 | 'Test-Win7-ShuttleA-HD2000-x86-Debug-DirectWrite', |
| 87 | 'Test-Win7-ShuttleA-HD2000-x86-Release', |
| 88 | 'Test-Win7-ShuttleA-HD2000-x86-Release-ANGLE', |
| 89 | 'Test-Win7-ShuttleA-HD2000-x86-Release-DirectWrite', |
| 90 | 'Test-Win7-ShuttleA-HD2000-x86_64-Debug', |
| 91 | 'Test-Win7-ShuttleA-HD2000-x86_64-Release', |
commit-bot@chromium.org | bf8ba65 | 2013-12-17 17:29:33 +0000 | [diff] [blame] | 92 | 'Test-Win8-ShuttleA-GTX660-x86-Debug', |
| 93 | 'Test-Win8-ShuttleA-GTX660-x86-Release', |
| 94 | 'Test-Win8-ShuttleA-GTX660-x86-Release-NVPR', |
| 95 | 'Test-Win8-ShuttleA-GTX660-x86_64-Debug', |
| 96 | 'Test-Win8-ShuttleA-GTX660-x86_64-Release', |
| 97 | 'Test-Win8-ShuttleA-HD7770-x86-Debug', |
| 98 | 'Test-Win8-ShuttleA-HD7770-x86-Release', |
| 99 | 'Test-Win8-ShuttleA-HD7770-x86_64-Debug', |
| 100 | 'Test-Win8-ShuttleA-HD7770-x86_64-Release', |
epoger@google.com | 44f165f | 2013-08-24 20:45:31 +0000 | [diff] [blame] | 101 | ] |
epoger@google.com | 9166bf5 | 2013-05-30 15:46:19 +0000 | [diff] [blame] | 102 | |
borenet@google.com | b6e915d | 2013-09-16 18:13:13 +0000 | [diff] [blame] | 103 | # TODO: Get this from builder_name_schema in buildbot. |
| 104 | TRYBOT_SUFFIX = '-Trybot' |
| 105 | |
| 106 | |
epoger@google.com | 66ba9f9 | 2013-07-11 19:20:30 +0000 | [diff] [blame] | 107 | class _InternalException(Exception): |
epoger@google.com | 2a192a8 | 2013-08-02 20:54:46 +0000 | [diff] [blame] | 108 | pass |
epoger@google.com | db29a31 | 2013-06-04 14:58:47 +0000 | [diff] [blame] | 109 | |
epoger@google.com | ffcbdbf | 2013-07-16 17:35:39 +0000 | [diff] [blame] | 110 | class ExceptionHandler(object): |
epoger@google.com | 93acfd2 | 2013-09-03 18:27:19 +0000 | [diff] [blame] | 111 | """ Object that handles exceptions, either raising them immediately or |
| 112 | collecting them to display later on.""" |
epoger@google.com | ffcbdbf | 2013-07-16 17:35:39 +0000 | [diff] [blame] | 113 | |
epoger@google.com | 2a192a8 | 2013-08-02 20:54:46 +0000 | [diff] [blame] | 114 | # params: |
epoger@google.com | 2a192a8 | 2013-08-02 20:54:46 +0000 | [diff] [blame] | 115 | def __init__(self, keep_going_on_failure=False): |
epoger@google.com | 93acfd2 | 2013-09-03 18:27:19 +0000 | [diff] [blame] | 116 | """ |
| 117 | params: |
| 118 | keep_going_on_failure: if False, report failures and quit right away; |
| 119 | if True, collect failures until |
| 120 | ReportAllFailures() is called |
| 121 | """ |
epoger@google.com | 2a192a8 | 2013-08-02 20:54:46 +0000 | [diff] [blame] | 122 | self._keep_going_on_failure = keep_going_on_failure |
| 123 | self._failures_encountered = [] |
epoger@google.com | ffcbdbf | 2013-07-16 17:35:39 +0000 | [diff] [blame] | 124 | |
epoger@google.com | 93acfd2 | 2013-09-03 18:27:19 +0000 | [diff] [blame] | 125 | def RaiseExceptionOrContinue(self): |
| 126 | """ We have encountered an exception; either collect the info and keep |
| 127 | going, or exit the program right away.""" |
| 128 | # Get traceback information about the most recently raised exception. |
| 129 | exc_info = sys.exc_info() |
epoger@google.com | ffcbdbf | 2013-07-16 17:35:39 +0000 | [diff] [blame] | 130 | |
epoger@google.com | 2a192a8 | 2013-08-02 20:54:46 +0000 | [diff] [blame] | 131 | if self._keep_going_on_failure: |
epoger@google.com | 93acfd2 | 2013-09-03 18:27:19 +0000 | [diff] [blame] | 132 | print >> sys.stderr, ('WARNING: swallowing exception %s' % |
| 133 | repr(exc_info[1])) |
| 134 | self._failures_encountered.append(exc_info) |
epoger@google.com | 2a192a8 | 2013-08-02 20:54:46 +0000 | [diff] [blame] | 135 | else: |
epoger@google.com | 2a192a8 | 2013-08-02 20:54:46 +0000 | [diff] [blame] | 136 | print >> sys.stderr, ( |
epoger@google.com | 80f9cf1 | 2013-09-06 17:26:09 +0000 | [diff] [blame] | 137 | '\nHalting at first exception.\n' + |
| 138 | 'Please file a bug to epoger@google.com at ' + |
| 139 | 'https://code.google.com/p/skia/issues/entry, containing the ' + |
| 140 | 'command you ran and the following stack trace.\n\n' + |
| 141 | 'Afterwards, you can re-run with the --keep-going-on-failure ' + |
| 142 | 'option set.\n') |
epoger@google.com | 93acfd2 | 2013-09-03 18:27:19 +0000 | [diff] [blame] | 143 | raise exc_info[1], None, exc_info[2] |
epoger@google.com | ffcbdbf | 2013-07-16 17:35:39 +0000 | [diff] [blame] | 144 | |
epoger@google.com | 2a192a8 | 2013-08-02 20:54:46 +0000 | [diff] [blame] | 145 | def ReportAllFailures(self): |
| 146 | if self._failures_encountered: |
| 147 | print >> sys.stderr, ('Encountered %d failures (see above).' % |
| 148 | len(self._failures_encountered)) |
epoger@google.com | 93acfd2 | 2013-09-03 18:27:19 +0000 | [diff] [blame] | 149 | sys.exit(1) |
epoger@google.com | ffcbdbf | 2013-07-16 17:35:39 +0000 | [diff] [blame] | 150 | |
| 151 | |
epoger@google.com | 99a8ec9 | 2013-06-19 18:56:59 +0000 | [diff] [blame] | 152 | # Object that rebaselines a JSON expectations file (not individual image files). |
epoger@google.com | 99a8ec9 | 2013-06-19 18:56:59 +0000 | [diff] [blame] | 153 | class JsonRebaseliner(object): |
epoger@google.com | 9166bf5 | 2013-05-30 15:46:19 +0000 | [diff] [blame] | 154 | |
epoger@google.com | 2a192a8 | 2013-08-02 20:54:46 +0000 | [diff] [blame] | 155 | # params: |
| 156 | # expectations_root: root directory of all expectations JSON files |
| 157 | # expectations_input_filename: filename (under expectations_root) of JSON |
| 158 | # expectations file to read; typically |
| 159 | # "expected-results.json" |
| 160 | # expectations_output_filename: filename (under expectations_root) to |
| 161 | # which updated expectations should be |
| 162 | # written; typically the same as |
| 163 | # expectations_input_filename, to overwrite |
| 164 | # the old content |
| 165 | # actuals_base_url: base URL from which to read actual-result JSON files |
| 166 | # actuals_filename: filename (under actuals_base_url) from which to read a |
| 167 | # summary of results; typically "actual-results.json" |
| 168 | # exception_handler: reference to rebaseline.ExceptionHandler object |
| 169 | # tests: list of tests to rebaseline, or None if we should rebaseline |
| 170 | # whatever files the JSON results summary file tells us to |
| 171 | # configs: which configs to run for each test, or None if we should |
| 172 | # rebaseline whatever configs the JSON results summary file tells |
| 173 | # us to |
| 174 | # add_new: if True, add expectations for tests which don't have any yet |
bsalomon@google.com | 46484ec | 2013-10-16 15:10:45 +0000 | [diff] [blame] | 175 | # add_ignored: if True, add expectations for tests for which failures are |
| 176 | # currently ignored |
epoger@google.com | 06e626d | 2013-09-03 17:32:15 +0000 | [diff] [blame] | 177 | # bugs: optional list of bug numbers which pertain to these expectations |
| 178 | # notes: free-form text notes to add to all updated expectations |
| 179 | # mark_unreviewed: if True, mark these expectations as NOT having been |
| 180 | # reviewed by a human; otherwise, leave that field blank. |
| 181 | # Currently, there is no way to make this script mark |
| 182 | # expectations as reviewed-by-human=True. |
| 183 | # TODO(epoger): Add that capability to a review tool. |
senorblanco@chromium.org | bfc0d6b | 2013-09-18 19:14:43 +0000 | [diff] [blame] | 184 | # mark_ignore_failure: if True, mark failures of a given test as being |
| 185 | # ignored. |
scroggo@google.com | d23e683 | 2013-10-10 21:09:24 +0000 | [diff] [blame] | 186 | # from_trybot: if True, read actual-result JSON files generated from a |
| 187 | # trybot run rather than a waterfall run. |
epoger@google.com | 2a192a8 | 2013-08-02 20:54:46 +0000 | [diff] [blame] | 188 | def __init__(self, expectations_root, expectations_input_filename, |
| 189 | expectations_output_filename, actuals_base_url, |
| 190 | actuals_filename, exception_handler, |
bsalomon@google.com | 46484ec | 2013-10-16 15:10:45 +0000 | [diff] [blame] | 191 | tests=None, configs=None, add_new=False, add_ignored=False, |
| 192 | bugs=None, notes=None, mark_unreviewed=None, |
| 193 | mark_ignore_failure=False, from_trybot=False): |
epoger@google.com | 2a192a8 | 2013-08-02 20:54:46 +0000 | [diff] [blame] | 194 | self._expectations_root = expectations_root |
| 195 | self._expectations_input_filename = expectations_input_filename |
| 196 | self._expectations_output_filename = expectations_output_filename |
| 197 | self._tests = tests |
| 198 | self._configs = configs |
| 199 | self._actuals_base_url = actuals_base_url |
| 200 | self._actuals_filename = actuals_filename |
| 201 | self._exception_handler = exception_handler |
| 202 | self._add_new = add_new |
bsalomon@google.com | 46484ec | 2013-10-16 15:10:45 +0000 | [diff] [blame] | 203 | self._add_ignored = add_ignored |
epoger@google.com | 06e626d | 2013-09-03 17:32:15 +0000 | [diff] [blame] | 204 | self._bugs = bugs |
| 205 | self._notes = notes |
| 206 | self._mark_unreviewed = mark_unreviewed |
senorblanco@chromium.org | bfc0d6b | 2013-09-18 19:14:43 +0000 | [diff] [blame] | 207 | self._mark_ignore_failure = mark_ignore_failure; |
scroggo@google.com | d23e683 | 2013-10-10 21:09:24 +0000 | [diff] [blame] | 208 | if self._tests or self._configs: |
| 209 | self._image_filename_re = re.compile(gm_json.IMAGE_FILENAME_PATTERN) |
| 210 | else: |
| 211 | self._image_filename_re = None |
epoger@google.com | 2a192a8 | 2013-08-02 20:54:46 +0000 | [diff] [blame] | 212 | self._using_svn = os.path.isdir(os.path.join(expectations_root, '.svn')) |
borenet@google.com | b6e915d | 2013-09-16 18:13:13 +0000 | [diff] [blame] | 213 | self._from_trybot = from_trybot |
epoger@google.com | 27e1c00 | 2013-07-24 15:38:39 +0000 | [diff] [blame] | 214 | |
epoger@google.com | 2a192a8 | 2013-08-02 20:54:46 +0000 | [diff] [blame] | 215 | # Executes subprocess.call(cmd). |
| 216 | # Raises an Exception if the command fails. |
| 217 | def _Call(self, cmd): |
| 218 | if subprocess.call(cmd) != 0: |
| 219 | raise _InternalException('error running command: ' + ' '.join(cmd)) |
epoger@google.com | 9166bf5 | 2013-05-30 15:46:19 +0000 | [diff] [blame] | 220 | |
epoger@google.com | 2a192a8 | 2013-08-02 20:54:46 +0000 | [diff] [blame] | 221 | # Returns the full contents of filepath, as a single string. |
| 222 | # If filepath looks like a URL, try to read it that way instead of as |
| 223 | # a path on local storage. |
| 224 | # |
| 225 | # Raises _InternalException if there is a problem. |
| 226 | def _GetFileContents(self, filepath): |
| 227 | if filepath.startswith('http:') or filepath.startswith('https:'): |
| 228 | try: |
| 229 | return urllib2.urlopen(filepath).read() |
| 230 | except urllib2.HTTPError as e: |
| 231 | raise _InternalException('unable to read URL %s: %s' % ( |
| 232 | filepath, e)) |
| 233 | else: |
| 234 | return open(filepath, 'r').read() |
epoger@google.com | 99ba65a | 2013-06-05 15:43:37 +0000 | [diff] [blame] | 235 | |
epoger@google.com | 2a192a8 | 2013-08-02 20:54:46 +0000 | [diff] [blame] | 236 | # Returns a dictionary of actual results from actual-results.json file. |
| 237 | # |
| 238 | # The dictionary returned has this format: |
| 239 | # { |
| 240 | # u'imageblur_565.png': [u'bitmap-64bitMD5', 3359963596899141322], |
| 241 | # u'imageblur_8888.png': [u'bitmap-64bitMD5', 4217923806027861152], |
| 242 | # u'shadertext3_8888.png': [u'bitmap-64bitMD5', 3713708307125704716] |
| 243 | # } |
| 244 | # |
| 245 | # If the JSON actual result summary file cannot be loaded, logs a warning |
| 246 | # message and returns None. |
| 247 | # If the JSON actual result summary file can be loaded, but we have |
| 248 | # trouble parsing it, raises an Exception. |
| 249 | # |
| 250 | # params: |
| 251 | # json_url: URL pointing to a JSON actual result summary file |
| 252 | # sections: a list of section names to include in the results, e.g. |
| 253 | # [gm_json.JSONKEY_ACTUALRESULTS_FAILED, |
| 254 | # gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON] ; |
| 255 | # if None, then include ALL sections. |
| 256 | def _GetActualResults(self, json_url, sections=None): |
| 257 | try: |
| 258 | json_contents = self._GetFileContents(json_url) |
| 259 | except _InternalException: |
| 260 | print >> sys.stderr, ( |
| 261 | 'could not read json_url %s ; skipping this platform.' % |
| 262 | json_url) |
| 263 | return None |
| 264 | json_dict = gm_json.LoadFromString(json_contents) |
| 265 | results_to_return = {} |
| 266 | actual_results = json_dict[gm_json.JSONKEY_ACTUALRESULTS] |
| 267 | if not sections: |
| 268 | sections = actual_results.keys() |
| 269 | for section in sections: |
| 270 | section_results = actual_results[section] |
| 271 | if section_results: |
| 272 | results_to_return.update(section_results) |
| 273 | return results_to_return |
epoger@google.com | e78d207 | 2013-06-12 17:44:14 +0000 | [diff] [blame] | 274 | |
epoger@google.com | 2a192a8 | 2013-08-02 20:54:46 +0000 | [diff] [blame] | 275 | # Rebaseline all tests/types we specified in the constructor, |
epoger@google.com | fd04011 | 2013-08-20 16:21:55 +0000 | [diff] [blame] | 276 | # within this builder's subdirectory in expectations/gm . |
epoger@google.com | 2a192a8 | 2013-08-02 20:54:46 +0000 | [diff] [blame] | 277 | # |
| 278 | # params: |
epoger@google.com | 2a192a8 | 2013-08-02 20:54:46 +0000 | [diff] [blame] | 279 | # builder : e.g. 'Test-Win7-ShuttleA-HD2000-x86-Release' |
epoger@google.com | fd04011 | 2013-08-20 16:21:55 +0000 | [diff] [blame] | 280 | def RebaselineSubdir(self, builder): |
epoger@google.com | 2a192a8 | 2013-08-02 20:54:46 +0000 | [diff] [blame] | 281 | # Read in the actual result summary, and extract all the tests whose |
| 282 | # results we need to update. |
borenet@google.com | b6e915d | 2013-09-16 18:13:13 +0000 | [diff] [blame] | 283 | results_builder = str(builder) |
| 284 | if self._from_trybot: |
| 285 | results_builder = results_builder + TRYBOT_SUFFIX |
| 286 | actuals_url = '/'.join([self._actuals_base_url, results_builder, |
| 287 | self._actuals_filename]) |
epoger@google.com | 06e626d | 2013-09-03 17:32:15 +0000 | [diff] [blame] | 288 | # Only update results for tests that are currently failing. |
| 289 | # We don't want to rewrite results for tests that are already succeeding, |
| 290 | # because we don't want to add annotation fields (such as |
| 291 | # JSONKEY_EXPECTEDRESULTS_BUGS) except for tests whose expectations we |
| 292 | # are actually modifying. |
epoger@google.com | 7f5f6e6 | 2013-09-05 17:19:37 +0000 | [diff] [blame] | 293 | sections = [gm_json.JSONKEY_ACTUALRESULTS_FAILED] |
epoger@google.com | 2a192a8 | 2013-08-02 20:54:46 +0000 | [diff] [blame] | 294 | if self._add_new: |
| 295 | sections.append(gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON) |
bsalomon@google.com | 46484ec | 2013-10-16 15:10:45 +0000 | [diff] [blame] | 296 | if self._add_ignored: |
| 297 | sections.append(gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED) |
epoger@google.com | 2a192a8 | 2013-08-02 20:54:46 +0000 | [diff] [blame] | 298 | results_to_update = self._GetActualResults(json_url=actuals_url, |
| 299 | sections=sections) |
epoger@google.com | e78d207 | 2013-06-12 17:44:14 +0000 | [diff] [blame] | 300 | |
epoger@google.com | 2a192a8 | 2013-08-02 20:54:46 +0000 | [diff] [blame] | 301 | # Read in current expectations. |
| 302 | expectations_input_filepath = os.path.join( |
epoger@google.com | fd04011 | 2013-08-20 16:21:55 +0000 | [diff] [blame] | 303 | self._expectations_root, builder, self._expectations_input_filename) |
epoger@google.com | 2a192a8 | 2013-08-02 20:54:46 +0000 | [diff] [blame] | 304 | expectations_dict = gm_json.LoadFromFile(expectations_input_filepath) |
epoger@google.com | 93acfd2 | 2013-09-03 18:27:19 +0000 | [diff] [blame] | 305 | expected_results = expectations_dict.get(gm_json.JSONKEY_EXPECTEDRESULTS) |
| 306 | if not expected_results: |
| 307 | expected_results = {} |
| 308 | expectations_dict[gm_json.JSONKEY_EXPECTEDRESULTS] = expected_results |
epoger@google.com | a783f2b | 2013-07-08 17:51:58 +0000 | [diff] [blame] | 309 | |
epoger@google.com | 2a192a8 | 2013-08-02 20:54:46 +0000 | [diff] [blame] | 310 | # Update the expectations in memory, skipping any tests/configs that |
| 311 | # the caller asked to exclude. |
| 312 | skipped_images = [] |
| 313 | if results_to_update: |
| 314 | for (image_name, image_results) in results_to_update.iteritems(): |
scroggo@google.com | d23e683 | 2013-10-10 21:09:24 +0000 | [diff] [blame] | 315 | if self._image_filename_re: |
| 316 | (test, config) = self._image_filename_re.match(image_name).groups() |
| 317 | if self._tests: |
| 318 | if test not in self._tests: |
| 319 | skipped_images.append(image_name) |
| 320 | continue |
| 321 | if self._configs: |
| 322 | if config not in self._configs: |
| 323 | skipped_images.append(image_name) |
| 324 | continue |
epoger@google.com | 2a192a8 | 2013-08-02 20:54:46 +0000 | [diff] [blame] | 325 | if not expected_results.get(image_name): |
| 326 | expected_results[image_name] = {} |
epoger@google.com | 06e626d | 2013-09-03 17:32:15 +0000 | [diff] [blame] | 327 | expected_results[image_name]\ |
| 328 | [gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS]\ |
| 329 | = [image_results] |
| 330 | if self._mark_unreviewed: |
| 331 | expected_results[image_name]\ |
| 332 | [gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED]\ |
| 333 | = False |
senorblanco@chromium.org | bfc0d6b | 2013-09-18 19:14:43 +0000 | [diff] [blame] | 334 | if self._mark_ignore_failure: |
| 335 | expected_results[image_name]\ |
| 336 | [gm_json.JSONKEY_EXPECTEDRESULTS_IGNOREFAILURE]\ |
| 337 | = True |
epoger@google.com | 06e626d | 2013-09-03 17:32:15 +0000 | [diff] [blame] | 338 | if self._bugs: |
| 339 | expected_results[image_name]\ |
| 340 | [gm_json.JSONKEY_EXPECTEDRESULTS_BUGS]\ |
| 341 | = self._bugs |
| 342 | if self._notes: |
| 343 | expected_results[image_name]\ |
| 344 | [gm_json.JSONKEY_EXPECTEDRESULTS_NOTES]\ |
| 345 | = self._notes |
epoger@google.com | a783f2b | 2013-07-08 17:51:58 +0000 | [diff] [blame] | 346 | |
epoger@google.com | 2a192a8 | 2013-08-02 20:54:46 +0000 | [diff] [blame] | 347 | # Write out updated expectations. |
| 348 | expectations_output_filepath = os.path.join( |
epoger@google.com | fd04011 | 2013-08-20 16:21:55 +0000 | [diff] [blame] | 349 | self._expectations_root, builder, self._expectations_output_filename) |
epoger@google.com | 2a192a8 | 2013-08-02 20:54:46 +0000 | [diff] [blame] | 350 | gm_json.WriteToFile(expectations_dict, expectations_output_filepath) |
epoger@google.com | a783f2b | 2013-07-08 17:51:58 +0000 | [diff] [blame] | 351 | |
epoger@google.com | 2a192a8 | 2013-08-02 20:54:46 +0000 | [diff] [blame] | 352 | # Mark the JSON file as plaintext, so text-style diffs can be applied. |
| 353 | # Fixes https://code.google.com/p/skia/issues/detail?id=1442 |
| 354 | if self._using_svn: |
| 355 | self._Call(['svn', 'propset', '--quiet', 'svn:mime-type', |
| 356 | 'text/x-json', expectations_output_filepath]) |
epoger@google.com | ec3397b | 2013-05-29 17:09:43 +0000 | [diff] [blame] | 357 | |
epoger@google.com | 9166bf5 | 2013-05-30 15:46:19 +0000 | [diff] [blame] | 358 | # main... |
epoger@google.com | ec3397b | 2013-05-29 17:09:43 +0000 | [diff] [blame] | 359 | |
epoger@google.com | 6d28e8d | 2013-09-18 14:58:47 +0000 | [diff] [blame] | 360 | parser = argparse.ArgumentParser( |
| 361 | formatter_class=argparse.RawDescriptionHelpFormatter, |
| 362 | epilog='Here is the full set of builders we know about:' + |
| 363 | '\n '.join([''] + sorted(TEST_BUILDERS))) |
epoger@google.com | a783f2b | 2013-07-08 17:51:58 +0000 | [diff] [blame] | 364 | parser.add_argument('--actuals-base-url', |
epoger@google.com | 06e626d | 2013-09-03 17:32:15 +0000 | [diff] [blame] | 365 | help=('base URL from which to read files containing JSON ' |
| 366 | 'summaries of actual GM results; defaults to ' |
scroggo@google.com | d23e683 | 2013-10-10 21:09:24 +0000 | [diff] [blame] | 367 | '%(default)s. To get a specific revision (useful for ' |
| 368 | 'trybots) replace "svn" with "svn-history/r123". ' |
| 369 | 'If SKIMAGE is True, defaults to ' + |
| 370 | gm_json.SKIMAGE_ACTUALS_BASE_URL), |
epoger@google.com | a783f2b | 2013-07-08 17:51:58 +0000 | [diff] [blame] | 371 | default='http://skia-autogen.googlecode.com/svn/gm-actual') |
| 372 | parser.add_argument('--actuals-filename', |
epoger@google.com | 06e626d | 2013-09-03 17:32:15 +0000 | [diff] [blame] | 373 | help=('filename (within builder-specific subdirectories ' |
| 374 | 'of ACTUALS_BASE_URL) to read a summary of results ' |
| 375 | 'from; defaults to %(default)s'), |
epoger@google.com | a783f2b | 2013-07-08 17:51:58 +0000 | [diff] [blame] | 376 | default='actual-results.json') |
epoger@google.com | dad5310 | 2013-06-12 14:25:30 +0000 | [diff] [blame] | 377 | parser.add_argument('--add-new', action='store_true', |
epoger@google.com | 06e626d | 2013-09-03 17:32:15 +0000 | [diff] [blame] | 378 | help=('in addition to the standard behavior of ' |
| 379 | 'updating expectations for failing tests, add ' |
| 380 | 'expectations for tests which don\'t have ' |
| 381 | 'expectations yet.')) |
bsalomon@google.com | 46484ec | 2013-10-16 15:10:45 +0000 | [diff] [blame] | 382 | parser.add_argument('--add-ignored', action='store_true', |
| 383 | help=('in addition to the standard behavior of ' |
| 384 | 'updating expectations for failing tests, add ' |
| 385 | 'expectations for tests for which failures are ' |
| 386 | 'currently ignored.')) |
epoger@google.com | 06e626d | 2013-09-03 17:32:15 +0000 | [diff] [blame] | 387 | parser.add_argument('--bugs', metavar='BUG', type=int, nargs='+', |
| 388 | help=('Skia bug numbers (under ' |
| 389 | 'https://code.google.com/p/skia/issues/list ) which ' |
| 390 | 'pertain to this set of rebaselines.')) |
epoger@google.com | fd04011 | 2013-08-20 16:21:55 +0000 | [diff] [blame] | 391 | parser.add_argument('--builders', metavar='BUILDER', nargs='+', |
epoger@google.com | 06e626d | 2013-09-03 17:32:15 +0000 | [diff] [blame] | 392 | help=('which platforms to rebaseline; ' |
epoger@google.com | 6d28e8d | 2013-09-18 14:58:47 +0000 | [diff] [blame] | 393 | 'if unspecified, rebaseline all known platforms ' |
| 394 | '(see below for a list)')) |
epoger@google.com | a783f2b | 2013-07-08 17:51:58 +0000 | [diff] [blame] | 395 | # TODO(epoger): Add test that exercises --configs argument. |
epoger@google.com | 9166bf5 | 2013-05-30 15:46:19 +0000 | [diff] [blame] | 396 | parser.add_argument('--configs', metavar='CONFIG', nargs='+', |
epoger@google.com | 06e626d | 2013-09-03 17:32:15 +0000 | [diff] [blame] | 397 | help=('which configurations to rebaseline, e.g. ' |
| 398 | '"--configs 565 8888", as a filter over the full set ' |
| 399 | 'of results in ACTUALS_FILENAME; if unspecified, ' |
scroggo@google.com | 5187c43 | 2013-10-22 00:42:46 +0000 | [diff] [blame] | 400 | 'rebaseline *all* configs that are available.')) |
epoger@google.com | a783f2b | 2013-07-08 17:51:58 +0000 | [diff] [blame] | 401 | parser.add_argument('--expectations-filename', |
epoger@google.com | 06e626d | 2013-09-03 17:32:15 +0000 | [diff] [blame] | 402 | help=('filename (under EXPECTATIONS_ROOT) to read ' |
| 403 | 'current expectations from, and to write new ' |
| 404 | 'expectations into (unless a separate ' |
| 405 | 'EXPECTATIONS_FILENAME_OUTPUT has been specified); ' |
| 406 | 'defaults to %(default)s'), |
epoger@google.com | a783f2b | 2013-07-08 17:51:58 +0000 | [diff] [blame] | 407 | default='expected-results.json') |
epoger@google.com | c60e745 | 2013-07-24 19:36:51 +0000 | [diff] [blame] | 408 | parser.add_argument('--expectations-filename-output', |
epoger@google.com | 06e626d | 2013-09-03 17:32:15 +0000 | [diff] [blame] | 409 | help=('filename (under EXPECTATIONS_ROOT) to write ' |
| 410 | 'updated expectations into; by default, overwrites ' |
| 411 | 'the input file (EXPECTATIONS_FILENAME)'), |
epoger@google.com | c60e745 | 2013-07-24 19:36:51 +0000 | [diff] [blame] | 412 | default='') |
epoger@google.com | 99a8ec9 | 2013-06-19 18:56:59 +0000 | [diff] [blame] | 413 | parser.add_argument('--expectations-root', |
epoger@google.com | 06e626d | 2013-09-03 17:32:15 +0000 | [diff] [blame] | 414 | help=('root of expectations directory to update-- should ' |
| 415 | 'contain one or more builder subdirectories. ' |
scroggo@google.com | d23e683 | 2013-10-10 21:09:24 +0000 | [diff] [blame] | 416 | 'Defaults to %(default)s. If SKIMAGE is set, ' |
| 417 | ' defaults to ' + gm_json.SKIMAGE_EXPECTATIONS_ROOT), |
epoger@google.com | e94a7d2 | 2013-07-23 19:37:03 +0000 | [diff] [blame] | 418 | default=os.path.join('expectations', 'gm')) |
epoger@google.com | ffcbdbf | 2013-07-16 17:35:39 +0000 | [diff] [blame] | 419 | parser.add_argument('--keep-going-on-failure', action='store_true', |
epoger@google.com | 06e626d | 2013-09-03 17:32:15 +0000 | [diff] [blame] | 420 | help=('instead of halting at the first error encountered, ' |
| 421 | 'keep going and rebaseline as many tests as ' |
| 422 | 'possible, and then report the full set of errors ' |
| 423 | 'at the end')) |
| 424 | parser.add_argument('--notes', |
| 425 | help=('free-form text notes to add to all updated ' |
| 426 | 'expectations')) |
epoger@google.com | a783f2b | 2013-07-08 17:51:58 +0000 | [diff] [blame] | 427 | # TODO(epoger): Add test that exercises --tests argument. |
epoger@google.com | 99ba65a | 2013-06-05 15:43:37 +0000 | [diff] [blame] | 428 | parser.add_argument('--tests', metavar='TEST', nargs='+', |
epoger@google.com | 06e626d | 2013-09-03 17:32:15 +0000 | [diff] [blame] | 429 | help=('which tests to rebaseline, e.g. ' |
| 430 | '"--tests aaclip bigmatrix", as a filter over the ' |
| 431 | 'full set of results in ACTUALS_FILENAME; if ' |
| 432 | 'unspecified, rebaseline *all* tests that are ' |
scroggo@google.com | 5187c43 | 2013-10-22 00:42:46 +0000 | [diff] [blame] | 433 | 'available.')) |
epoger@google.com | 06e626d | 2013-09-03 17:32:15 +0000 | [diff] [blame] | 434 | parser.add_argument('--unreviewed', action='store_true', |
| 435 | help=('mark all expectations modified by this run as ' |
| 436 | '"%s": False' % |
| 437 | gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED)) |
senorblanco@chromium.org | bfc0d6b | 2013-09-18 19:14:43 +0000 | [diff] [blame] | 438 | parser.add_argument('--ignore-failure', action='store_true', |
| 439 | help=('mark all expectations modified by this run as ' |
| 440 | '"%s": True' % |
| 441 | gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED)) |
borenet@google.com | b6e915d | 2013-09-16 18:13:13 +0000 | [diff] [blame] | 442 | parser.add_argument('--from-trybot', action='store_true', |
| 443 | help=('pull the actual-results.json file from the ' |
| 444 | 'corresponding trybot, rather than the main builder')) |
scroggo@google.com | d23e683 | 2013-10-10 21:09:24 +0000 | [diff] [blame] | 445 | parser.add_argument('--skimage', action='store_true', |
| 446 | help=('Rebaseline skimage results instead of gm. Defaults ' |
| 447 | 'to False. If True, TESTS and CONFIGS are ignored, ' |
| 448 | 'and ACTUALS_BASE_URL and EXPECTATIONS_ROOT are set ' |
| 449 | 'to alternate defaults, specific to skimage.')) |
epoger@google.com | 9166bf5 | 2013-05-30 15:46:19 +0000 | [diff] [blame] | 450 | args = parser.parse_args() |
epoger@google.com | ffcbdbf | 2013-07-16 17:35:39 +0000 | [diff] [blame] | 451 | exception_handler = ExceptionHandler( |
| 452 | keep_going_on_failure=args.keep_going_on_failure) |
epoger@google.com | fd04011 | 2013-08-20 16:21:55 +0000 | [diff] [blame] | 453 | if args.builders: |
| 454 | builders = args.builders |
epoger@google.com | 2a192a8 | 2013-08-02 20:54:46 +0000 | [diff] [blame] | 455 | missing_json_is_fatal = True |
epoger@google.com | 99a8ec9 | 2013-06-19 18:56:59 +0000 | [diff] [blame] | 456 | else: |
epoger@google.com | fd04011 | 2013-08-20 16:21:55 +0000 | [diff] [blame] | 457 | builders = sorted(TEST_BUILDERS) |
epoger@google.com | 2a192a8 | 2013-08-02 20:54:46 +0000 | [diff] [blame] | 458 | missing_json_is_fatal = False |
scroggo@google.com | d23e683 | 2013-10-10 21:09:24 +0000 | [diff] [blame] | 459 | if args.skimage: |
| 460 | # Use a different default if --skimage is specified. |
| 461 | if args.actuals_base_url == parser.get_default('actuals_base_url'): |
| 462 | args.actuals_base_url = gm_json.SKIMAGE_ACTUALS_BASE_URL |
| 463 | if args.expectations_root == parser.get_default('expectations_root'): |
| 464 | args.expectations_root = gm_json.SKIMAGE_EXPECTATIONS_ROOT |
epoger@google.com | fd04011 | 2013-08-20 16:21:55 +0000 | [diff] [blame] | 465 | for builder in builders: |
| 466 | if not builder in TEST_BUILDERS: |
| 467 | raise Exception(('unrecognized builder "%s"; ' + |
epoger@google.com | 2a192a8 | 2013-08-02 20:54:46 +0000 | [diff] [blame] | 468 | 'should be one of %s') % ( |
epoger@google.com | fd04011 | 2013-08-20 16:21:55 +0000 | [diff] [blame] | 469 | builder, TEST_BUILDERS)) |
epoger@google.com | 99a8ec9 | 2013-06-19 18:56:59 +0000 | [diff] [blame] | 470 | |
epoger@google.com | fd04011 | 2013-08-20 16:21:55 +0000 | [diff] [blame] | 471 | expectations_json_file = os.path.join(args.expectations_root, builder, |
epoger@google.com | 2a192a8 | 2013-08-02 20:54:46 +0000 | [diff] [blame] | 472 | args.expectations_filename) |
| 473 | if os.path.isfile(expectations_json_file): |
| 474 | rebaseliner = JsonRebaseliner( |
| 475 | expectations_root=args.expectations_root, |
| 476 | expectations_input_filename=args.expectations_filename, |
| 477 | expectations_output_filename=(args.expectations_filename_output or |
| 478 | args.expectations_filename), |
| 479 | tests=args.tests, configs=args.configs, |
| 480 | actuals_base_url=args.actuals_base_url, |
| 481 | actuals_filename=args.actuals_filename, |
| 482 | exception_handler=exception_handler, |
bsalomon@google.com | 46484ec | 2013-10-16 15:10:45 +0000 | [diff] [blame] | 483 | add_new=args.add_new, add_ignored=args.add_ignored, |
| 484 | bugs=args.bugs, notes=args.notes, |
borenet@google.com | b6e915d | 2013-09-16 18:13:13 +0000 | [diff] [blame] | 485 | mark_unreviewed=args.unreviewed, |
senorblanco@chromium.org | bfc0d6b | 2013-09-18 19:14:43 +0000 | [diff] [blame] | 486 | mark_ignore_failure=args.ignore_failure, |
borenet@google.com | b6e915d | 2013-09-16 18:13:13 +0000 | [diff] [blame] | 487 | from_trybot=args.from_trybot) |
epoger@google.com | 3e7399f | 2013-07-10 17:23:47 +0000 | [diff] [blame] | 488 | try: |
epoger@google.com | fd04011 | 2013-08-20 16:21:55 +0000 | [diff] [blame] | 489 | rebaseliner.RebaselineSubdir(builder=builder) |
epoger@google.com | 93acfd2 | 2013-09-03 18:27:19 +0000 | [diff] [blame] | 490 | except: |
| 491 | exception_handler.RaiseExceptionOrContinue() |
epoger@google.com | 2a192a8 | 2013-08-02 20:54:46 +0000 | [diff] [blame] | 492 | else: |
epoger@google.com | 93acfd2 | 2013-09-03 18:27:19 +0000 | [diff] [blame] | 493 | try: |
| 494 | raise _InternalException('expectations_json_file %s not found' % |
| 495 | expectations_json_file) |
| 496 | except: |
| 497 | exception_handler.RaiseExceptionOrContinue() |
epoger@google.com | ffcbdbf | 2013-07-16 17:35:39 +0000 | [diff] [blame] | 498 | |
| 499 | exception_handler.ReportAllFailures() |