epoger@google.com | f9d134d | 2013-09-27 15:02:44 +0000 | [diff] [blame] | 1 | #!/usr/bin/python |
| 2 | |
epoger@google.com | 9fb6c8a | 2013-10-09 18:05:58 +0000 | [diff] [blame] | 3 | """ |
epoger@google.com | f9d134d | 2013-09-27 15:02:44 +0000 | [diff] [blame] | 4 | Copyright 2013 Google Inc. |
| 5 | |
| 6 | Use of this source code is governed by a BSD-style license that can be |
| 7 | found in the LICENSE file. |
epoger@google.com | f9d134d | 2013-09-27 15:02:44 +0000 | [diff] [blame] | 8 | |
epoger@google.com | f9d134d | 2013-09-27 15:02:44 +0000 | [diff] [blame] | 9 | Repackage expected/actual GM results as needed by our HTML rebaseline viewer. |
epoger@google.com | 9fb6c8a | 2013-10-09 18:05:58 +0000 | [diff] [blame] | 10 | """ |
epoger@google.com | f9d134d | 2013-09-27 15:02:44 +0000 | [diff] [blame] | 11 | |
| 12 | # System-level imports |
commit-bot@chromium.org | 7b06c8e | 2013-12-23 22:47:15 +0000 | [diff] [blame] | 13 | import argparse |
epoger@google.com | f9d134d | 2013-09-27 15:02:44 +0000 | [diff] [blame] | 14 | import fnmatch |
| 15 | import json |
epoger@google.com | dcb4e65 | 2013-10-11 18:45:33 +0000 | [diff] [blame] | 16 | import logging |
epoger@google.com | f9d134d | 2013-09-27 15:02:44 +0000 | [diff] [blame] | 17 | import os |
| 18 | import re |
| 19 | import sys |
epoger@google.com | 542b65f | 2013-10-15 20:10:33 +0000 | [diff] [blame] | 20 | import time |
epoger@google.com | f9d134d | 2013-09-27 15:02:44 +0000 | [diff] [blame] | 21 | |
| 22 | # Imports from within Skia |
| 23 | # |
| 24 | # We need to add the 'gm' directory, so that we can import gm_json.py within |
| 25 | # that directory. That script allows us to parse the actual-results.json file |
| 26 | # written out by the GM tool. |
| 27 | # Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end* |
| 28 | # so any dirs that are already in the PYTHONPATH will be preferred. |
commit-bot@chromium.org | 7b06c8e | 2013-12-23 22:47:15 +0000 | [diff] [blame] | 29 | PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) |
| 30 | GM_DIRECTORY = os.path.dirname(PARENT_DIRECTORY) |
commit-bot@chromium.org | 7498d95 | 2014-03-13 14:56:29 +0000 | [diff] [blame] | 31 | TRUNK_DIRECTORY = os.path.dirname(GM_DIRECTORY) |
epoger@google.com | f9d134d | 2013-09-27 15:02:44 +0000 | [diff] [blame] | 32 | if GM_DIRECTORY not in sys.path: |
| 33 | sys.path.append(GM_DIRECTORY) |
| 34 | import gm_json |
epoger@google.com | 9dddf6f | 2013-11-08 16:25:25 +0000 | [diff] [blame] | 35 | import imagediffdb |
commit-bot@chromium.org | 16f4180 | 2014-02-26 19:05:20 +0000 | [diff] [blame] | 36 | import imagepair |
| 37 | import imagepairset |
| 38 | |
| 39 | # Keys used to link an image to a particular GM test. |
| 40 | # NOTE: Keep these in sync with static/constants.js |
commit-bot@chromium.org | ea770f1 | 2014-03-13 16:33:36 +0000 | [diff] [blame^] | 41 | REBASELINE_SERVER_SCHEMA_VERSION_NUMBER = 1 |
commit-bot@chromium.org | 16f4180 | 2014-02-26 19:05:20 +0000 | [diff] [blame] | 42 | KEY__EXPECTATIONS__BUGS = gm_json.JSONKEY_EXPECTEDRESULTS_BUGS |
| 43 | KEY__EXPECTATIONS__IGNOREFAILURE = gm_json.JSONKEY_EXPECTEDRESULTS_IGNOREFAILURE |
| 44 | KEY__EXPECTATIONS__REVIEWED = gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED |
| 45 | KEY__EXTRACOLUMN__BUILDER = 'builder' |
| 46 | KEY__EXTRACOLUMN__CONFIG = 'config' |
| 47 | KEY__EXTRACOLUMN__RESULT_TYPE = 'resultType' |
| 48 | KEY__EXTRACOLUMN__TEST = 'test' |
commit-bot@chromium.org | 7498d95 | 2014-03-13 14:56:29 +0000 | [diff] [blame] | 49 | KEY__HEADER = 'header' |
| 50 | KEY__HEADER__DATAHASH = 'dataHash' |
| 51 | KEY__HEADER__IS_EDITABLE = 'isEditable' |
| 52 | KEY__HEADER__IS_EXPORTED = 'isExported' |
| 53 | KEY__HEADER__IS_STILL_LOADING = 'resultsStillLoading' |
commit-bot@chromium.org | 16f4180 | 2014-02-26 19:05:20 +0000 | [diff] [blame] | 54 | KEY__HEADER__RESULTS_ALL = 'all' |
| 55 | KEY__HEADER__RESULTS_FAILURES = 'failures' |
commit-bot@chromium.org | ea770f1 | 2014-03-13 16:33:36 +0000 | [diff] [blame^] | 56 | KEY__HEADER__SCHEMA_VERSION = 'schemaVersion' |
commit-bot@chromium.org | 7498d95 | 2014-03-13 14:56:29 +0000 | [diff] [blame] | 57 | KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE = 'timeNextUpdateAvailable' |
| 58 | KEY__HEADER__TIME_UPDATED = 'timeUpdated' |
| 59 | KEY__HEADER__TYPE = 'type' |
commit-bot@chromium.org | 16f4180 | 2014-02-26 19:05:20 +0000 | [diff] [blame] | 60 | KEY__NEW_IMAGE_URL = 'newImageUrl' |
| 61 | KEY__RESULT_TYPE__FAILED = gm_json.JSONKEY_ACTUALRESULTS_FAILED |
| 62 | KEY__RESULT_TYPE__FAILUREIGNORED = gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED |
| 63 | KEY__RESULT_TYPE__NOCOMPARISON = gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON |
| 64 | KEY__RESULT_TYPE__SUCCEEDED = gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED |
| 65 | |
| 66 | EXPECTATION_FIELDS_PASSED_THRU_VERBATIM = [ |
| 67 | KEY__EXPECTATIONS__BUGS, |
| 68 | KEY__EXPECTATIONS__IGNOREFAILURE, |
| 69 | KEY__EXPECTATIONS__REVIEWED, |
| 70 | ] |
epoger@google.com | f9d134d | 2013-09-27 15:02:44 +0000 | [diff] [blame] | 71 | |
| 72 | IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN) |
epoger@google.com | eb83259 | 2013-10-23 15:07:26 +0000 | [diff] [blame] | 73 | IMAGE_FILENAME_FORMATTER = '%s_%s.png' # pass in (testname, config) |
| 74 | |
commit-bot@chromium.org | 16f4180 | 2014-02-26 19:05:20 +0000 | [diff] [blame] | 75 | IMAGEPAIR_SET_DESCRIPTIONS = ('expected image', 'actual image') |
epoger@google.com | 055e3b5 | 2013-10-26 14:31:11 +0000 | [diff] [blame] | 76 | |
commit-bot@chromium.org | 7498d95 | 2014-03-13 14:56:29 +0000 | [diff] [blame] | 77 | DEFAULT_ACTUALS_DIR = '.gm-actuals' |
| 78 | DEFAULT_EXPECTATIONS_DIR = os.path.join(TRUNK_DIRECTORY, 'expectations', 'gm') |
| 79 | DEFAULT_GENERATED_IMAGES_ROOT = os.path.join(PARENT_DIRECTORY, 'static', |
| 80 | 'generated-images') |
| 81 | |
epoger@google.com | f9d134d | 2013-09-27 15:02:44 +0000 | [diff] [blame] | 82 | |
| 83 | class Results(object): |
commit-bot@chromium.org | 16f4180 | 2014-02-26 19:05:20 +0000 | [diff] [blame] | 84 | """ Loads actual and expected GM results into an ImagePairSet. |
| 85 | |
| 86 | Loads actual and expected results from all builders, except for those skipped |
| 87 | by _ignore_builder(). |
epoger@google.com | dcb4e65 | 2013-10-11 18:45:33 +0000 | [diff] [blame] | 88 | |
epoger@google.com | eb83259 | 2013-10-23 15:07:26 +0000 | [diff] [blame] | 89 | Once this object has been constructed, the results (in self._results[]) |
| 90 | are immutable. If you want to update the results based on updated JSON |
| 91 | file contents, you will need to create a new Results object.""" |
epoger@google.com | f9d134d | 2013-09-27 15:02:44 +0000 | [diff] [blame] | 92 | |
commit-bot@chromium.org | 7498d95 | 2014-03-13 14:56:29 +0000 | [diff] [blame] | 93 | def __init__(self, actuals_root=DEFAULT_ACTUALS_DIR, |
| 94 | expected_root=DEFAULT_EXPECTATIONS_DIR, |
| 95 | generated_images_root=DEFAULT_GENERATED_IMAGES_ROOT): |
epoger@google.com | f9d134d | 2013-09-27 15:02:44 +0000 | [diff] [blame] | 96 | """ |
epoger@google.com | 9fb6c8a | 2013-10-09 18:05:58 +0000 | [diff] [blame] | 97 | Args: |
epoger@google.com | f9d134d | 2013-09-27 15:02:44 +0000 | [diff] [blame] | 98 | actuals_root: root directory containing all actual-results.json files |
| 99 | expected_root: root directory containing all expected-results.json files |
epoger@google.com | 214a024 | 2013-11-22 19:26:18 +0000 | [diff] [blame] | 100 | generated_images_root: directory within which to create all pixel diffs; |
epoger@google.com | 9dddf6f | 2013-11-08 16:25:25 +0000 | [diff] [blame] | 101 | if this directory does not yet exist, it will be created |
epoger@google.com | f9d134d | 2013-09-27 15:02:44 +0000 | [diff] [blame] | 102 | """ |
commit-bot@chromium.org | a6ecbb8 | 2013-12-19 19:08:31 +0000 | [diff] [blame] | 103 | time_start = int(time.time()) |
epoger@google.com | 9dddf6f | 2013-11-08 16:25:25 +0000 | [diff] [blame] | 104 | self._image_diff_db = imagediffdb.ImageDiffDB(generated_images_root) |
epoger@google.com | eb83259 | 2013-10-23 15:07:26 +0000 | [diff] [blame] | 105 | self._actuals_root = actuals_root |
| 106 | self._expected_root = expected_root |
| 107 | self._load_actual_and_expected() |
epoger@google.com | 542b65f | 2013-10-15 20:10:33 +0000 | [diff] [blame] | 108 | self._timestamp = int(time.time()) |
commit-bot@chromium.org | a6ecbb8 | 2013-12-19 19:08:31 +0000 | [diff] [blame] | 109 | logging.info('Results complete; took %d seconds.' % |
| 110 | (self._timestamp - time_start)) |
epoger@google.com | 542b65f | 2013-10-15 20:10:33 +0000 | [diff] [blame] | 111 | |
| 112 | def get_timestamp(self): |
| 113 | """Return the time at which this object was created, in seconds past epoch |
| 114 | (UTC). |
| 115 | """ |
| 116 | return self._timestamp |
epoger@google.com | f9d134d | 2013-09-27 15:02:44 +0000 | [diff] [blame] | 117 | |
epoger@google.com | eb83259 | 2013-10-23 15:07:26 +0000 | [diff] [blame] | 118 | def edit_expectations(self, modifications): |
| 119 | """Edit the expectations stored within this object and write them back |
| 120 | to disk. |
| 121 | |
| 122 | Note that this will NOT update the results stored in self._results[] ; |
| 123 | in order to see those updates, you must instantiate a new Results object |
| 124 | based on the (now updated) files on disk. |
| 125 | |
| 126 | Args: |
| 127 | modifications: a list of dictionaries, one for each expectation to update: |
| 128 | |
| 129 | [ |
| 130 | { |
commit-bot@chromium.org | 16f4180 | 2014-02-26 19:05:20 +0000 | [diff] [blame] | 131 | imagepair.KEY__EXPECTATIONS_DATA: { |
| 132 | KEY__EXPECTATIONS__BUGS: [123, 456], |
| 133 | KEY__EXPECTATIONS__IGNOREFAILURE: false, |
| 134 | KEY__EXPECTATIONS__REVIEWED: true, |
| 135 | }, |
| 136 | imagepair.KEY__EXTRA_COLUMN_VALUES: { |
| 137 | KEY__EXTRACOLUMN__BUILDER: 'Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug', |
| 138 | KEY__EXTRACOLUMN__CONFIG: '8888', |
| 139 | KEY__EXTRACOLUMN__TEST: 'bigmatrix', |
| 140 | }, |
| 141 | KEY__NEW_IMAGE_URL: 'bitmap-64bitMD5/bigmatrix/10894408024079689926.png', |
epoger@google.com | eb83259 | 2013-10-23 15:07:26 +0000 | [diff] [blame] | 142 | }, |
| 143 | ... |
| 144 | ] |
| 145 | |
epoger@google.com | eb83259 | 2013-10-23 15:07:26 +0000 | [diff] [blame] | 146 | """ |
| 147 | expected_builder_dicts = Results._read_dicts_from_root(self._expected_root) |
| 148 | for mod in modifications: |
commit-bot@chromium.org | 16f4180 | 2014-02-26 19:05:20 +0000 | [diff] [blame] | 149 | image_name = IMAGE_FILENAME_FORMATTER % ( |
| 150 | mod[imagepair.KEY__EXTRA_COLUMN_VALUES][KEY__EXTRACOLUMN__TEST], |
| 151 | mod[imagepair.KEY__EXTRA_COLUMN_VALUES][KEY__EXTRACOLUMN__CONFIG]) |
| 152 | _, hash_type, hash_digest = gm_json.SplitGmRelativeUrl( |
| 153 | mod[KEY__NEW_IMAGE_URL]) |
| 154 | allowed_digests = [[hash_type, int(hash_digest)]] |
epoger@google.com | eb83259 | 2013-10-23 15:07:26 +0000 | [diff] [blame] | 155 | new_expectations = { |
| 156 | gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS: allowed_digests, |
epoger@google.com | eb83259 | 2013-10-23 15:07:26 +0000 | [diff] [blame] | 157 | } |
commit-bot@chromium.org | 16f4180 | 2014-02-26 19:05:20 +0000 | [diff] [blame] | 158 | for field in EXPECTATION_FIELDS_PASSED_THRU_VERBATIM: |
| 159 | value = mod[imagepair.KEY__EXPECTATIONS_DATA].get(field) |
epoger@google.com | 055e3b5 | 2013-10-26 14:31:11 +0000 | [diff] [blame] | 160 | if value is not None: |
| 161 | new_expectations[field] = value |
commit-bot@chromium.org | 16f4180 | 2014-02-26 19:05:20 +0000 | [diff] [blame] | 162 | builder_dict = expected_builder_dicts[ |
| 163 | mod[imagepair.KEY__EXTRA_COLUMN_VALUES][KEY__EXTRACOLUMN__BUILDER]] |
epoger@google.com | eb83259 | 2013-10-23 15:07:26 +0000 | [diff] [blame] | 164 | builder_expectations = builder_dict.get(gm_json.JSONKEY_EXPECTEDRESULTS) |
| 165 | if not builder_expectations: |
| 166 | builder_expectations = {} |
| 167 | builder_dict[gm_json.JSONKEY_EXPECTEDRESULTS] = builder_expectations |
| 168 | builder_expectations[image_name] = new_expectations |
| 169 | Results._write_dicts_to_root(expected_builder_dicts, self._expected_root) |
| 170 | |
commit-bot@chromium.org | 7498d95 | 2014-03-13 14:56:29 +0000 | [diff] [blame] | 171 | def get_results_of_type(self, results_type): |
| 172 | """Return results of some/all tests (depending on 'results_type' parameter). |
epoger@google.com | dcb4e65 | 2013-10-11 18:45:33 +0000 | [diff] [blame] | 173 | |
| 174 | Args: |
commit-bot@chromium.org | 7498d95 | 2014-03-13 14:56:29 +0000 | [diff] [blame] | 175 | results_type: string describing which types of results to include; must |
| 176 | be one of the RESULTS_* constants |
epoger@google.com | dcb4e65 | 2013-10-11 18:45:33 +0000 | [diff] [blame] | 177 | |
commit-bot@chromium.org | 16f4180 | 2014-02-26 19:05:20 +0000 | [diff] [blame] | 178 | Results are returned in a dictionary as output by ImagePairSet.as_dict(). |
epoger@google.com | f9d134d | 2013-09-27 15:02:44 +0000 | [diff] [blame] | 179 | """ |
commit-bot@chromium.org | 7498d95 | 2014-03-13 14:56:29 +0000 | [diff] [blame] | 180 | return self._results[results_type] |
| 181 | |
| 182 | def get_packaged_results_of_type(self, results_type, reload_seconds=None, |
| 183 | is_editable=False, is_exported=True): |
| 184 | """ Package the results of some/all tests as a complete response_dict. |
| 185 | |
| 186 | Args: |
| 187 | results_type: string indicating which set of results to return; |
| 188 | must be one of the RESULTS_* constants |
| 189 | reload_seconds: if specified, note that new results may be available once |
| 190 | these results are reload_seconds old |
| 191 | is_editable: whether clients are allowed to submit new baselines |
| 192 | is_exported: whether these results are being made available to other |
| 193 | network hosts |
| 194 | """ |
| 195 | response_dict = self._results[results_type] |
| 196 | time_updated = self.get_timestamp() |
| 197 | response_dict[KEY__HEADER] = { |
commit-bot@chromium.org | ea770f1 | 2014-03-13 16:33:36 +0000 | [diff] [blame^] | 198 | KEY__HEADER__SCHEMA_VERSION: REBASELINE_SERVER_SCHEMA_VERSION_NUMBER, |
| 199 | |
commit-bot@chromium.org | 7498d95 | 2014-03-13 14:56:29 +0000 | [diff] [blame] | 200 | # Timestamps: |
| 201 | # 1. when this data was last updated |
| 202 | # 2. when the caller should check back for new data (if ever) |
| 203 | KEY__HEADER__TIME_UPDATED: time_updated, |
| 204 | KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE: ( |
| 205 | (time_updated+reload_seconds) if reload_seconds else None), |
| 206 | |
| 207 | # The type we passed to get_results_of_type() |
| 208 | KEY__HEADER__TYPE: results_type, |
| 209 | |
| 210 | # Hash of dataset, which the client must return with any edits-- |
| 211 | # this ensures that the edits were made to a particular dataset. |
| 212 | KEY__HEADER__DATAHASH: str(hash(repr( |
| 213 | response_dict[imagepairset.KEY__IMAGEPAIRS]))), |
| 214 | |
| 215 | # Whether the server will accept edits back. |
| 216 | KEY__HEADER__IS_EDITABLE: is_editable, |
| 217 | |
| 218 | # Whether the service is accessible from other hosts. |
| 219 | KEY__HEADER__IS_EXPORTED: is_exported, |
| 220 | } |
| 221 | return response_dict |
epoger@google.com | f9d134d | 2013-09-27 15:02:44 +0000 | [diff] [blame] | 222 | |
| 223 | @staticmethod |
commit-bot@chromium.org | 04747a5 | 2014-01-15 19:16:09 +0000 | [diff] [blame] | 224 | def _ignore_builder(builder): |
| 225 | """Returns True if we should ignore expectations and actuals for a builder. |
| 226 | |
| 227 | This allows us to ignore builders for which we don't maintain expectations |
| 228 | (trybots, Valgrind, ASAN, TSAN), and avoid problems like |
| 229 | https://code.google.com/p/skia/issues/detail?id=2036 ('rebaseline_server |
| 230 | produces error when trying to add baselines for ASAN/TSAN builders') |
| 231 | |
| 232 | Args: |
| 233 | builder: name of this builder, as a string |
| 234 | |
| 235 | Returns: |
| 236 | True if we should ignore expectations and actuals for this builder. |
| 237 | """ |
| 238 | return (builder.endswith('-Trybot') or |
| 239 | ('Valgrind' in builder) or |
| 240 | ('TSAN' in builder) or |
| 241 | ('ASAN' in builder)) |
| 242 | |
| 243 | @staticmethod |
epoger@google.com | eb83259 | 2013-10-23 15:07:26 +0000 | [diff] [blame] | 244 | def _read_dicts_from_root(root, pattern='*.json'): |
epoger@google.com | 9fb6c8a | 2013-10-09 18:05:58 +0000 | [diff] [blame] | 245 | """Read all JSON dictionaries within a directory tree. |
epoger@google.com | f9d134d | 2013-09-27 15:02:44 +0000 | [diff] [blame] | 246 | |
epoger@google.com | 9fb6c8a | 2013-10-09 18:05:58 +0000 | [diff] [blame] | 247 | Args: |
epoger@google.com | f9d134d | 2013-09-27 15:02:44 +0000 | [diff] [blame] | 248 | root: path to root of directory tree |
| 249 | pattern: which files to read within root (fnmatch-style pattern) |
epoger@google.com | 9fb6c8a | 2013-10-09 18:05:58 +0000 | [diff] [blame] | 250 | |
| 251 | Returns: |
| 252 | A meta-dictionary containing all the JSON dictionaries found within |
| 253 | the directory tree, keyed by the builder name of each dictionary. |
epoger@google.com | 542b65f | 2013-10-15 20:10:33 +0000 | [diff] [blame] | 254 | |
| 255 | Raises: |
| 256 | IOError if root does not refer to an existing directory |
epoger@google.com | f9d134d | 2013-09-27 15:02:44 +0000 | [diff] [blame] | 257 | """ |
epoger@google.com | 542b65f | 2013-10-15 20:10:33 +0000 | [diff] [blame] | 258 | if not os.path.isdir(root): |
| 259 | raise IOError('no directory found at path %s' % root) |
epoger@google.com | f9d134d | 2013-09-27 15:02:44 +0000 | [diff] [blame] | 260 | meta_dict = {} |
| 261 | for dirpath, dirnames, filenames in os.walk(root): |
| 262 | for matching_filename in fnmatch.filter(filenames, pattern): |
| 263 | builder = os.path.basename(dirpath) |
commit-bot@chromium.org | 04747a5 | 2014-01-15 19:16:09 +0000 | [diff] [blame] | 264 | if Results._ignore_builder(builder): |
epoger@google.com | f9d134d | 2013-09-27 15:02:44 +0000 | [diff] [blame] | 265 | continue |
| 266 | fullpath = os.path.join(dirpath, matching_filename) |
| 267 | meta_dict[builder] = gm_json.LoadFromFile(fullpath) |
| 268 | return meta_dict |
| 269 | |
epoger@google.com | eb83259 | 2013-10-23 15:07:26 +0000 | [diff] [blame] | 270 | @staticmethod |
commit-bot@chromium.org | 16f4180 | 2014-02-26 19:05:20 +0000 | [diff] [blame] | 271 | def _create_relative_url(hashtype_and_digest, test_name): |
| 272 | """Returns the URL for this image, relative to GM_ACTUALS_ROOT_HTTP_URL. |
| 273 | |
| 274 | If we don't have a record of this image, returns None. |
| 275 | |
| 276 | Args: |
| 277 | hashtype_and_digest: (hash_type, hash_digest) tuple, or None if we |
| 278 | don't have a record of this image |
| 279 | test_name: string; name of the GM test that created this image |
| 280 | """ |
| 281 | if not hashtype_and_digest: |
| 282 | return None |
| 283 | return gm_json.CreateGmRelativeUrl( |
| 284 | test_name=test_name, |
| 285 | hash_type=hashtype_and_digest[0], |
| 286 | hash_digest=hashtype_and_digest[1]) |
| 287 | |
| 288 | @staticmethod |
epoger@google.com | eb83259 | 2013-10-23 15:07:26 +0000 | [diff] [blame] | 289 | def _write_dicts_to_root(meta_dict, root, pattern='*.json'): |
| 290 | """Write all per-builder dictionaries within meta_dict to files under |
| 291 | the root path. |
| 292 | |
| 293 | Security note: this will only write to files that already exist within |
| 294 | the root path (as found by os.walk() within root), so we don't need to |
| 295 | worry about malformed content writing to disk outside of root. |
| 296 | However, the data written to those files is not double-checked, so it |
| 297 | could contain poisonous data. |
| 298 | |
| 299 | Args: |
| 300 | meta_dict: a builder-keyed meta-dictionary containing all the JSON |
| 301 | dictionaries we want to write out |
| 302 | root: path to root of directory tree within which to write files |
| 303 | pattern: which files to write within root (fnmatch-style pattern) |
| 304 | |
| 305 | Raises: |
| 306 | IOError if root does not refer to an existing directory |
| 307 | KeyError if the set of per-builder dictionaries written out was |
| 308 | different than expected |
| 309 | """ |
| 310 | if not os.path.isdir(root): |
| 311 | raise IOError('no directory found at path %s' % root) |
| 312 | actual_builders_written = [] |
| 313 | for dirpath, dirnames, filenames in os.walk(root): |
| 314 | for matching_filename in fnmatch.filter(filenames, pattern): |
| 315 | builder = os.path.basename(dirpath) |
commit-bot@chromium.org | 04747a5 | 2014-01-15 19:16:09 +0000 | [diff] [blame] | 316 | if Results._ignore_builder(builder): |
epoger@google.com | eb83259 | 2013-10-23 15:07:26 +0000 | [diff] [blame] | 317 | continue |
| 318 | per_builder_dict = meta_dict.get(builder) |
commit-bot@chromium.org | 7dd5d6e | 2013-12-11 20:19:42 +0000 | [diff] [blame] | 319 | if per_builder_dict is not None: |
epoger@google.com | eb83259 | 2013-10-23 15:07:26 +0000 | [diff] [blame] | 320 | fullpath = os.path.join(dirpath, matching_filename) |
| 321 | gm_json.WriteToFile(per_builder_dict, fullpath) |
| 322 | actual_builders_written.append(builder) |
| 323 | |
| 324 | # Check: did we write out the set of per-builder dictionaries we |
| 325 | # expected to? |
| 326 | expected_builders_written = sorted(meta_dict.keys()) |
| 327 | actual_builders_written.sort() |
| 328 | if expected_builders_written != actual_builders_written: |
| 329 | raise KeyError( |
| 330 | 'expected to write dicts for builders %s, but actually wrote them ' |
| 331 | 'for builders %s' % ( |
| 332 | expected_builders_written, actual_builders_written)) |
| 333 | |
| 334 | def _load_actual_and_expected(self): |
| 335 | """Loads the results of all tests, across all builders (based on the |
| 336 | files within self._actuals_root and self._expected_root), |
epoger@google.com | dcb4e65 | 2013-10-11 18:45:33 +0000 | [diff] [blame] | 337 | and stores them in self._results. |
epoger@google.com | f9d134d | 2013-09-27 15:02:44 +0000 | [diff] [blame] | 338 | """ |
commit-bot@chromium.org | a6ecbb8 | 2013-12-19 19:08:31 +0000 | [diff] [blame] | 339 | logging.info('Reading actual-results JSON files from %s...' % |
| 340 | self._actuals_root) |
epoger@google.com | eb83259 | 2013-10-23 15:07:26 +0000 | [diff] [blame] | 341 | actual_builder_dicts = Results._read_dicts_from_root(self._actuals_root) |
commit-bot@chromium.org | a6ecbb8 | 2013-12-19 19:08:31 +0000 | [diff] [blame] | 342 | logging.info('Reading expected-results JSON files from %s...' % |
| 343 | self._expected_root) |
epoger@google.com | eb83259 | 2013-10-23 15:07:26 +0000 | [diff] [blame] | 344 | expected_builder_dicts = Results._read_dicts_from_root(self._expected_root) |
| 345 | |
commit-bot@chromium.org | 16f4180 | 2014-02-26 19:05:20 +0000 | [diff] [blame] | 346 | all_image_pairs = imagepairset.ImagePairSet(IMAGEPAIR_SET_DESCRIPTIONS) |
| 347 | failing_image_pairs = imagepairset.ImagePairSet(IMAGEPAIR_SET_DESCRIPTIONS) |
epoger@google.com | 055e3b5 | 2013-10-26 14:31:11 +0000 | [diff] [blame] | 348 | |
commit-bot@chromium.org | 16f4180 | 2014-02-26 19:05:20 +0000 | [diff] [blame] | 349 | all_image_pairs.ensure_extra_column_values_in_summary( |
| 350 | column_id=KEY__EXTRACOLUMN__RESULT_TYPE, values=[ |
| 351 | KEY__RESULT_TYPE__FAILED, |
| 352 | KEY__RESULT_TYPE__FAILUREIGNORED, |
| 353 | KEY__RESULT_TYPE__NOCOMPARISON, |
| 354 | KEY__RESULT_TYPE__SUCCEEDED, |
epoger@google.com | 5f2bb00 | 2013-10-02 18:57:48 +0000 | [diff] [blame] | 355 | ]) |
commit-bot@chromium.org | 16f4180 | 2014-02-26 19:05:20 +0000 | [diff] [blame] | 356 | failing_image_pairs.ensure_extra_column_values_in_summary( |
| 357 | column_id=KEY__EXTRACOLUMN__RESULT_TYPE, values=[ |
| 358 | KEY__RESULT_TYPE__FAILED, |
| 359 | KEY__RESULT_TYPE__FAILUREIGNORED, |
| 360 | KEY__RESULT_TYPE__NOCOMPARISON, |
epoger@google.com | dcb4e65 | 2013-10-11 18:45:33 +0000 | [diff] [blame] | 361 | ]) |
epoger@google.com | 5f2bb00 | 2013-10-02 18:57:48 +0000 | [diff] [blame] | 362 | |
commit-bot@chromium.org | a6ecbb8 | 2013-12-19 19:08:31 +0000 | [diff] [blame] | 363 | builders = sorted(actual_builder_dicts.keys()) |
| 364 | num_builders = len(builders) |
| 365 | builder_num = 0 |
| 366 | for builder in builders: |
| 367 | builder_num += 1 |
| 368 | logging.info('Generating pixel diffs for builder #%d of %d, "%s"...' % |
| 369 | (builder_num, num_builders, builder)) |
epoger@google.com | f9d134d | 2013-09-27 15:02:44 +0000 | [diff] [blame] | 370 | actual_results_for_this_builder = ( |
epoger@google.com | eb83259 | 2013-10-23 15:07:26 +0000 | [diff] [blame] | 371 | actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS]) |
epoger@google.com | f9d134d | 2013-09-27 15:02:44 +0000 | [diff] [blame] | 372 | for result_type in sorted(actual_results_for_this_builder.keys()): |
| 373 | results_of_this_type = actual_results_for_this_builder[result_type] |
| 374 | if not results_of_this_type: |
| 375 | continue |
| 376 | for image_name in sorted(results_of_this_type.keys()): |
commit-bot@chromium.org | 16f4180 | 2014-02-26 19:05:20 +0000 | [diff] [blame] | 377 | (test, config) = IMAGE_FILENAME_RE.match(image_name).groups() |
| 378 | actual_image_relative_url = Results._create_relative_url( |
| 379 | hashtype_and_digest=results_of_this_type[image_name], |
| 380 | test_name=test) |
epoger@google.com | 055e3b5 | 2013-10-26 14:31:11 +0000 | [diff] [blame] | 381 | |
| 382 | # Default empty expectations; overwrite these if we find any real ones |
| 383 | expectations_per_test = None |
commit-bot@chromium.org | 16f4180 | 2014-02-26 19:05:20 +0000 | [diff] [blame] | 384 | expected_image_relative_url = None |
| 385 | expectations_dict = None |
epoger@google.com | f9d134d | 2013-09-27 15:02:44 +0000 | [diff] [blame] | 386 | try: |
epoger@google.com | 055e3b5 | 2013-10-26 14:31:11 +0000 | [diff] [blame] | 387 | expectations_per_test = ( |
| 388 | expected_builder_dicts |
| 389 | [builder][gm_json.JSONKEY_EXPECTEDRESULTS][image_name]) |
commit-bot@chromium.org | 16f4180 | 2014-02-26 19:05:20 +0000 | [diff] [blame] | 390 | # TODO(epoger): assumes a single allowed digest per test, which is |
| 391 | # fine; see https://code.google.com/p/skia/issues/detail?id=1787 |
| 392 | expected_image_hashtype_and_digest = ( |
epoger@google.com | 055e3b5 | 2013-10-26 14:31:11 +0000 | [diff] [blame] | 393 | expectations_per_test |
| 394 | [gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS][0]) |
commit-bot@chromium.org | 16f4180 | 2014-02-26 19:05:20 +0000 | [diff] [blame] | 395 | expected_image_relative_url = Results._create_relative_url( |
| 396 | hashtype_and_digest=expected_image_hashtype_and_digest, |
| 397 | test_name=test) |
| 398 | expectations_dict = {} |
| 399 | for field in EXPECTATION_FIELDS_PASSED_THRU_VERBATIM: |
| 400 | expectations_dict[field] = expectations_per_test.get(field) |
epoger@google.com | f9d134d | 2013-09-27 15:02:44 +0000 | [diff] [blame] | 401 | except (KeyError, TypeError): |
| 402 | # There are several cases in which we would expect to find |
| 403 | # no expectations for a given test: |
| 404 | # |
| 405 | # 1. result_type == NOCOMPARISON |
| 406 | # There are no expectations for this test yet! |
| 407 | # |
epoger@google.com | 055e3b5 | 2013-10-26 14:31:11 +0000 | [diff] [blame] | 408 | # 2. alternate rendering mode failures (e.g. serialized) |
epoger@google.com | f9d134d | 2013-09-27 15:02:44 +0000 | [diff] [blame] | 409 | # In cases like |
| 410 | # https://code.google.com/p/skia/issues/detail?id=1684 |
| 411 | # ('tileimagefilter GM test failing in serialized render mode'), |
| 412 | # the gm-actuals will list a failure for the alternate |
| 413 | # rendering mode even though we don't have explicit expectations |
| 414 | # for the test (the implicit expectation is that it must |
| 415 | # render the same in all rendering modes). |
| 416 | # |
epoger@google.com | 055e3b5 | 2013-10-26 14:31:11 +0000 | [diff] [blame] | 417 | # Don't log type 1, because it is common. |
epoger@google.com | f9d134d | 2013-09-27 15:02:44 +0000 | [diff] [blame] | 418 | # Log other types, because they are rare and we should know about |
| 419 | # them, but don't throw an exception, because we need to keep our |
| 420 | # tools working in the meanwhile! |
commit-bot@chromium.org | 16f4180 | 2014-02-26 19:05:20 +0000 | [diff] [blame] | 421 | if result_type != KEY__RESULT_TYPE__NOCOMPARISON: |
epoger@google.com | dcb4e65 | 2013-10-11 18:45:33 +0000 | [diff] [blame] | 422 | logging.warning('No expectations found for test: %s' % { |
commit-bot@chromium.org | 16f4180 | 2014-02-26 19:05:20 +0000 | [diff] [blame] | 423 | KEY__EXTRACOLUMN__BUILDER: builder, |
| 424 | KEY__EXTRACOLUMN__RESULT_TYPE: result_type, |
epoger@google.com | f9d134d | 2013-09-27 15:02:44 +0000 | [diff] [blame] | 425 | 'image_name': image_name, |
epoger@google.com | dcb4e65 | 2013-10-11 18:45:33 +0000 | [diff] [blame] | 426 | }) |
epoger@google.com | f9d134d | 2013-09-27 15:02:44 +0000 | [diff] [blame] | 427 | |
| 428 | # If this test was recently rebaselined, it will remain in |
epoger@google.com | 9fb6c8a | 2013-10-09 18:05:58 +0000 | [diff] [blame] | 429 | # the 'failed' set of actuals until all the bots have |
epoger@google.com | f9d134d | 2013-09-27 15:02:44 +0000 | [diff] [blame] | 430 | # cycled (although the expectations have indeed been set |
| 431 | # from the most recent actuals). Treat these as successes |
| 432 | # instead of failures. |
| 433 | # |
| 434 | # TODO(epoger): Do we need to do something similar in |
| 435 | # other cases, such as when we have recently marked a test |
epoger@google.com | 9fb6c8a | 2013-10-09 18:05:58 +0000 | [diff] [blame] | 436 | # as ignoreFailure but it still shows up in the 'failed' |
epoger@google.com | f9d134d | 2013-09-27 15:02:44 +0000 | [diff] [blame] | 437 | # category? Maybe we should not rely on the result_type |
| 438 | # categories recorded within the gm_actuals AT ALL, and |
| 439 | # instead evaluate the result_type ourselves based on what |
| 440 | # we see in expectations vs actual checksum? |
commit-bot@chromium.org | 16f4180 | 2014-02-26 19:05:20 +0000 | [diff] [blame] | 441 | if expected_image_relative_url == actual_image_relative_url: |
| 442 | updated_result_type = KEY__RESULT_TYPE__SUCCEEDED |
epoger@google.com | f9d134d | 2013-09-27 15:02:44 +0000 | [diff] [blame] | 443 | else: |
| 444 | updated_result_type = result_type |
commit-bot@chromium.org | 16f4180 | 2014-02-26 19:05:20 +0000 | [diff] [blame] | 445 | extra_columns_dict = { |
| 446 | KEY__EXTRACOLUMN__RESULT_TYPE: updated_result_type, |
| 447 | KEY__EXTRACOLUMN__BUILDER: builder, |
| 448 | KEY__EXTRACOLUMN__TEST: test, |
| 449 | KEY__EXTRACOLUMN__CONFIG: config, |
epoger@google.com | afaad3d | 2013-09-30 15:06:25 +0000 | [diff] [blame] | 450 | } |
commit-bot@chromium.org | 3b98759 | 2014-03-04 00:58:21 +0000 | [diff] [blame] | 451 | try: |
| 452 | image_pair = imagepair.ImagePair( |
| 453 | image_diff_db=self._image_diff_db, |
| 454 | base_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL, |
| 455 | imageA_relative_url=expected_image_relative_url, |
| 456 | imageB_relative_url=actual_image_relative_url, |
| 457 | expectations=expectations_dict, |
| 458 | extra_columns=extra_columns_dict) |
| 459 | all_image_pairs.add_image_pair(image_pair) |
| 460 | if updated_result_type != KEY__RESULT_TYPE__SUCCEEDED: |
| 461 | failing_image_pairs.add_image_pair(image_pair) |
| 462 | except Exception: |
| 463 | logging.exception('got exception while creating new ImagePair') |
epoger@google.com | dcb4e65 | 2013-10-11 18:45:33 +0000 | [diff] [blame] | 464 | |
| 465 | self._results = { |
commit-bot@chromium.org | 16f4180 | 2014-02-26 19:05:20 +0000 | [diff] [blame] | 466 | KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(), |
| 467 | KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(), |
epoger@google.com | dcb4e65 | 2013-10-11 18:45:33 +0000 | [diff] [blame] | 468 | } |
epoger@google.com | afaad3d | 2013-09-30 15:06:25 +0000 | [diff] [blame] | 469 | |
commit-bot@chromium.org | 7b06c8e | 2013-12-23 22:47:15 +0000 | [diff] [blame] | 470 | |
| 471 | def main(): |
| 472 | logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', |
| 473 | datefmt='%m/%d/%Y %H:%M:%S', |
| 474 | level=logging.INFO) |
| 475 | parser = argparse.ArgumentParser() |
| 476 | parser.add_argument( |
commit-bot@chromium.org | 7498d95 | 2014-03-13 14:56:29 +0000 | [diff] [blame] | 477 | '--actuals', default=DEFAULT_ACTUALS_DIR, |
commit-bot@chromium.org | 7b06c8e | 2013-12-23 22:47:15 +0000 | [diff] [blame] | 478 | help='Directory containing all actual-result JSON files') |
| 479 | parser.add_argument( |
commit-bot@chromium.org | 7498d95 | 2014-03-13 14:56:29 +0000 | [diff] [blame] | 480 | '--expectations', default=DEFAULT_EXPECTATIONS_DIR, |
| 481 | help='Directory containing all expected-result JSON files; defaults to ' |
| 482 | '\'%(default)s\' .') |
commit-bot@chromium.org | 7b06c8e | 2013-12-23 22:47:15 +0000 | [diff] [blame] | 483 | parser.add_argument( |
| 484 | '--outfile', required=True, |
commit-bot@chromium.org | 7498d95 | 2014-03-13 14:56:29 +0000 | [diff] [blame] | 485 | help='File to write result summary into, in JSON format.') |
commit-bot@chromium.org | 7b06c8e | 2013-12-23 22:47:15 +0000 | [diff] [blame] | 486 | parser.add_argument( |
commit-bot@chromium.org | 7498d95 | 2014-03-13 14:56:29 +0000 | [diff] [blame] | 487 | '--results', default=KEY__HEADER__RESULTS_FAILURES, |
| 488 | help='Which result types to include. Defaults to \'%(default)s\'; ' |
| 489 | 'must be one of ' + |
| 490 | str([KEY__HEADER__RESULTS_FAILURES, KEY__HEADER__RESULTS_ALL])) |
| 491 | parser.add_argument( |
| 492 | '--workdir', default=DEFAULT_GENERATED_IMAGES_ROOT, |
| 493 | help='Directory within which to download images and generate diffs; ' |
| 494 | 'defaults to \'%(default)s\' .') |
commit-bot@chromium.org | 7b06c8e | 2013-12-23 22:47:15 +0000 | [diff] [blame] | 495 | args = parser.parse_args() |
| 496 | results = Results(actuals_root=args.actuals, |
| 497 | expected_root=args.expectations, |
| 498 | generated_images_root=args.workdir) |
commit-bot@chromium.org | 7498d95 | 2014-03-13 14:56:29 +0000 | [diff] [blame] | 499 | gm_json.WriteToFile( |
| 500 | results.get_packaged_results_of_type(results_type=args.results), |
| 501 | args.outfile) |
commit-bot@chromium.org | 7b06c8e | 2013-12-23 22:47:15 +0000 | [diff] [blame] | 502 | |
| 503 | |
| 504 | if __name__ == '__main__': |
| 505 | main() |