epoger@google.com | 9d33154 | 2013-05-28 15:25:38 +0000 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | # Copyright (c) 2013 The Chromium Authors. All rights reserved. |
| 3 | # Use of this source code is governed by a BSD-style license that can be |
| 4 | # found in the LICENSE file. |
| 5 | |
| 6 | """Schema of the JSON summary file written out by the GM tool. |
| 7 | |
| 8 | This must be kept in sync with the kJsonKey_ constants in gm_expectations.cpp ! |
| 9 | """ |
| 10 | |
| 11 | __author__ = 'Elliot Poger' |
| 12 | |
| 13 | |
| 14 | # system-level imports |
commit-bot@chromium.org | 83aaf88 | 2013-12-18 15:28:24 +0000 | [diff] [blame] | 15 | import io |
epoger@google.com | 9d33154 | 2013-05-28 15:25:38 +0000 | [diff] [blame] | 16 | import json |
scroggo@google.com | d23e683 | 2013-10-10 21:09:24 +0000 | [diff] [blame] | 17 | import os |
commit-bot@chromium.org | 16f4180 | 2014-02-26 19:05:20 +0000 | [diff] [blame] | 18 | import posixpath |
| 19 | import re |
epoger@google.com | 9d33154 | 2013-05-28 15:25:38 +0000 | [diff] [blame] | 20 | |
| 21 | |
epoger@google.com | a783f2b | 2013-07-08 17:51:58 +0000 | [diff] [blame] | 22 | # Key strings used in GM results JSON files (both expected-results.json and |
| 23 | # actual-results.json). |
| 24 | # |
commit-bot@chromium.org | 16f4180 | 2014-02-26 19:05:20 +0000 | [diff] [blame] | 25 | # NOTE: These constants must be kept in sync with the kJsonKey_ constants in |
epoger@google.com | 9d33154 | 2013-05-28 15:25:38 +0000 | [diff] [blame] | 26 | # gm_expectations.cpp ! |
epoger@google.com | a783f2b | 2013-07-08 17:51:58 +0000 | [diff] [blame] | 27 | |
epoger@google.com | 06e626d | 2013-09-03 17:32:15 +0000 | [diff] [blame] | 28 | |
epoger@google.com | 9d33154 | 2013-05-28 15:25:38 +0000 | [diff] [blame] | 29 | JSONKEY_ACTUALRESULTS = 'actual-results' |
epoger@google.com | 06e626d | 2013-09-03 17:32:15 +0000 | [diff] [blame] | 30 | |
epoger@google.com | a783f2b | 2013-07-08 17:51:58 +0000 | [diff] [blame] | 31 | # Tests whose results failed to match expectations. |
epoger@google.com | 9d33154 | 2013-05-28 15:25:38 +0000 | [diff] [blame] | 32 | JSONKEY_ACTUALRESULTS_FAILED = 'failed' |
epoger@google.com | 06e626d | 2013-09-03 17:32:15 +0000 | [diff] [blame] | 33 | |
epoger@google.com | a783f2b | 2013-07-08 17:51:58 +0000 | [diff] [blame] | 34 | # Tests whose results failed to match expectations, but IGNOREFAILURE causes |
| 35 | # us to take them less seriously. |
epoger@google.com | 9d33154 | 2013-05-28 15:25:38 +0000 | [diff] [blame] | 36 | JSONKEY_ACTUALRESULTS_FAILUREIGNORED = 'failure-ignored' |
epoger@google.com | 06e626d | 2013-09-03 17:32:15 +0000 | [diff] [blame] | 37 | |
epoger@google.com | a783f2b | 2013-07-08 17:51:58 +0000 | [diff] [blame] | 38 | # Tests for which we do not have any expectations. They may be new tests that |
| 39 | # we haven't had a chance to check in expectations for yet, or we may have |
| 40 | # consciously decided to leave them without expectations because we are unhappy |
| 41 | # with the results (although we should try to move away from that, and instead |
| 42 | # check in expectations with the IGNOREFAILURE flag set). |
epoger@google.com | 9d33154 | 2013-05-28 15:25:38 +0000 | [diff] [blame] | 43 | JSONKEY_ACTUALRESULTS_NOCOMPARISON = 'no-comparison' |
epoger@google.com | 06e626d | 2013-09-03 17:32:15 +0000 | [diff] [blame] | 44 | |
epoger@google.com | a783f2b | 2013-07-08 17:51:58 +0000 | [diff] [blame] | 45 | # Tests whose results matched their expectations. |
epoger@google.com | 9d33154 | 2013-05-28 15:25:38 +0000 | [diff] [blame] | 46 | JSONKEY_ACTUALRESULTS_SUCCEEDED = 'succeeded' |
| 47 | |
epoger@google.com | 06e626d | 2013-09-03 17:32:15 +0000 | [diff] [blame] | 48 | |
epoger@google.com | 53953b4 | 2013-07-02 20:22:27 +0000 | [diff] [blame] | 49 | JSONKEY_EXPECTEDRESULTS = 'expected-results' |
epoger@google.com | 06e626d | 2013-09-03 17:32:15 +0000 | [diff] [blame] | 50 | |
epoger@google.com | a783f2b | 2013-07-08 17:51:58 +0000 | [diff] [blame] | 51 | # One or more [HashType/DigestValue] pairs representing valid results for this |
| 52 | # test. Typically, there will just be one pair, but we allow for multiple |
| 53 | # expectations, and the test will pass if any one of them is matched. |
epoger@google.com | 53953b4 | 2013-07-02 20:22:27 +0000 | [diff] [blame] | 54 | JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS = 'allowed-digests' |
epoger@google.com | 06e626d | 2013-09-03 17:32:15 +0000 | [diff] [blame] | 55 | |
| 56 | # Optional: one or more integers listing Skia bugs (under |
| 57 | # https://code.google.com/p/skia/issues/list ) that pertain to this expectation. |
| 58 | JSONKEY_EXPECTEDRESULTS_BUGS = 'bugs' |
| 59 | |
epoger@google.com | a783f2b | 2013-07-08 17:51:58 +0000 | [diff] [blame] | 60 | # If IGNOREFAILURE is set to True, a failure of this test will be reported |
| 61 | # within the FAILUREIGNORED section (thus NOT causing the buildbots to go red) |
| 62 | # rather than the FAILED section (which WOULD cause the buildbots to go red). |
epoger@google.com | 53953b4 | 2013-07-02 20:22:27 +0000 | [diff] [blame] | 63 | JSONKEY_EXPECTEDRESULTS_IGNOREFAILURE = 'ignore-failure' |
| 64 | |
epoger@google.com | 06e626d | 2013-09-03 17:32:15 +0000 | [diff] [blame] | 65 | # Optional: a free-form text string with human-readable information about |
| 66 | # this expectation. |
| 67 | JSONKEY_EXPECTEDRESULTS_NOTES = 'notes' |
| 68 | |
| 69 | # Optional: boolean indicating whether this expectation was reviewed/approved |
| 70 | # by a human being. |
| 71 | # If True: a human looked at this image and approved it. |
| 72 | # If False: this expectation was committed blind. (In such a case, please |
| 73 | # add notes indicating why!) |
| 74 | # If absent: this expectation was committed by a tool that didn't enforce human |
| 75 | # review of expectations. |
| 76 | JSONKEY_EXPECTEDRESULTS_REVIEWED = 'reviewed-by-human' |
| 77 | |
| 78 | |
epoger@google.com | a783f2b | 2013-07-08 17:51:58 +0000 | [diff] [blame] | 79 | # Allowed hash types for test expectations. |
epoger@google.com | 53953b4 | 2013-07-02 20:22:27 +0000 | [diff] [blame] | 80 | JSONKEY_HASHTYPE_BITMAP_64BITMD5 = 'bitmap-64bitMD5' |
| 81 | |
epoger@google.com | 61822a2 | 2013-07-16 18:56:32 +0000 | [diff] [blame] | 82 | # Root directory where the buildbots store their actually-generated images... |
| 83 | # as a publicly readable HTTP URL: |
| 84 | GM_ACTUALS_ROOT_HTTP_URL = ( |
| 85 | 'http://chromium-skia-gm.commondatastorage.googleapis.com/gm') |
| 86 | # as a GS URL that allows credential-protected write access: |
| 87 | GM_ACTUALS_ROOT_GS_URL = 'gs://chromium-skia-gm/gm' |
| 88 | |
scroggo@google.com | d23e683 | 2013-10-10 21:09:24 +0000 | [diff] [blame] | 89 | # Root directory where buildbots store skimage actual results json files. |
| 90 | SKIMAGE_ACTUALS_BASE_URL = ( |
| 91 | 'http://chromium-skia-gm.commondatastorage.googleapis.com/skimage/actuals') |
| 92 | # Root directory inside trunk where skimage expectations are stored. |
| 93 | SKIMAGE_EXPECTATIONS_ROOT = os.path.join('expectations', 'skimage') |
| 94 | |
epoger@google.com | 61822a2 | 2013-07-16 18:56:32 +0000 | [diff] [blame] | 95 | # Pattern used to assemble each image's filename |
commit-bot@chromium.org | 69ac6de | 2014-01-07 17:21:11 +0000 | [diff] [blame] | 96 | IMAGE_FILENAME_PATTERN = '(.+)_(.+)\.png' # matches (testname, config) |
epoger@google.com | 61822a2 | 2013-07-16 18:56:32 +0000 | [diff] [blame] | 97 | |
commit-bot@chromium.org | 16f4180 | 2014-02-26 19:05:20 +0000 | [diff] [blame] | 98 | # Pattern used to create image URLs, relative to some base URL. |
| 99 | GM_RELATIVE_URL_FORMATTER = '%s/%s/%s.png' # pass in (hash_type, test_name, |
| 100 | # hash_digest) |
| 101 | GM_RELATIVE_URL_PATTERN = '(.+)/(.+)/(.+).png' # matches (hash_type, test_name, |
| 102 | # hash_digest) |
| 103 | GM_RELATIVE_URL_RE = re.compile(GM_RELATIVE_URL_PATTERN) |
| 104 | |
| 105 | |
epoger@google.com | 61822a2 | 2013-07-16 18:56:32 +0000 | [diff] [blame] | 106 | def CreateGmActualUrl(test_name, hash_type, hash_digest, |
| 107 | gm_actuals_root_url=GM_ACTUALS_ROOT_HTTP_URL): |
| 108 | """Return the URL we can use to download a particular version of |
| 109 | the actually-generated image for this particular GM test. |
| 110 | |
| 111 | test_name: name of the test, e.g. 'perlinnoise' |
| 112 | hash_type: string indicating the hash type used to generate hash_digest, |
| 113 | e.g. JSONKEY_HASHTYPE_BITMAP_64BITMD5 |
| 114 | hash_digest: the hash digest of the image to retrieve |
| 115 | gm_actuals_root_url: root url where actual images are stored |
| 116 | """ |
commit-bot@chromium.org | 16f4180 | 2014-02-26 19:05:20 +0000 | [diff] [blame] | 117 | return posixpath.join( |
| 118 | gm_actuals_root_url, CreateGmRelativeUrl( |
| 119 | test_name=test_name, hash_type=hash_type, hash_digest=hash_digest)) |
| 120 | |
| 121 | |
| 122 | def CreateGmRelativeUrl(test_name, hash_type, hash_digest): |
| 123 | """Returns a relative URL pointing at a test result's image. |
| 124 | |
| 125 | Returns the URL we can use to download a particular version of |
| 126 | the actually-generated image for this particular GM test, |
| 127 | relative to the URL root. |
| 128 | |
| 129 | Args: |
| 130 | test_name: name of the test, e.g. 'perlinnoise' |
| 131 | hash_type: string indicating the hash type used to generate hash_digest, |
| 132 | e.g. JSONKEY_HASHTYPE_BITMAP_64BITMD5 |
| 133 | hash_digest: the hash digest of the image to retrieve |
| 134 | """ |
| 135 | return GM_RELATIVE_URL_FORMATTER % (hash_type, test_name, hash_digest) |
| 136 | |
| 137 | |
| 138 | def SplitGmRelativeUrl(url): |
| 139 | """Splits the relative URL into (test_name, hash_type, hash_digest) tuple. |
| 140 | |
| 141 | This is the inverse of CreateGmRelativeUrl(). |
| 142 | |
| 143 | Args: |
| 144 | url: a URL generated with CreateGmRelativeUrl(). |
| 145 | |
| 146 | Returns: (test_name, hash_type, hash_digest) tuple. |
| 147 | """ |
| 148 | hash_type, test_name, hash_digest = GM_RELATIVE_URL_RE.match(url).groups() |
| 149 | return (test_name, hash_type, hash_digest) |
| 150 | |
epoger@google.com | 61822a2 | 2013-07-16 18:56:32 +0000 | [diff] [blame] | 151 | |
epoger@google.com | 4075fd4 | 2013-06-04 17:50:36 +0000 | [diff] [blame] | 152 | def LoadFromString(file_contents): |
epoger@google.com | 9d33154 | 2013-05-28 15:25:38 +0000 | [diff] [blame] | 153 | """Loads the JSON summary written out by the GM tool. |
| 154 | Returns a dictionary keyed by the values listed as JSONKEY_ constants |
| 155 | above.""" |
epoger@google.com | 4075fd4 | 2013-06-04 17:50:36 +0000 | [diff] [blame] | 156 | # TODO(epoger): we should add a version number to the JSON file to ensure |
epoger@google.com | 9d33154 | 2013-05-28 15:25:38 +0000 | [diff] [blame] | 157 | # that the writer and reader agree on the schema (raising an exception |
| 158 | # otherwise). |
epoger@google.com | 4075fd4 | 2013-06-04 17:50:36 +0000 | [diff] [blame] | 159 | json_dict = json.loads(file_contents) |
epoger@google.com | 9d33154 | 2013-05-28 15:25:38 +0000 | [diff] [blame] | 160 | return json_dict |
epoger@google.com | 4075fd4 | 2013-06-04 17:50:36 +0000 | [diff] [blame] | 161 | |
commit-bot@chromium.org | 16f4180 | 2014-02-26 19:05:20 +0000 | [diff] [blame] | 162 | |
epoger@google.com | 4075fd4 | 2013-06-04 17:50:36 +0000 | [diff] [blame] | 163 | def LoadFromFile(file_path): |
| 164 | """Loads the JSON summary written out by the GM tool. |
| 165 | Returns a dictionary keyed by the values listed as JSONKEY_ constants |
| 166 | above.""" |
| 167 | file_contents = open(file_path, 'r').read() |
| 168 | return LoadFromString(file_contents) |
epoger@google.com | a783f2b | 2013-07-08 17:51:58 +0000 | [diff] [blame] | 169 | |
commit-bot@chromium.org | 16f4180 | 2014-02-26 19:05:20 +0000 | [diff] [blame] | 170 | |
epoger@google.com | a783f2b | 2013-07-08 17:51:58 +0000 | [diff] [blame] | 171 | def WriteToFile(json_dict, file_path): |
commit-bot@chromium.org | 83aaf88 | 2013-12-18 15:28:24 +0000 | [diff] [blame] | 172 | """Writes the JSON summary in json_dict out to file_path. |
| 173 | |
| 174 | The file is written Unix-style (each line ends with just LF, not CRLF); |
| 175 | see https://code.google.com/p/skia/issues/detail?id=1815 for reasons.""" |
| 176 | with io.open(file_path, mode='w', newline='', encoding='utf-8') as outfile: |
| 177 | outfile.write(unicode(json.dumps(json_dict, outfile, sort_keys=True, |
| 178 | indent=2))) |