epoger@google.com | 9d33154 | 2013-05-28 15:25:38 +0000 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | # Copyright (c) 2013 The Chromium Authors. All rights reserved. |
| 3 | # Use of this source code is governed by a BSD-style license that can be |
| 4 | # found in the LICENSE file. |
| 5 | |
| 6 | """Schema of the JSON summary file written out by the GM tool. |
| 7 | |
| 8 | This must be kept in sync with the kJsonKey_ constants in gm_expectations.cpp ! |
| 9 | """ |
| 10 | |
| 11 | __author__ = 'Elliot Poger' |
| 12 | |
| 13 | |
| 14 | # system-level imports |
| 15 | import json |
| 16 | |
| 17 | |
epoger@google.com | a783f2b | 2013-07-08 17:51:58 +0000 | [diff] [blame] | 18 | # Key strings used in GM results JSON files (both expected-results.json and |
| 19 | # actual-results.json). |
| 20 | # |
epoger@google.com | 9d33154 | 2013-05-28 15:25:38 +0000 | [diff] [blame] | 21 | # These constants must be kept in sync with the kJsonKey_ constants in |
| 22 | # gm_expectations.cpp ! |
epoger@google.com | a783f2b | 2013-07-08 17:51:58 +0000 | [diff] [blame] | 23 | |
epoger@google.com | 9d33154 | 2013-05-28 15:25:38 +0000 | [diff] [blame] | 24 | JSONKEY_ACTUALRESULTS = 'actual-results' |
epoger@google.com | a783f2b | 2013-07-08 17:51:58 +0000 | [diff] [blame] | 25 | # Tests whose results failed to match expectations. |
epoger@google.com | 9d33154 | 2013-05-28 15:25:38 +0000 | [diff] [blame] | 26 | JSONKEY_ACTUALRESULTS_FAILED = 'failed' |
epoger@google.com | a783f2b | 2013-07-08 17:51:58 +0000 | [diff] [blame] | 27 | # Tests whose results failed to match expectations, but IGNOREFAILURE causes |
| 28 | # us to take them less seriously. |
epoger@google.com | 9d33154 | 2013-05-28 15:25:38 +0000 | [diff] [blame] | 29 | JSONKEY_ACTUALRESULTS_FAILUREIGNORED = 'failure-ignored' |
epoger@google.com | a783f2b | 2013-07-08 17:51:58 +0000 | [diff] [blame] | 30 | # Tests for which we do not have any expectations. They may be new tests that |
| 31 | # we haven't had a chance to check in expectations for yet, or we may have |
| 32 | # consciously decided to leave them without expectations because we are unhappy |
| 33 | # with the results (although we should try to move away from that, and instead |
| 34 | # check in expectations with the IGNOREFAILURE flag set). |
epoger@google.com | 9d33154 | 2013-05-28 15:25:38 +0000 | [diff] [blame] | 35 | JSONKEY_ACTUALRESULTS_NOCOMPARISON = 'no-comparison' |
epoger@google.com | a783f2b | 2013-07-08 17:51:58 +0000 | [diff] [blame] | 36 | # Tests whose results matched their expectations. |
epoger@google.com | 9d33154 | 2013-05-28 15:25:38 +0000 | [diff] [blame] | 37 | JSONKEY_ACTUALRESULTS_SUCCEEDED = 'succeeded' |
| 38 | |
epoger@google.com | 53953b4 | 2013-07-02 20:22:27 +0000 | [diff] [blame] | 39 | JSONKEY_EXPECTEDRESULTS = 'expected-results' |
epoger@google.com | a783f2b | 2013-07-08 17:51:58 +0000 | [diff] [blame] | 40 | # One or more [HashType/DigestValue] pairs representing valid results for this |
| 41 | # test. Typically, there will just be one pair, but we allow for multiple |
| 42 | # expectations, and the test will pass if any one of them is matched. |
epoger@google.com | 53953b4 | 2013-07-02 20:22:27 +0000 | [diff] [blame] | 43 | JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS = 'allowed-digests' |
epoger@google.com | a783f2b | 2013-07-08 17:51:58 +0000 | [diff] [blame] | 44 | # If IGNOREFAILURE is set to True, a failure of this test will be reported |
| 45 | # within the FAILUREIGNORED section (thus NOT causing the buildbots to go red) |
| 46 | # rather than the FAILED section (which WOULD cause the buildbots to go red). |
epoger@google.com | 53953b4 | 2013-07-02 20:22:27 +0000 | [diff] [blame] | 47 | JSONKEY_EXPECTEDRESULTS_IGNOREFAILURE = 'ignore-failure' |
| 48 | |
epoger@google.com | a783f2b | 2013-07-08 17:51:58 +0000 | [diff] [blame] | 49 | # Allowed hash types for test expectations. |
epoger@google.com | 53953b4 | 2013-07-02 20:22:27 +0000 | [diff] [blame] | 50 | JSONKEY_HASHTYPE_BITMAP_64BITMD5 = 'bitmap-64bitMD5' |
| 51 | |
epoger@google.com | 61822a2 | 2013-07-16 18:56:32 +0000 | [diff] [blame] | 52 | # Root directory where the buildbots store their actually-generated images... |
| 53 | # as a publicly readable HTTP URL: |
| 54 | GM_ACTUALS_ROOT_HTTP_URL = ( |
| 55 | 'http://chromium-skia-gm.commondatastorage.googleapis.com/gm') |
| 56 | # as a GS URL that allows credential-protected write access: |
| 57 | GM_ACTUALS_ROOT_GS_URL = 'gs://chromium-skia-gm/gm' |
| 58 | |
| 59 | # Pattern used to assemble each image's filename |
| 60 | IMAGE_FILENAME_PATTERN = '(\S+)_(\S+).png' # matches (testname, config) |
| 61 | |
| 62 | def CreateGmActualUrl(test_name, hash_type, hash_digest, |
| 63 | gm_actuals_root_url=GM_ACTUALS_ROOT_HTTP_URL): |
| 64 | """Return the URL we can use to download a particular version of |
| 65 | the actually-generated image for this particular GM test. |
| 66 | |
| 67 | test_name: name of the test, e.g. 'perlinnoise' |
| 68 | hash_type: string indicating the hash type used to generate hash_digest, |
| 69 | e.g. JSONKEY_HASHTYPE_BITMAP_64BITMD5 |
| 70 | hash_digest: the hash digest of the image to retrieve |
| 71 | gm_actuals_root_url: root url where actual images are stored |
| 72 | """ |
| 73 | return '%s/%s/%s/%s.png' % (gm_actuals_root_url, hash_type, test_name, |
| 74 | hash_digest) |
| 75 | |
epoger@google.com | 4075fd4 | 2013-06-04 17:50:36 +0000 | [diff] [blame] | 76 | def LoadFromString(file_contents): |
epoger@google.com | 9d33154 | 2013-05-28 15:25:38 +0000 | [diff] [blame] | 77 | """Loads the JSON summary written out by the GM tool. |
| 78 | Returns a dictionary keyed by the values listed as JSONKEY_ constants |
| 79 | above.""" |
epoger@google.com | 4075fd4 | 2013-06-04 17:50:36 +0000 | [diff] [blame] | 80 | # TODO(epoger): we should add a version number to the JSON file to ensure |
epoger@google.com | 9d33154 | 2013-05-28 15:25:38 +0000 | [diff] [blame] | 81 | # that the writer and reader agree on the schema (raising an exception |
| 82 | # otherwise). |
epoger@google.com | 4075fd4 | 2013-06-04 17:50:36 +0000 | [diff] [blame] | 83 | json_dict = json.loads(file_contents) |
epoger@google.com | 9d33154 | 2013-05-28 15:25:38 +0000 | [diff] [blame] | 84 | return json_dict |
epoger@google.com | 4075fd4 | 2013-06-04 17:50:36 +0000 | [diff] [blame] | 85 | |
| 86 | def LoadFromFile(file_path): |
| 87 | """Loads the JSON summary written out by the GM tool. |
| 88 | Returns a dictionary keyed by the values listed as JSONKEY_ constants |
| 89 | above.""" |
| 90 | file_contents = open(file_path, 'r').read() |
| 91 | return LoadFromString(file_contents) |
epoger@google.com | a783f2b | 2013-07-08 17:51:58 +0000 | [diff] [blame] | 92 | |
| 93 | def WriteToFile(json_dict, file_path): |
| 94 | """Writes the JSON summary in json_dict out to file_path.""" |
| 95 | with open(file_path, 'w') as outfile: |
| 96 | json.dump(json_dict, outfile, sort_keys=True, indent=2) |