rebaseline_server: apply ignored-tests.txt within rebaseline_server, not just on the bots
(SkipBuildbotRuns)
BUG=skia:2514
NOTREECHECKS=True
NOTRY=True
R=rmistry@google.com
Author: epoger@google.com
Review URL: https://codereview.chromium.org/287623006
git-svn-id: http://skia.googlecode.com/svn/trunk@14740 2bbb7eff-a529-9590-31e7-b0007b416f81
diff --git a/gm/rebaseline_server/compare_to_expectations.py b/gm/rebaseline_server/compare_to_expectations.py
index 0f3c352..e9677bd 100755
--- a/gm/rebaseline_server/compare_to_expectations.py
+++ b/gm/rebaseline_server/compare_to_expectations.py
@@ -48,6 +48,7 @@
results.KEY__EXPECTATIONS__REVIEWED,
]
DEFAULT_EXPECTATIONS_DIR = os.path.join(TRUNK_DIRECTORY, 'expectations', 'gm')
+DEFAULT_IGNORE_FAILURES_FILE = 'ignored-tests.txt'
IMAGEPAIR_SET_DESCRIPTIONS = ('expected image', 'actual image')
@@ -64,12 +65,15 @@
def __init__(self, actuals_root=results.DEFAULT_ACTUALS_DIR,
expected_root=DEFAULT_EXPECTATIONS_DIR,
+ ignore_failures_file=DEFAULT_IGNORE_FAILURES_FILE,
generated_images_root=results.DEFAULT_GENERATED_IMAGES_ROOT,
diff_base_url=None, builder_regex_list=None):
"""
Args:
actuals_root: root directory containing all actual-results.json files
expected_root: root directory containing all expected-results.json files
+ ignore_failures_file: if a file with this name is found within
+ expected_root, ignore failures for any tests listed in the file
generated_images_root: directory within which to create all pixel diffs;
if this directory does not yet exist, it will be created
diff_base_url: base URL within which the client should look for diff
@@ -87,6 +91,11 @@
download_actuals.create_filepath_url(generated_images_root))
self._actuals_root = actuals_root
self._expected_root = expected_root
+ self._ignore_failures_on_these_tests = []
+ if ignore_failures_file:
+ self._ignore_failures_on_these_tests = (
+ ExpectationComparisons._read_noncomment_lines(
+ os.path.join(expected_root, ignore_failures_file)))
self._load_actual_and_expected()
self._timestamp = int(time.time())
logging.info('Results complete; took %d seconds.' %
@@ -316,10 +325,11 @@
# categories recorded within the gm_actuals AT ALL, and
# instead evaluate the result_type ourselves based on what
# we see in expectations vs actual checksum?
- # See related http://skbug.com/2514 ('rebaseline_server: apply
- # ignored-tests.txt within rebaseline_server, not just on the bots')
if expected_image_relative_url == actual_image_relative_url:
updated_result_type = results.KEY__RESULT_TYPE__SUCCEEDED
+ elif ((result_type == results.KEY__RESULT_TYPE__FAILED) and
+ (test in self._ignore_failures_on_these_tests)):
+ updated_result_type = results.KEY__RESULT_TYPE__FAILUREIGNORED
else:
updated_result_type = result_type
extra_columns_dict = {
@@ -362,6 +372,11 @@
help='Directory containing all expected-result JSON files; defaults to '
'\'%(default)s\' .')
parser.add_argument(
+ '--ignore-failures-file', default=DEFAULT_IGNORE_FAILURES_FILE,
+ help='If a file with this name is found within the EXPECTATIONS dir, '
+ 'ignore failures for any tests listed in the file; defaults to '
+ '\'%(default)s\' .')
+ parser.add_argument(
'--outfile', required=True,
help='File to write result summary into, in JSON format.')
parser.add_argument(
@@ -375,9 +390,11 @@
help='Directory within which to download images and generate diffs; '
'defaults to \'%(default)s\' .')
args = parser.parse_args()
- results_obj = ExpectationComparisons(actuals_root=args.actuals,
- expected_root=args.expectations,
- generated_images_root=args.workdir)
+ results_obj = ExpectationComparisons(
+ actuals_root=args.actuals,
+ expected_root=args.expectations,
+ ignore_failures_file=args.ignore_failures_file,
+ generated_images_root=args.workdir)
gm_json.WriteToFile(
results_obj.get_packaged_results_of_type(results_type=args.results),
args.outfile)
diff --git a/gm/rebaseline_server/results.py b/gm/rebaseline_server/results.py
index 45ab133..70b2342 100755
--- a/gm/rebaseline_server/results.py
+++ b/gm/rebaseline_server/results.py
@@ -246,6 +246,22 @@
return meta_dict
@staticmethod
+ def _read_noncomment_lines(path):
+ """Return a list of all noncomment lines within a file.
+
+ (A "noncomment" line is one that does not start with a '#'.)
+
+ Args:
+ path: path to file
+ """
+ lines = []
+ with open(path, 'r') as fh:
+ for line in fh:
+ if not line.startswith('#'):
+ lines.append(line.strip())
+ return lines
+
+ @staticmethod
def _create_relative_url(hashtype_and_digest, test_name):
"""Returns the URL for this image, relative to GM_ACTUALS_ROOT_HTTP_URL.
diff --git a/gm/rebaseline_server/testdata/inputs/gm-actuals/Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug/actual-results.json b/gm/rebaseline_server/testdata/inputs/gm-actuals/Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug/actual-results.json
index 0a1de74..354f439 100644
--- a/gm/rebaseline_server/testdata/inputs/gm-actuals/Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug/actual-results.json
+++ b/gm/rebaseline_server/testdata/inputs/gm-actuals/Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug/actual-results.json
@@ -1,6 +1,9 @@
{
"actual-results" : {
- "failed" : null,
+ "failed" : {
+ "blanket-ignored_565.png" : [ "bitmap-64bitMD5", 22222222 ],
+ "blanket-ignored_8888.png" : [ "bitmap-64bitMD5", 22222222 ]
+ },
"failure-ignored" : {
"displacement_565.png" : [ "bitmap-64bitMD5", 4569468668668628191 ],
"displacement_8888.png" : [ "bitmap-64bitMD5", 11401048196735046263 ],
@@ -54,6 +57,18 @@
],
"ignore-failure" : false
},
+ "blanket-ignored_565.png" : {
+ "allowed-digests" : [
+ [ "bitmap-64bitMD5", 111111111 ]
+ ],
+ "ignore-failure" : false
+ },
+ "blanket-ignored_8888.png" : {
+ "allowed-digests" : [
+ [ "bitmap-64bitMD5", 111111111 ]
+ ],
+ "ignore-failure" : false
+ },
"displacement_565.png" : {
"allowed-digests" : [
[ "bitmap-64bitMD5", 16249664097236120848 ]
diff --git a/gm/rebaseline_server/testdata/inputs/gm-expectations/Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug/expected-results.json b/gm/rebaseline_server/testdata/inputs/gm-expectations/Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug/expected-results.json
index 1b3fdc6..f3f49bb 100644
--- a/gm/rebaseline_server/testdata/inputs/gm-expectations/Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug/expected-results.json
+++ b/gm/rebaseline_server/testdata/inputs/gm-expectations/Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug/expected-results.json
@@ -66,6 +66,24 @@
],
"reviewed-by-human": false
},
+ "blanket-ignored_565.png": {
+ "allowed-digests": [
+ [
+ "bitmap-64bitMD5",
+ 111111111
+ ]
+ ],
+ "ignore-failure": false
+ },
+ "blanket-ignored_8888.png": {
+ "allowed-digests": [
+ [
+ "bitmap-64bitMD5",
+ 111111111
+ ]
+ ],
+ "ignore-failure": false
+ },
"displacement_565.png": {
"allowed-digests": [
[
diff --git a/gm/rebaseline_server/testdata/inputs/gm-expectations/ignored-tests.txt b/gm/rebaseline_server/testdata/inputs/gm-expectations/ignored-tests.txt
new file mode 100644
index 0000000..b14eb96
--- /dev/null
+++ b/gm/rebaseline_server/testdata/inputs/gm-expectations/ignored-tests.txt
@@ -0,0 +1,10 @@
+# Failures of any GM tests listed in this file will be ignored [1], as
+# if they had been marked "ignore-failure": true in the per-builder
+# expected-results.json files.
+#
+
+blanket-ignored
+
+# "texdata" tests should NOT be ignored, since the next lines is commented out:
+# texdata
+
diff --git a/gm/rebaseline_server/testdata/outputs/expected/compare_configs_test.CompareConfigsTest.test_gm/gm.json b/gm/rebaseline_server/testdata/outputs/expected/compare_configs_test.CompareConfigsTest.test_gm/gm.json
index 6d63113..d98ac41 100644
--- a/gm/rebaseline_server/testdata/outputs/expected/compare_configs_test.CompareConfigsTest.test_gm/gm.json
+++ b/gm/rebaseline_server/testdata/outputs/expected/compare_configs_test.CompareConfigsTest.test_gm/gm.json
@@ -15,7 +15,7 @@
],
[
"Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug",
- 3
+ 4
]
]
},
@@ -26,7 +26,7 @@
"valuesAndCounts": [
[
"TODO",
- 13
+ 14
]
]
},
@@ -41,7 +41,7 @@
],
[
"no-comparison",
- 7
+ 8
],
[
"succeeded",
@@ -71,6 +71,10 @@
1
],
[
+ "blanket-ignored",
+ 1
+ ],
+ [
"displacement",
2
],
@@ -90,7 +94,7 @@
}
},
"header": {
- "dataHash": "-7043844904261310530",
+ "dataHash": "-3232866296817265056",
"isEditable": false,
"isExported": true,
"schemaVersion": 3,
@@ -232,6 +236,17 @@
"isDifferent": false
},
{
+ "extraColumns": {
+ "builder": "Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug",
+ "config": "TODO",
+ "resultType": "no-comparison",
+ "test": "blanket-ignored"
+ },
+ "imageAUrl": "bitmap-64bitMD5/blanket-ignored/22222222.png",
+ "imageBUrl": null,
+ "isDifferent": true
+ },
+ {
"differenceData": {
"maxDiffPerChannel": [
136,
diff --git a/gm/rebaseline_server/testdata/outputs/expected/compare_to_expectations_test.CompareToExpectationsTest.test_gm/gm.json b/gm/rebaseline_server/testdata/outputs/expected/compare_to_expectations_test.CompareToExpectationsTest.test_gm/gm.json
index 4f4eef7..9ad771e 100644
--- a/gm/rebaseline_server/testdata/outputs/expected/compare_to_expectations_test.CompareToExpectationsTest.test_gm/gm.json
+++ b/gm/rebaseline_server/testdata/outputs/expected/compare_to_expectations_test.CompareToExpectationsTest.test_gm/gm.json
@@ -11,7 +11,7 @@
],
[
"Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug",
- 15
+ 17
]
]
},
@@ -22,11 +22,11 @@
"valuesAndCounts": [
[
"565",
- 9
+ 10
],
[
"8888",
- 9
+ 10
],
[
"gpu",
@@ -53,7 +53,7 @@
],
[
"failure-ignored",
- 4
+ 6
],
[
"no-comparison",
@@ -87,6 +87,10 @@
2
],
[
+ "blanket-ignored",
+ 2
+ ],
+ [
"displacement",
5
],
@@ -106,7 +110,7 @@
}
},
"header": {
- "dataHash": "2775016045957284034",
+ "dataHash": "-7988419419043310401",
"isEditable": false,
"isExported": true,
"schemaVersion": 3,
@@ -379,6 +383,38 @@
"extraColumns": {
"builder": "Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug",
"config": "565",
+ "resultType": "failure-ignored",
+ "test": "blanket-ignored"
+ },
+ "imageAUrl": "bitmap-64bitMD5/blanket-ignored/111111111.png",
+ "imageBUrl": "bitmap-64bitMD5/blanket-ignored/22222222.png",
+ "isDifferent": true
+ },
+ {
+ "expectations": {
+ "bugs": null,
+ "ignore-failure": false,
+ "reviewed-by-human": null
+ },
+ "extraColumns": {
+ "builder": "Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug",
+ "config": "8888",
+ "resultType": "failure-ignored",
+ "test": "blanket-ignored"
+ },
+ "imageAUrl": "bitmap-64bitMD5/blanket-ignored/111111111.png",
+ "imageBUrl": "bitmap-64bitMD5/blanket-ignored/22222222.png",
+ "isDifferent": true
+ },
+ {
+ "expectations": {
+ "bugs": null,
+ "ignore-failure": false,
+ "reviewed-by-human": null
+ },
+ "extraColumns": {
+ "builder": "Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug",
+ "config": "565",
"resultType": "succeeded",
"test": "displacement"
},