rebaseline_server: ignore builders we don't have any expectations for
(SkipBuildbotRuns)
BUG=skia:2486
NOTREECHECKS=True
NOTRY=True
R=rmistry@google.com
TBR=rmistry
Author: epoger@google.com
Review URL: https://codereview.chromium.org/269543002
git-svn-id: http://skia.googlecode.com/svn/trunk@14488 2bbb7eff-a529-9590-31e7-b0007b416f81
diff --git a/gm/rebaseline_server/compare_to_expectations.py b/gm/rebaseline_server/compare_to_expectations.py
index 2389b61..97286b9 100755
--- a/gm/rebaseline_server/compare_to_expectations.py
+++ b/gm/rebaseline_server/compare_to_expectations.py
@@ -226,7 +226,14 @@
results.KEY__RESULT_TYPE__NOCOMPARISON,
])
- builders = sorted(actual_builder_dicts.keys())
+ # Only consider builders we have both expected and actual results for.
+ # Fixes http://skbug.com/2486 ('rebaseline_server shows actual results
+ # (but not expectations) for Test-Ubuntu12-ShuttleA-NoGPU-x86_64-Debug
+ # builder')
+ actual_builder_set = set(actual_builder_dicts.keys())
+ expected_builder_set = set(expected_builder_dicts.keys())
+ builders = sorted(actual_builder_set.intersection(expected_builder_set))
+
num_builders = len(builders)
builder_num = 0
for builder in builders:
diff --git a/gm/rebaseline_server/testdata/inputs/gm-actuals/Test-Builder-We-Have-No-Expectations-File-For/actual-results.json b/gm/rebaseline_server/testdata/inputs/gm-actuals/Test-Builder-We-Have-No-Expectations-File-For/actual-results.json
new file mode 100644
index 0000000..0a1de74
--- /dev/null
+++ b/gm/rebaseline_server/testdata/inputs/gm-actuals/Test-Builder-We-Have-No-Expectations-File-For/actual-results.json
@@ -0,0 +1,88 @@
+{
+ "actual-results" : {
+ "failed" : null,
+ "failure-ignored" : {
+ "displacement_565.png" : [ "bitmap-64bitMD5", 4569468668668628191 ],
+ "displacement_8888.png" : [ "bitmap-64bitMD5", 11401048196735046263 ],
+ "displacement_gpu.png" : [ "bitmap-64bitMD5", 5698561127291561694 ],
+ "displacement_pdf-mac.png" : [ "bitmap-64bitMD5", 12901125495691049846 ],
+ "displacement_pdf-poppler.png" : [ "bitmap-64bitMD5", 16285974094717334658 ]
+ },
+ "no-comparison" : {
+ "bigblurs_565.png" : [ "bitmap-64bitMD5", 14704206703218007573 ],
+ "bigblurs_8888.png" : [ "bitmap-64bitMD5", 17309852422285247848 ],
+ "bigblurs_gpu.png" : [ "bitmap-64bitMD5", 1822195599289208664 ],
+ "bigblurs_pdf-mac.png" : [ "bitmap-64bitMD5", 16171608477794909861 ],
+ "bigblurs_pdf-poppler.png" : [ "bitmap-64bitMD5", 6539050160610613353 ]
+ },
+ "succeeded" : {
+ "3x3bitmaprect_565.png" : [ "bitmap-64bitMD5", 16998423976396106083 ],
+ "3x3bitmaprect_8888.png" : [ "bitmap-64bitMD5", 2054956815327187963 ],
+ "3x3bitmaprect_gpu.png" : [ "bitmap-64bitMD5", 2054956815327187963 ],
+ "3x3bitmaprect_pdf-mac.png" : [ "bitmap-64bitMD5", 8518347971308375604 ],
+ "3x3bitmaprect_pdf-poppler.png" : [ "bitmap-64bitMD5", 16723580409414313678 ]
+ }
+ },
+ "expected-results" : {
+ "3x3bitmaprect_565.png" : {
+ "allowed-digests" : [
+ [ "bitmap-64bitMD5", 16998423976396106083 ]
+ ],
+ "ignore-failure" : false
+ },
+ "3x3bitmaprect_8888.png" : {
+ "allowed-digests" : [
+ [ "bitmap-64bitMD5", 2054956815327187963 ]
+ ],
+ "ignore-failure" : false
+ },
+ "3x3bitmaprect_gpu.png" : {
+ "allowed-digests" : [
+ [ "bitmap-64bitMD5", 2054956815327187963 ]
+ ],
+ "ignore-failure" : false
+ },
+ "3x3bitmaprect_pdf-mac.png" : {
+ "allowed-digests" : [
+ [ "bitmap-64bitMD5", 8518347971308375604 ]
+ ],
+ "ignore-failure" : false
+ },
+ "3x3bitmaprect_pdf-poppler.png" : {
+ "allowed-digests" : [
+ [ "bitmap-64bitMD5", 16723580409414313678 ]
+ ],
+ "ignore-failure" : false
+ },
+ "displacement_565.png" : {
+ "allowed-digests" : [
+ [ "bitmap-64bitMD5", 16249664097236120848 ]
+ ],
+ "ignore-failure" : true
+ },
+ "displacement_8888.png" : {
+ "allowed-digests" : [
+ [ "bitmap-64bitMD5", 12120271618448785903 ]
+ ],
+ "ignore-failure" : true
+ },
+ "displacement_gpu.png" : {
+ "allowed-digests" : [
+ [ "bitmap-64bitMD5", 14033538044845358121 ]
+ ],
+ "ignore-failure" : true
+ },
+ "displacement_pdf-mac.png" : {
+ "allowed-digests" : [
+ [ "bitmap-64bitMD5", 10149713146651176710 ]
+ ],
+ "ignore-failure" : true
+ },
+ "displacement_pdf-poppler.png" : {
+ "allowed-digests" : [
+ [ "bitmap-64bitMD5", 4741823272437589681 ]
+ ],
+ "ignore-failure" : true
+ }
+ }
+}
diff --git a/gm/rebaseline_server/testdata/outputs/expected/compare_configs_test.CompareConfigsTest.test_gm/gm.json b/gm/rebaseline_server/testdata/outputs/expected/compare_configs_test.CompareConfigsTest.test_gm/gm.json
index c8fbcf9..aabbdbf 100644
--- a/gm/rebaseline_server/testdata/outputs/expected/compare_configs_test.CompareConfigsTest.test_gm/gm.json
+++ b/gm/rebaseline_server/testdata/outputs/expected/compare_configs_test.CompareConfigsTest.test_gm/gm.json
@@ -6,6 +6,7 @@
"isSortable": true,
"valuesAndCounts": {
"Test-Android-GalaxyNexus-SGX540-Arm7-Release": 7,
+ "Test-Builder-We-Have-No-Expectations-File-For": 3,
"Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug": 3
}
},
@@ -14,7 +15,7 @@
"isFilterable": true,
"isSortable": true,
"valuesAndCounts": {
- "TODO": 10
+ "TODO": 13
}
},
"resultType": {
@@ -22,9 +23,9 @@
"isFilterable": true,
"isSortable": true,
"valuesAndCounts": {
- "failed": 2,
+ "failed": 4,
"no-comparison": 7,
- "succeeded": 1
+ "succeeded": 2
}
},
"test": {
@@ -32,11 +33,11 @@
"isFilterable": true,
"isSortable": true,
"valuesAndCounts": {
- "3x3bitmaprect": 2,
+ "3x3bitmaprect": 3,
"aaclip": 1,
- "bigblurs": 2,
+ "bigblurs": 3,
"bitmapsource": 1,
- "displacement": 1,
+ "displacement": 2,
"filterbitmap_checkerboard_192_192": 1,
"filterbitmap_checkerboard_32_2": 1,
"texdata": 1
@@ -44,7 +45,7 @@
}
},
"header": {
- "dataHash": "7627783634950094306",
+ "dataHash": "-7043844904261310530",
"isEditable": false,
"isExported": true,
"schemaVersion": 2,
@@ -143,6 +144,61 @@
"weightedDiffMeasure": 0.06644571780084584
},
"extraColumns": {
+ "builder": "Test-Builder-We-Have-No-Expectations-File-For",
+ "config": "TODO",
+ "resultType": "failed",
+ "test": "displacement"
+ },
+ "imageAUrl": "bitmap-64bitMD5/displacement/11401048196735046263.png",
+ "imageBUrl": "bitmap-64bitMD5/displacement/5698561127291561694.png",
+ "isDifferent": true
+ },
+ {
+ "differenceData": {
+ "maxDiffPerChannel": [
+ 255,
+ 221,
+ 221
+ ],
+ "numDifferingPixels": 50097,
+ "percentDifferingPixels": 30.5767822265625,
+ "perceptualDifference": 3.3917,
+ "weightedDiffMeasure": 1.4826590790651433
+ },
+ "extraColumns": {
+ "builder": "Test-Builder-We-Have-No-Expectations-File-For",
+ "config": "TODO",
+ "resultType": "failed",
+ "test": "bigblurs"
+ },
+ "imageAUrl": "bitmap-64bitMD5/bigblurs/17309852422285247848.png",
+ "imageBUrl": "bitmap-64bitMD5/bigblurs/1822195599289208664.png",
+ "isDifferent": true
+ },
+ {
+ "extraColumns": {
+ "builder": "Test-Builder-We-Have-No-Expectations-File-For",
+ "config": "TODO",
+ "resultType": "succeeded",
+ "test": "3x3bitmaprect"
+ },
+ "imageAUrl": "bitmap-64bitMD5/3x3bitmaprect/2054956815327187963.png",
+ "imageBUrl": "bitmap-64bitMD5/3x3bitmaprect/2054956815327187963.png",
+ "isDifferent": false
+ },
+ {
+ "differenceData": {
+ "maxDiffPerChannel": [
+ 136,
+ 68,
+ 34
+ ],
+ "numDifferingPixels": 6081,
+ "percentDifferingPixels": 2.4324,
+ "perceptualDifference": 1.917199999999994,
+ "weightedDiffMeasure": 0.06644571780084584
+ },
+ "extraColumns": {
"builder": "Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug",
"config": "TODO",
"resultType": "failed",