Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 1 | # Copyright (C) 2010 Google Inc. All rights reserved. |
| 2 | # |
| 3 | # Redistribution and use in source and binary forms, with or without |
| 4 | # modification, are permitted provided that the following conditions are |
| 5 | # met: |
| 6 | # |
| 7 | # * Redistributions of source code must retain the above copyright |
| 8 | # notice, this list of conditions and the following disclaimer. |
| 9 | # * Redistributions in binary form must reproduce the above |
| 10 | # copyright notice, this list of conditions and the following disclaimer |
| 11 | # in the documentation and/or other materials provided with the |
| 12 | # distribution. |
| 13 | # * Neither the name of Google Inc. nor the names of its |
| 14 | # contributors may be used to endorse or promote products derived from |
| 15 | # this software without specific prior written permission. |
| 16 | # |
| 17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 18 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 19 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 20 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 21 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 22 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 23 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 24 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 25 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 26 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 28 | |
Torne (Richard Coles) | 926b001 | 2013-03-28 15:32:48 +0000 | [diff] [blame] | 29 | import unittest2 as unittest |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 30 | |
| 31 | from webkitpy.common.system.outputcapture import OutputCapture |
| 32 | from webkitpy.common.checkout.baselineoptimizer import BaselineOptimizer |
| 33 | from webkitpy.common.net.buildbot.buildbot_mock import MockBuilder |
| 34 | from webkitpy.common.system.executive_mock import MockExecutive2 |
| 35 | from webkitpy.thirdparty.mock import Mock |
| 36 | from webkitpy.tool.commands.rebaseline import * |
| 37 | from webkitpy.tool.mocktool import MockTool, MockOptions |
| 38 | |
| 39 | |
| 40 | class _BaseTestCase(unittest.TestCase): |
| 41 | MOCK_WEB_RESULT = 'MOCK Web result, convert 404 to None=True' |
| 42 | WEB_PREFIX = 'http://example.com/f/builders/WebKit Mac10.7/results/layout-test-results' |
| 43 | |
| 44 | command_constructor = None |
| 45 | |
| 46 | def setUp(self): |
| 47 | self.tool = MockTool() |
Torne (Richard Coles) | 926b001 | 2013-03-28 15:32:48 +0000 | [diff] [blame] | 48 | self.command = self.command_constructor() # lint warns that command_constructor might not be set, but this is intentional; pylint: disable=E1102 |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 49 | self.command.bind_to_tool(self.tool) |
| 50 | self.lion_port = self.tool.port_factory.get_from_builder_name("WebKit Mac10.7") |
Torne (Richard Coles) | 53e740f | 2013-05-09 18:38:43 +0100 | [diff] [blame] | 51 | self.lion_expectations_path = self.lion_port.path_to_generic_test_expectations_file() |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 52 | |
| 53 | # FIXME: we should override builders._exact_matches here to point to a set |
| 54 | # of test ports and restore the value in tearDown(), and that way the |
| 55 | # individual tests wouldn't have to worry about it. |
| 56 | |
| 57 | def _expand(self, path): |
| 58 | if self.tool.filesystem.isabs(path): |
| 59 | return path |
| 60 | return self.tool.filesystem.join(self.lion_port.layout_tests_dir(), path) |
| 61 | |
| 62 | def _read(self, path): |
| 63 | return self.tool.filesystem.read_text_file(self._expand(path)) |
| 64 | |
| 65 | def _write(self, path, contents): |
| 66 | self.tool.filesystem.write_text_file(self._expand(path), contents) |
| 67 | |
| 68 | def _zero_out_test_expectations(self): |
| 69 | for port_name in self.tool.port_factory.all_port_names(): |
| 70 | port = self.tool.port_factory.get(port_name) |
| 71 | for path in port.expectations_files(): |
| 72 | self._write(path, '') |
| 73 | self.tool.filesystem.written_files = {} |
| 74 | |
| 75 | |
| 76 | class TestRebaselineTest(_BaseTestCase): |
| 77 | command_constructor = RebaselineTest # AKA webkit-patch rebaseline-test-internal |
| 78 | |
| 79 | def setUp(self): |
| 80 | super(TestRebaselineTest, self).setUp() |
| 81 | self.options = MockOptions(builder="WebKit Mac10.7", test="userscripts/another-test.html", suffixes="txt", |
| 82 | move_overwritten_baselines_to=None, results_directory=None) |
| 83 | |
| 84 | def test_baseline_directory(self): |
| 85 | command = self.command |
Torne (Richard Coles) | 926b001 | 2013-03-28 15:32:48 +0000 | [diff] [blame] | 86 | self.assertMultiLineEqual(command._baseline_directory("WebKit Mac10.7"), "/mock-checkout/LayoutTests/platform/chromium-mac-lion") |
| 87 | self.assertMultiLineEqual(command._baseline_directory("WebKit Mac10.6"), "/mock-checkout/LayoutTests/platform/chromium-mac-snowleopard") |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 88 | |
| 89 | def test_rebaseline_updates_expectations_file_noop(self): |
| 90 | self._zero_out_test_expectations() |
| 91 | self._write(self.lion_expectations_path, """Bug(B) [ Mac Linux XP Debug ] fast/dom/Window/window-postmessage-clone-really-deep-array.html [ Pass ] |
| 92 | Bug(A) [ Debug ] : fast/css/large-list-of-rules-crash.html [ Failure ] |
| 93 | """) |
| 94 | self._write("fast/dom/Window/window-postmessage-clone-really-deep-array.html", "Dummy test contents") |
| 95 | self._write("fast/css/large-list-of-rules-crash.html", "Dummy test contents") |
| 96 | self._write("userscripts/another-test.html", "Dummy test contents") |
| 97 | |
| 98 | self.options.suffixes = "png,wav,txt" |
| 99 | self.command._rebaseline_test_and_update_expectations(self.options) |
| 100 | |
Torne (Richard Coles) | 926b001 | 2013-03-28 15:32:48 +0000 | [diff] [blame] | 101 | self.assertItemsEqual(self.tool.web.urls_fetched, |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 102 | [self.WEB_PREFIX + '/userscripts/another-test-actual.png', |
| 103 | self.WEB_PREFIX + '/userscripts/another-test-actual.wav', |
| 104 | self.WEB_PREFIX + '/userscripts/another-test-actual.txt']) |
| 105 | new_expectations = self._read(self.lion_expectations_path) |
Torne (Richard Coles) | 926b001 | 2013-03-28 15:32:48 +0000 | [diff] [blame] | 106 | self.assertMultiLineEqual(new_expectations, """Bug(B) [ Mac Linux XP Debug ] fast/dom/Window/window-postmessage-clone-really-deep-array.html [ Pass ] |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 107 | Bug(A) [ Debug ] : fast/css/large-list-of-rules-crash.html [ Failure ] |
| 108 | """) |
| 109 | |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 110 | def test_rebaseline_test(self): |
| 111 | self.command._rebaseline_test("WebKit Linux", "userscripts/another-test.html", None, "txt", self.WEB_PREFIX) |
Torne (Richard Coles) | 926b001 | 2013-03-28 15:32:48 +0000 | [diff] [blame] | 112 | self.assertItemsEqual(self.tool.web.urls_fetched, [self.WEB_PREFIX + '/userscripts/another-test-actual.txt']) |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 113 | |
| 114 | def test_rebaseline_test_with_results_directory(self): |
Torne (Richard Coles) | 81a5157 | 2013-05-13 16:52:28 +0100 | [diff] [blame] | 115 | self._write("userscripts/another-test.html", "test data") |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 116 | self._write(self.lion_expectations_path, "Bug(x) [ Mac ] userscripts/another-test.html [ ImageOnlyFailure ]\nbug(z) [ Linux ] userscripts/another-test.html [ ImageOnlyFailure ]\n") |
| 117 | self.options.results_directory = '/tmp' |
| 118 | self.command._rebaseline_test_and_update_expectations(self.options) |
Torne (Richard Coles) | 926b001 | 2013-03-28 15:32:48 +0000 | [diff] [blame] | 119 | self.assertItemsEqual(self.tool.web.urls_fetched, ['file:///tmp/userscripts/another-test-actual.txt']) |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 120 | |
Torne (Richard Coles) | 53e740f | 2013-05-09 18:38:43 +0100 | [diff] [blame] | 121 | def test_rebaseline_reftest(self): |
Torne (Richard Coles) | 81a5157 | 2013-05-13 16:52:28 +0100 | [diff] [blame] | 122 | self._write("userscripts/another-test.html", "test data") |
Torne (Richard Coles) | 53e740f | 2013-05-09 18:38:43 +0100 | [diff] [blame] | 123 | self._write("userscripts/another-test-expected.html", "generic result") |
| 124 | OutputCapture().assert_outputs(self, self.command._rebaseline_test_and_update_expectations, args=[self.options], |
| 125 | expected_logs="Cannot rebaseline reftest: userscripts/another-test.html\n") |
Torne (Richard Coles) | 81a5157 | 2013-05-13 16:52:28 +0100 | [diff] [blame] | 126 | self.assertDictEqual(self.command._scm_changes, {'add': [], 'remove-lines': []}) |
Torne (Richard Coles) | 53e740f | 2013-05-09 18:38:43 +0100 | [diff] [blame] | 127 | |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 128 | def test_rebaseline_test_and_print_scm_changes(self): |
| 129 | self.command._print_scm_changes = True |
| 130 | self.command._scm_changes = {'add': [], 'delete': []} |
| 131 | self.tool._scm.exists = lambda x: False |
| 132 | |
| 133 | self.command._rebaseline_test("WebKit Linux", "userscripts/another-test.html", None, "txt", None) |
| 134 | |
Torne (Richard Coles) | 926b001 | 2013-03-28 15:32:48 +0000 | [diff] [blame] | 135 | self.assertDictEqual(self.command._scm_changes, {'add': ['/mock-checkout/LayoutTests/platform/chromium-linux/userscripts/another-test-expected.txt'], 'delete': []}) |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 136 | |
| 137 | def test_rebaseline_and_copy_test(self): |
| 138 | self._write("userscripts/another-test-expected.txt", "generic result") |
| 139 | |
| 140 | self.command._rebaseline_test("WebKit Mac10.7", "userscripts/another-test.html", ["chromium-mac-snowleopard"], "txt", None) |
| 141 | |
Torne (Richard Coles) | 926b001 | 2013-03-28 15:32:48 +0000 | [diff] [blame] | 142 | self.assertMultiLineEqual(self._read('platform/chromium-mac-lion/userscripts/another-test-expected.txt'), self.MOCK_WEB_RESULT) |
| 143 | self.assertMultiLineEqual(self._read('platform/chromium-mac-snowleopard/userscripts/another-test-expected.txt'), 'generic result') |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 144 | |
| 145 | def test_rebaseline_and_copy_test_no_existing_result(self): |
| 146 | self.command._rebaseline_test("WebKit Mac10.7", "userscripts/another-test.html", ["chromium-mac-snowleopard"], "txt", None) |
| 147 | |
Torne (Richard Coles) | 926b001 | 2013-03-28 15:32:48 +0000 | [diff] [blame] | 148 | self.assertMultiLineEqual(self._read('platform/chromium-mac-lion/userscripts/another-test-expected.txt'), self.MOCK_WEB_RESULT) |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 149 | self.assertFalse(self.tool.filesystem.exists(self._expand('platform/chromium-mac-snowleopard/userscripts/another-test-expected.txt'))) |
| 150 | |
| 151 | def test_rebaseline_and_copy_test_with_lion_result(self): |
| 152 | self._write("platform/chromium-mac-lion/userscripts/another-test-expected.txt", "original lion result") |
| 153 | |
| 154 | self.command._rebaseline_test("WebKit Mac10.7", "userscripts/another-test.html", ["chromium-mac-snowleopard"], "txt", self.WEB_PREFIX) |
| 155 | |
Torne (Richard Coles) | 926b001 | 2013-03-28 15:32:48 +0000 | [diff] [blame] | 156 | self.assertItemsEqual(self.tool.web.urls_fetched, [self.WEB_PREFIX + '/userscripts/another-test-actual.txt']) |
| 157 | self.assertMultiLineEqual(self._read("platform/chromium-mac-snowleopard/userscripts/another-test-expected.txt"), "original lion result") |
| 158 | self.assertMultiLineEqual(self._read("platform/chromium-mac-lion/userscripts/another-test-expected.txt"), self.MOCK_WEB_RESULT) |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 159 | |
| 160 | def test_rebaseline_and_copy_no_overwrite_test(self): |
| 161 | self._write("platform/chromium-mac-lion/userscripts/another-test-expected.txt", "original lion result") |
| 162 | self._write("platform/chromium-mac-snowleopard/userscripts/another-test-expected.txt", "original snowleopard result") |
| 163 | |
| 164 | self.command._rebaseline_test("WebKit Mac10.7", "userscripts/another-test.html", ["chromium-mac-snowleopard"], "txt", None) |
| 165 | |
Torne (Richard Coles) | 926b001 | 2013-03-28 15:32:48 +0000 | [diff] [blame] | 166 | self.assertMultiLineEqual(self._read("platform/chromium-mac-snowleopard/userscripts/another-test-expected.txt"), "original snowleopard result") |
| 167 | self.assertMultiLineEqual(self._read("platform/chromium-mac-lion/userscripts/another-test-expected.txt"), self.MOCK_WEB_RESULT) |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 168 | |
Torne (Richard Coles) | 53e740f | 2013-05-09 18:38:43 +0100 | [diff] [blame] | 169 | def test_rebaseline_test_internal_with_copying_overwritten_baseline_first(self): |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 170 | self.tool.executive = MockExecutive2() |
| 171 | |
| 172 | # FIXME: it's confusing that this is the test- port, and not the regular lion port. Really all of the tests should be using the test ports. |
| 173 | port = self.tool.port_factory.get('test-mac-snowleopard') |
| 174 | self._write(port._filesystem.join(port.layout_tests_dir(), 'platform/test-mac-snowleopard/failures/expected/image-expected.txt'), 'original snowleopard result') |
| 175 | |
| 176 | old_exact_matches = builders._exact_matches |
| 177 | oc = OutputCapture() |
| 178 | try: |
| 179 | builders._exact_matches = { |
| 180 | "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, |
| 181 | "MOCK SnowLeopard": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])}, |
| 182 | } |
| 183 | |
| 184 | options = MockOptions(optimize=True, builder="MOCK SnowLeopard", suffixes="txt", |
Torne (Richard Coles) | 53e740f | 2013-05-09 18:38:43 +0100 | [diff] [blame] | 185 | move_overwritten_baselines_to=None, verbose=True, test="failures/expected/image.html", |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 186 | results_directory=None) |
| 187 | |
| 188 | oc.capture_output() |
| 189 | self.command.execute(options, [], self.tool) |
| 190 | finally: |
| 191 | out, _, _ = oc.restore_output() |
| 192 | builders._exact_matches = old_exact_matches |
| 193 | |
Torne (Richard Coles) | 926b001 | 2013-03-28 15:32:48 +0000 | [diff] [blame] | 194 | self.assertMultiLineEqual(self._read(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/test-mac-leopard/failures/expected/image-expected.txt')), 'original snowleopard result') |
Torne (Richard Coles) | 53e740f | 2013-05-09 18:38:43 +0100 | [diff] [blame] | 195 | self.assertMultiLineEqual(out, '{"add": [], "remove-lines": [{"test": "failures/expected/image.html", "builder": "MOCK SnowLeopard"}]}\n') |
| 196 | |
| 197 | def test_rebaseline_test_internal_with_copying_overwritten_baseline_first_to_multiple_locations(self): |
| 198 | self.tool.executive = MockExecutive2() |
| 199 | |
| 200 | # FIXME: it's confusing that this is the test- port, and not the regular win port. Really all of the tests should be using the test ports. |
| 201 | port = self.tool.port_factory.get('test-win-win7') |
| 202 | self._write(port._filesystem.join(port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/image-expected.txt'), 'original win7 result') |
| 203 | |
| 204 | old_exact_matches = builders._exact_matches |
| 205 | oc = OutputCapture() |
| 206 | try: |
| 207 | builders._exact_matches = { |
| 208 | "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, |
| 209 | "MOCK Linux": {"port_name": "test-linux-x86_64", "specifiers": set(["mock-specifier"])}, |
| 210 | "MOCK Vista": {"port_name": "test-win-vista", "specifiers": set(["mock-specifier"])}, |
| 211 | "MOCK Win7": {"port_name": "test-win-win7", "specifiers": set(["mock-specifier"])}, |
| 212 | } |
| 213 | |
| 214 | options = MockOptions(optimize=True, builder="MOCK Win7", suffixes="txt", |
| 215 | move_overwritten_baselines_to=None, verbose=True, test="failures/expected/image.html", |
| 216 | results_directory=None) |
| 217 | |
| 218 | oc.capture_output() |
| 219 | self.command.execute(options, [], self.tool) |
| 220 | finally: |
| 221 | out, _, _ = oc.restore_output() |
| 222 | builders._exact_matches = old_exact_matches |
| 223 | |
| 224 | self.assertMultiLineEqual(self._read(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/test-linux-x86_64/failures/expected/image-expected.txt')), 'original win7 result') |
| 225 | self.assertMultiLineEqual(self._read(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/test-win-vista/failures/expected/image-expected.txt')), 'original win7 result') |
| 226 | self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/chromium-mac-leopard/userscripts/another-test-expected.txt'))) |
| 227 | self.assertMultiLineEqual(out, '{"add": [], "remove-lines": [{"test": "failures/expected/image.html", "builder": "MOCK Win7"}]}\n') |
| 228 | |
| 229 | def test_rebaseline_test_internal_with_no_overwrite_existing_baseline(self): |
| 230 | self.tool.executive = MockExecutive2() |
| 231 | |
| 232 | # FIXME: it's confusing that this is the test- port, and not the regular win port. Really all of the tests should be using the test ports. |
| 233 | port = self.tool.port_factory.get('test-win-win7') |
| 234 | self._write(port._filesystem.join(port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/image-expected.txt'), 'original win7 result') |
| 235 | self._write(port._filesystem.join(port.layout_tests_dir(), 'platform/test-win-vista/failures/expected/image-expected.txt'), 'original vista result') |
| 236 | |
| 237 | old_exact_matches = builders._exact_matches |
| 238 | oc = OutputCapture() |
| 239 | try: |
| 240 | builders._exact_matches = { |
| 241 | "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, |
| 242 | "MOCK Linux": {"port_name": "test-linux-x86_64", "specifiers": set(["mock-specifier"])}, |
| 243 | "MOCK Vista": {"port_name": "test-win-vista", "specifiers": set(["mock-specifier"])}, |
| 244 | "MOCK Win7": {"port_name": "test-win-win7", "specifiers": set(["mock-specifier"])}, |
| 245 | } |
| 246 | |
| 247 | options = MockOptions(optimize=True, builder="MOCK Win7", suffixes="txt", |
| 248 | move_overwritten_baselines_to=None, verbose=True, test="failures/expected/image.html", |
| 249 | results_directory=None) |
| 250 | |
| 251 | oc.capture_output() |
| 252 | self.command.execute(options, [], self.tool) |
| 253 | finally: |
| 254 | out, _, _ = oc.restore_output() |
| 255 | builders._exact_matches = old_exact_matches |
| 256 | |
| 257 | self.assertMultiLineEqual(self._read(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/test-linux-x86_64/failures/expected/image-expected.txt')), 'original win7 result') |
| 258 | self.assertMultiLineEqual(self._read(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/test-win-vista/failures/expected/image-expected.txt')), 'original vista result') |
| 259 | self.assertMultiLineEqual(self._read(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/image-expected.txt')), 'MOCK Web result, convert 404 to None=True') |
| 260 | self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/chromium-mac-leopard/userscripts/another-test-expected.txt'))) |
| 261 | self.assertMultiLineEqual(out, '{"add": [], "remove-lines": [{"test": "failures/expected/image.html", "builder": "MOCK Win7"}]}\n') |
| 262 | |
| 263 | def test_rebaseline_test_internal_with_port_that_lacks_buildbot(self): |
| 264 | self.tool.executive = MockExecutive2() |
| 265 | |
| 266 | # FIXME: it's confusing that this is the test- port, and not the regular win port. Really all of the tests should be using the test ports. |
| 267 | port = self.tool.port_factory.get('test-win-vista') |
| 268 | self._write(port._filesystem.join(port.layout_tests_dir(), 'platform/test-win-vista/failures/expected/image-expected.txt'), 'original vista result') |
| 269 | |
| 270 | old_exact_matches = builders._exact_matches |
| 271 | oc = OutputCapture() |
| 272 | try: |
| 273 | builders._exact_matches = { |
| 274 | "MOCK XP": {"port_name": "test-win-xp"}, |
| 275 | "MOCK Vista": {"port_name": "test-win-vista"}, |
| 276 | } |
| 277 | |
| 278 | options = MockOptions(optimize=True, builder="MOCK Vista", suffixes="txt", |
| 279 | move_overwritten_baselines_to=None, verbose=True, test="failures/expected/image.html", |
| 280 | results_directory=None) |
| 281 | |
| 282 | oc.capture_output() |
| 283 | self.command.execute(options, [], self.tool) |
| 284 | finally: |
| 285 | out, _, _ = oc.restore_output() |
| 286 | builders._exact_matches = old_exact_matches |
| 287 | |
| 288 | self.assertMultiLineEqual(self._read(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/test-win-vista/failures/expected/image-expected.txt')), 'MOCK Web result, convert 404 to None=True') |
| 289 | self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/test-win-xp/failures/expected/image-expected.txt'))) |
| 290 | self.assertMultiLineEqual(out, '{"add": [], "remove-lines": [{"test": "failures/expected/image.html", "builder": "MOCK Vista"}]}\n') |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 291 | |
| 292 | |
| 293 | class TestRebaselineJson(_BaseTestCase): |
| 294 | command_constructor = RebaselineJson |
| 295 | |
| 296 | def setUp(self): |
| 297 | super(TestRebaselineJson, self).setUp() |
| 298 | self.tool.executive = MockExecutive2() |
| 299 | self.old_exact_matches = builders._exact_matches |
| 300 | builders._exact_matches = { |
| 301 | "MOCK builder": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"]), |
| 302 | "move_overwritten_baselines_to": ["test-mac-leopard"]}, |
| 303 | "MOCK builder (Debug)": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier", "debug"])}, |
| 304 | } |
| 305 | |
| 306 | def tearDown(self): |
| 307 | builders._exact_matches = self.old_exact_matches |
| 308 | super(TestRebaselineJson, self).tearDown() |
| 309 | |
| 310 | def test_rebaseline_all(self): |
| 311 | options = MockOptions(optimize=True, verbose=True, move_overwritten_baselines=False, results_directory=None) |
Torne (Richard Coles) | 81a5157 | 2013-05-13 16:52:28 +0100 | [diff] [blame] | 312 | self._write("user-scripts/another-test.html", "Dummy test contents") |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 313 | self.command._rebaseline(options, {"user-scripts/another-test.html": {"MOCK builder": ["txt", "png"]}}) |
| 314 | |
| 315 | # Note that we have one run_in_parallel() call followed by a run_command() |
Torne (Richard Coles) | 926b001 | 2013-03-28 15:32:48 +0000 | [diff] [blame] | 316 | self.assertEqual(self.tool.executive.calls, |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 317 | [[['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'user-scripts/another-test.html', '--verbose']], |
| 318 | ['echo', '--verbose', 'optimize-baselines', '--suffixes', 'txt,png', 'user-scripts/another-test.html']]) |
| 319 | |
| 320 | def test_rebaseline_debug(self): |
| 321 | options = MockOptions(optimize=True, verbose=True, move_overwritten_baselines=False, results_directory=None) |
Torne (Richard Coles) | 81a5157 | 2013-05-13 16:52:28 +0100 | [diff] [blame] | 322 | self._write("user-scripts/another-test.html", "Dummy test contents") |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 323 | self.command._rebaseline(options, {"user-scripts/another-test.html": {"MOCK builder (Debug)": ["txt", "png"]}}) |
| 324 | |
| 325 | # Note that we have one run_in_parallel() call followed by a run_command() |
Torne (Richard Coles) | 926b001 | 2013-03-28 15:32:48 +0000 | [diff] [blame] | 326 | self.assertEqual(self.tool.executive.calls, |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 327 | [[['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder (Debug)', '--test', 'user-scripts/another-test.html', '--verbose']], |
| 328 | ['echo', '--verbose', 'optimize-baselines', '--suffixes', 'txt,png', 'user-scripts/another-test.html']]) |
| 329 | |
| 330 | def test_move_overwritten(self): |
| 331 | options = MockOptions(optimize=True, verbose=True, move_overwritten_baselines=True, results_directory=None) |
Torne (Richard Coles) | 81a5157 | 2013-05-13 16:52:28 +0100 | [diff] [blame] | 332 | self._write("user-scripts/another-test.html", "Dummy test contents") |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 333 | self.command._rebaseline(options, {"user-scripts/another-test.html": {"MOCK builder": ["txt", "png"]}}) |
| 334 | |
| 335 | # Note that we have one run_in_parallel() call followed by a run_command() |
Torne (Richard Coles) | 926b001 | 2013-03-28 15:32:48 +0000 | [diff] [blame] | 336 | self.assertEqual(self.tool.executive.calls, |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 337 | [[['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'user-scripts/another-test.html', '--move-overwritten-baselines-to', 'test-mac-leopard', '--verbose']], |
| 338 | ['echo', '--verbose', 'optimize-baselines', '--suffixes', 'txt,png', 'user-scripts/another-test.html']]) |
| 339 | |
| 340 | def test_no_optimize(self): |
| 341 | options = MockOptions(optimize=False, verbose=True, move_overwritten_baselines=False, results_directory=None) |
Torne (Richard Coles) | 81a5157 | 2013-05-13 16:52:28 +0100 | [diff] [blame] | 342 | self._write("user-scripts/another-test.html", "Dummy test contents") |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 343 | self.command._rebaseline(options, {"user-scripts/another-test.html": {"MOCK builder (Debug)": ["txt", "png"]}}) |
| 344 | |
| 345 | # Note that we have only one run_in_parallel() call |
Torne (Richard Coles) | 926b001 | 2013-03-28 15:32:48 +0000 | [diff] [blame] | 346 | self.assertEqual(self.tool.executive.calls, |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 347 | [[['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder (Debug)', '--test', 'user-scripts/another-test.html', '--verbose']]]) |
| 348 | |
| 349 | def test_results_directory(self): |
| 350 | options = MockOptions(optimize=False, verbose=True, move_overwritten_baselines=False, results_directory='/tmp') |
Torne (Richard Coles) | 81a5157 | 2013-05-13 16:52:28 +0100 | [diff] [blame] | 351 | self._write("user-scripts/another-test.html", "Dummy test contents") |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 352 | self.command._rebaseline(options, {"user-scripts/another-test.html": {"MOCK builder": ["txt", "png"]}}) |
| 353 | |
| 354 | # Note that we have only one run_in_parallel() call |
Torne (Richard Coles) | 926b001 | 2013-03-28 15:32:48 +0000 | [diff] [blame] | 355 | self.assertEqual(self.tool.executive.calls, |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 356 | [[['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'user-scripts/another-test.html', '--results-directory', '/tmp', '--verbose']]]) |
| 357 | |
Torne (Richard Coles) | 53e740f | 2013-05-09 18:38:43 +0100 | [diff] [blame] | 358 | class TestRebaselineJsonUpdatesExpectationsFiles(_BaseTestCase): |
| 359 | command_constructor = RebaselineJson |
| 360 | |
| 361 | def setUp(self): |
| 362 | super(TestRebaselineJsonUpdatesExpectationsFiles, self).setUp() |
| 363 | self.tool.executive = MockExecutive2() |
| 364 | |
| 365 | def mock_run_command(args, |
| 366 | cwd=None, |
| 367 | input=None, |
| 368 | error_handler=None, |
| 369 | return_exit_code=False, |
| 370 | return_stderr=True, |
| 371 | decode_output=False, |
| 372 | env=None): |
| 373 | return '{"add": [], "remove-lines": [{"test": "userscripts/another-test.html", "builder": "WebKit Mac10.7"}]}\n' |
| 374 | self.tool.executive.run_command = mock_run_command |
| 375 | |
| 376 | def test_rebaseline_updates_expectations_file(self): |
| 377 | options = MockOptions(optimize=False, verbose=True, move_overwritten_baselines=False, results_directory=None) |
| 378 | |
| 379 | self._write(self.lion_expectations_path, "Bug(x) [ Mac ] userscripts/another-test.html [ ImageOnlyFailure ]\nbug(z) [ Linux ] userscripts/another-test.html [ ImageOnlyFailure ]\n") |
| 380 | self._write("userscripts/another-test.html", "Dummy test contents") |
| 381 | |
| 382 | self.command._rebaseline(options, {"userscripts/another-test.html": {"WebKit Mac10.7": ["txt", "png"]}}) |
| 383 | |
| 384 | new_expectations = self._read(self.lion_expectations_path) |
| 385 | self.assertMultiLineEqual(new_expectations, "Bug(x) [ MountainLion SnowLeopard ] userscripts/another-test.html [ ImageOnlyFailure ]\nbug(z) [ Linux ] userscripts/another-test.html [ ImageOnlyFailure ]\n") |
| 386 | |
| 387 | def test_rebaseline_updates_expectations_file_all_platforms(self): |
| 388 | options = MockOptions(optimize=False, verbose=True, move_overwritten_baselines=False, results_directory=None) |
| 389 | |
| 390 | self._write(self.lion_expectations_path, "Bug(x) userscripts/another-test.html [ ImageOnlyFailure ]\n") |
| 391 | self._write("userscripts/another-test.html", "Dummy test contents") |
| 392 | |
| 393 | self.command._rebaseline(options, {"userscripts/another-test.html": {"WebKit Mac10.7": ["txt", "png"]}}) |
| 394 | |
| 395 | new_expectations = self._read(self.lion_expectations_path) |
| 396 | self.assertMultiLineEqual(new_expectations, "Bug(x) [ Linux MountainLion SnowLeopard Win ] userscripts/another-test.html [ ImageOnlyFailure ]\n") |
| 397 | |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 398 | |
| 399 | class TestRebaseline(_BaseTestCase): |
| 400 | # This command shares most of its logic with RebaselineJson, so these tests just test what is different. |
| 401 | |
| 402 | command_constructor = Rebaseline # AKA webkit-patch rebaseline |
| 403 | |
| 404 | def test_tests_to_update(self): |
| 405 | build = Mock() |
| 406 | OutputCapture().assert_outputs(self, self.command._tests_to_update, [build]) |
| 407 | |
| 408 | def test_rebaseline(self): |
| 409 | self.command._builders_to_pull_from = lambda: [MockBuilder('MOCK builder')] |
| 410 | self.command._tests_to_update = lambda builder: ['mock/path/to/test.html'] |
| 411 | |
Torne (Richard Coles) | 81a5157 | 2013-05-13 16:52:28 +0100 | [diff] [blame] | 412 | self._write("mock/path/to/test.html", "Dummy test contents") |
| 413 | |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 414 | self._zero_out_test_expectations() |
| 415 | |
| 416 | old_exact_matches = builders._exact_matches |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 417 | try: |
| 418 | builders._exact_matches = { |
| 419 | "MOCK builder": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, |
| 420 | } |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 421 | self.command.execute(MockOptions(optimize=False, builders=None, suffixes="txt,png", verbose=True, move_overwritten_baselines=False), [], self.tool) |
| 422 | finally: |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 423 | builders._exact_matches = old_exact_matches |
| 424 | |
| 425 | calls = filter(lambda x: x != ['qmake', '-v'] and x[0] != 'perl', self.tool.executive.calls) |
Torne (Richard Coles) | 926b001 | 2013-03-28 15:32:48 +0000 | [diff] [blame] | 426 | self.assertEqual(calls, |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 427 | [[['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'mock/path/to/test.html', '--verbose']]]) |
| 428 | |
Torne (Richard Coles) | 81a5157 | 2013-05-13 16:52:28 +0100 | [diff] [blame] | 429 | def test_rebaseline_directory(self): |
| 430 | self.command._builders_to_pull_from = lambda: [MockBuilder('MOCK builder')] |
| 431 | self.command._tests_to_update = lambda builder: ['userscripts'] |
| 432 | |
| 433 | self._write("userscripts/first-test.html", "test data") |
| 434 | self._write("userscripts/second-test.html", "test data") |
| 435 | |
| 436 | old_exact_matches = builders._exact_matches |
| 437 | try: |
| 438 | builders._exact_matches = { |
| 439 | "MOCK builder": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, |
| 440 | } |
| 441 | self.command.execute(MockOptions(optimize=False, builders=None, suffixes="txt,png", verbose=True, move_overwritten_baselines=False), [], self.tool) |
| 442 | finally: |
| 443 | builders._exact_matches = old_exact_matches |
| 444 | |
| 445 | calls = filter(lambda x: x != ['qmake', '-v'] and x[0] != 'perl', self.tool.executive.calls) |
| 446 | self.assertEqual(calls, |
| 447 | [[['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--verbose'], |
| 448 | ['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/second-test.html', '--verbose']]]) |
| 449 | |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 450 | |
| 451 | class TestRebaselineExpectations(_BaseTestCase): |
| 452 | command_constructor = RebaselineExpectations |
| 453 | |
| 454 | def setUp(self): |
| 455 | super(TestRebaselineExpectations, self).setUp() |
| 456 | self.options = MockOptions(optimize=False, builders=None, suffixes=['txt'], verbose=False, platform=None, |
| 457 | move_overwritten_baselines=False, results_directory=None) |
| 458 | |
| 459 | def test_rebaseline_expectations(self): |
| 460 | self._zero_out_test_expectations() |
| 461 | |
| 462 | self.tool.executive = MockExecutive2() |
| 463 | |
Torne (Richard Coles) | 81a5157 | 2013-05-13 16:52:28 +0100 | [diff] [blame] | 464 | self._write("userscripts/another-test.html", "Dummy test contents") |
| 465 | self._write("userscripts/images.svg", "Dummy test contents") |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 466 | self.command._tests_to_rebaseline = lambda port: {'userscripts/another-test.html': set(['txt']), 'userscripts/images.svg': set(['png'])} |
| 467 | self.command.execute(self.options, [], self.tool) |
| 468 | |
| 469 | # FIXME: change this to use the test- ports. |
| 470 | calls = filter(lambda x: x != ['qmake', '-v'], self.tool.executive.calls) |
Torne (Richard Coles) | 926b001 | 2013-03-28 15:32:48 +0000 | [diff] [blame] | 471 | self.assertEqual(len(calls), 1) |
Torne (Richard Coles) | 53e740f | 2013-05-09 18:38:43 +0100 | [diff] [blame] | 472 | self.assertEqual(len(calls[0]), 14) |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 473 | |
| 474 | def test_rebaseline_expectations_noop(self): |
| 475 | self._zero_out_test_expectations() |
| 476 | |
| 477 | oc = OutputCapture() |
| 478 | try: |
| 479 | oc.capture_output() |
| 480 | self.command.execute(self.options, [], self.tool) |
| 481 | finally: |
| 482 | _, _, logs = oc.restore_output() |
Torne (Richard Coles) | 926b001 | 2013-03-28 15:32:48 +0000 | [diff] [blame] | 483 | self.assertEqual(self.tool.filesystem.written_files, {}) |
| 484 | self.assertEqual(logs, 'Did not find any tests marked Rebaseline.\n') |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 485 | |
| 486 | def disabled_test_overrides_are_included_correctly(self): |
| 487 | # This tests that the any tests marked as REBASELINE in the overrides are found, but |
| 488 | # that the overrides do not get written into the main file. |
| 489 | self._zero_out_test_expectations() |
| 490 | |
| 491 | self._write(self.lion_expectations_path, '') |
| 492 | self.lion_port.expectations_dict = lambda: { |
| 493 | self.lion_expectations_path: '', |
| 494 | 'overrides': ('Bug(x) userscripts/another-test.html [ Failure Rebaseline ]\n' |
| 495 | 'Bug(y) userscripts/test.html [ Crash ]\n')} |
| 496 | self._write('/userscripts/another-test.html', '') |
| 497 | |
Torne (Richard Coles) | 926b001 | 2013-03-28 15:32:48 +0000 | [diff] [blame] | 498 | self.assertDictEqual(self.command._tests_to_rebaseline(self.lion_port), {'userscripts/another-test.html': set(['png', 'txt', 'wav'])}) |
| 499 | self.assertEqual(self._read(self.lion_expectations_path), '') |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 500 | |
Torne (Richard Coles) | 93ac45c | 2013-05-29 14:40:20 +0100 | [diff] [blame^] | 501 | def test_rebaseline_without_other_expectations(self): |
| 502 | self._write("userscripts/another-test.html", "Dummy test contents") |
| 503 | self._write(self.lion_expectations_path, "Bug(x) userscripts/another-test.html [ Rebaseline ]\n") |
| 504 | self.assertDictEqual(self.command._tests_to_rebaseline(self.lion_port), {'userscripts/another-test.html': ('png', 'wav', 'txt')}) |
| 505 | |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 506 | |
| 507 | class _FakeOptimizer(BaselineOptimizer): |
| 508 | def read_results_by_directory(self, baseline_name): |
| 509 | if baseline_name.endswith('txt'): |
Torne (Richard Coles) | 93ac45c | 2013-05-29 14:40:20 +0100 | [diff] [blame^] | 510 | return {'LayoutTests/passes/text.html': '123456'} |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 511 | return {} |
| 512 | |
| 513 | |
| 514 | class TestAnalyzeBaselines(_BaseTestCase): |
| 515 | command_constructor = AnalyzeBaselines |
| 516 | |
| 517 | def setUp(self): |
| 518 | super(TestAnalyzeBaselines, self).setUp() |
| 519 | self.port = self.tool.port_factory.get('test') |
| 520 | self.tool.port_factory.get = (lambda port_name=None, options=None: self.port) |
| 521 | self.lines = [] |
| 522 | self.command._optimizer_class = _FakeOptimizer |
Torne (Richard Coles) | 926b001 | 2013-03-28 15:32:48 +0000 | [diff] [blame] | 523 | self.command._write = (lambda msg: self.lines.append(msg)) # pylint bug warning about unnecessary lambda? pylint: disable=W0108 |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 524 | |
| 525 | def test_default(self): |
| 526 | self.command.execute(MockOptions(suffixes='txt', missing=False, platform=None), ['passes/text.html'], self.tool) |
Torne (Richard Coles) | 926b001 | 2013-03-28 15:32:48 +0000 | [diff] [blame] | 527 | self.assertEqual(self.lines, |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 528 | ['passes/text-expected.txt:', |
Torne (Richard Coles) | 93ac45c | 2013-05-29 14:40:20 +0100 | [diff] [blame^] | 529 | ' (generic): 123456']) |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 530 | |
| 531 | def test_missing_baselines(self): |
| 532 | self.command.execute(MockOptions(suffixes='png,txt', missing=True, platform=None), ['passes/text.html'], self.tool) |
Torne (Richard Coles) | 926b001 | 2013-03-28 15:32:48 +0000 | [diff] [blame] | 533 | self.assertEqual(self.lines, |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 534 | ['passes/text-expected.png: (no baselines found)', |
| 535 | 'passes/text-expected.txt:', |
Torne (Richard Coles) | 93ac45c | 2013-05-29 14:40:20 +0100 | [diff] [blame^] | 536 | ' (generic): 123456']) |