Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 1 | # Copyright (C) 2010 Google Inc. All rights reserved. |
| 2 | # |
| 3 | # Redistribution and use in source and binary forms, with or without |
| 4 | # modification, are permitted provided that the following conditions are |
| 5 | # met: |
| 6 | # |
| 7 | # * Redistributions of source code must retain the above copyright |
| 8 | # notice, this list of conditions and the following disclaimer. |
| 9 | # * Redistributions in binary form must reproduce the above |
| 10 | # copyright notice, this list of conditions and the following disclaimer |
| 11 | # in the documentation and/or other materials provided with the |
| 12 | # distribution. |
| 13 | # * Neither the Google name nor the names of its |
| 14 | # contributors may be used to endorse or promote products derived from |
| 15 | # this software without specific prior written permission. |
| 16 | # |
| 17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 18 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 19 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 20 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 21 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 22 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 23 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 24 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 25 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 26 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 28 | |
| 29 | import base64 |
Torne (Richard Coles) | 93ac45c | 2013-05-29 14:40:20 +0100 | [diff] [blame] | 30 | import copy |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 31 | import sys |
| 32 | import time |
| 33 | |
| 34 | from webkitpy.layout_tests.port import Port, Driver, DriverOutput |
| 35 | from webkitpy.layout_tests.port.base import VirtualTestSuite |
| 36 | from webkitpy.layout_tests.models.test_configuration import TestConfiguration |
| 37 | from webkitpy.common.system.filesystem_mock import MockFileSystem |
| 38 | from webkitpy.common.system.crashlogs import CrashLogs |
| 39 | |
| 40 | |
| 41 | # This sets basic expectations for a test. Each individual expectation |
| 42 | # can be overridden by a keyword argument in TestList.add(). |
| 43 | class TestInstance(object): |
| 44 | def __init__(self, name): |
| 45 | self.name = name |
| 46 | self.base = name[(name.rfind("/") + 1):name.rfind(".")] |
| 47 | self.crash = False |
| 48 | self.web_process_crash = False |
| 49 | self.exception = False |
| 50 | self.hang = False |
| 51 | self.keyboard = False |
| 52 | self.error = '' |
| 53 | self.timeout = False |
| 54 | self.is_reftest = False |
| 55 | |
| 56 | # The values of each field are treated as raw byte strings. They |
| 57 | # will be converted to unicode strings where appropriate using |
| 58 | # FileSystem.read_text_file(). |
| 59 | self.actual_text = self.base + '-txt' |
| 60 | self.actual_checksum = self.base + '-checksum' |
| 61 | |
| 62 | # We add the '\x8a' for the image file to prevent the value from |
| 63 | # being treated as UTF-8 (the character is invalid) |
| 64 | self.actual_image = self.base + '\x8a' + '-png' + 'tEXtchecksum\x00' + self.actual_checksum |
| 65 | |
| 66 | self.expected_text = self.actual_text |
| 67 | self.expected_image = self.actual_image |
| 68 | |
| 69 | self.actual_audio = None |
| 70 | self.expected_audio = None |
| 71 | |
| 72 | |
| 73 | # This is an in-memory list of tests, what we want them to produce, and |
| 74 | # what we want to claim are the expected results. |
| 75 | class TestList(object): |
| 76 | def __init__(self): |
| 77 | self.tests = {} |
| 78 | |
| 79 | def add(self, name, **kwargs): |
| 80 | test = TestInstance(name) |
| 81 | for key, value in kwargs.items(): |
| 82 | test.__dict__[key] = value |
| 83 | self.tests[name] = test |
| 84 | |
| 85 | def add_reftest(self, name, reference_name, same_image): |
| 86 | self.add(name, actual_checksum='xxx', actual_image='XXX', is_reftest=True) |
| 87 | if same_image: |
| 88 | self.add(reference_name, actual_checksum='xxx', actual_image='XXX', is_reftest=True) |
| 89 | else: |
| 90 | self.add(reference_name, actual_checksum='yyy', actual_image='YYY', is_reftest=True) |
| 91 | |
| 92 | def keys(self): |
| 93 | return self.tests.keys() |
| 94 | |
| 95 | def __contains__(self, item): |
| 96 | return item in self.tests |
| 97 | |
| 98 | def __getitem__(self, item): |
| 99 | return self.tests[item] |
| 100 | |
Torne (Richard Coles) | 926b001 | 2013-03-28 15:32:48 +0000 | [diff] [blame] | 101 | # |
| 102 | # These numbers may need to be updated whenever we add or delete tests. |
| 103 | # |
Ben Murdoch | e69819b | 2013-07-17 14:56:49 +0100 | [diff] [blame] | 104 | TOTAL_TESTS = 106 |
| 105 | TOTAL_SKIPS = 27 |
Torne (Richard Coles) | 926b001 | 2013-03-28 15:32:48 +0000 | [diff] [blame] | 106 | |
Ben Murdoch | 3c9e4ae | 2013-08-12 14:20:44 +0100 | [diff] [blame^] | 107 | UNEXPECTED_PASSES = 1 |
| 108 | UNEXPECTED_FAILURES = 22 |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 109 | |
| 110 | def unit_test_list(): |
| 111 | tests = TestList() |
| 112 | tests.add('failures/expected/crash.html', crash=True) |
| 113 | tests.add('failures/expected/exception.html', exception=True) |
| 114 | tests.add('failures/expected/timeout.html', timeout=True) |
| 115 | tests.add('failures/expected/hang.html', hang=True) |
| 116 | tests.add('failures/expected/missing_text.html', expected_text=None) |
Ben Murdoch | 591b958 | 2013-07-10 11:41:44 +0100 | [diff] [blame] | 117 | tests.add('failures/expected/needsrebaseline.html', actual_text='needsrebaseline text') |
Ben Murdoch | e69819b | 2013-07-17 14:56:49 +0100 | [diff] [blame] | 118 | tests.add('failures/expected/needsmanualrebaseline.html', actual_text='needsmanualrebaseline text') |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 119 | tests.add('failures/expected/image.html', |
| 120 | actual_image='image_fail-pngtEXtchecksum\x00checksum_fail', |
| 121 | expected_image='image-pngtEXtchecksum\x00checksum-png') |
| 122 | tests.add('failures/expected/image_checksum.html', |
| 123 | actual_checksum='image_checksum_fail-checksum', |
| 124 | actual_image='image_checksum_fail-png') |
| 125 | tests.add('failures/expected/audio.html', |
| 126 | actual_audio=base64.b64encode('audio_fail-wav'), expected_audio='audio-wav', |
| 127 | actual_text=None, expected_text=None, |
| 128 | actual_image=None, expected_image=None, |
| 129 | actual_checksum=None) |
| 130 | tests.add('failures/expected/keyboard.html', keyboard=True) |
| 131 | tests.add('failures/expected/missing_check.html', |
| 132 | expected_image='missing_check-png') |
| 133 | tests.add('failures/expected/missing_image.html', expected_image=None) |
| 134 | tests.add('failures/expected/missing_audio.html', expected_audio=None, |
| 135 | actual_text=None, expected_text=None, |
| 136 | actual_image=None, expected_image=None, |
| 137 | actual_checksum=None) |
| 138 | tests.add('failures/expected/missing_text.html', expected_text=None) |
| 139 | tests.add('failures/expected/newlines_leading.html', |
| 140 | expected_text="\nfoo\n", actual_text="foo\n") |
| 141 | tests.add('failures/expected/newlines_trailing.html', |
| 142 | expected_text="foo\n\n", actual_text="foo\n") |
| 143 | tests.add('failures/expected/newlines_with_excess_CR.html', |
| 144 | expected_text="foo\r\r\r\n", actual_text="foo\n") |
| 145 | tests.add('failures/expected/text.html', actual_text='text_fail-png') |
| 146 | tests.add('failures/expected/skip_text.html', actual_text='text diff') |
| 147 | tests.add('failures/flaky/text.html') |
| 148 | tests.add('failures/unexpected/missing_text.html', expected_text=None) |
| 149 | tests.add('failures/unexpected/missing_check.html', expected_image='missing-check-png') |
| 150 | tests.add('failures/unexpected/missing_image.html', expected_image=None) |
| 151 | tests.add('failures/unexpected/missing_render_tree_dump.html', actual_text="""layer at (0,0) size 800x600 |
| 152 | RenderView at (0,0) size 800x600 |
| 153 | layer at (0,0) size 800x34 |
| 154 | RenderBlock {HTML} at (0,0) size 800x34 |
| 155 | RenderBody {BODY} at (8,8) size 784x18 |
| 156 | RenderText {#text} at (0,0) size 133x18 |
| 157 | text run at (0,0) width 133: "This is an image test!" |
| 158 | """, expected_text=None) |
| 159 | tests.add('failures/unexpected/crash.html', crash=True) |
| 160 | tests.add('failures/unexpected/crash-with-stderr.html', crash=True, |
| 161 | error="mock-std-error-output") |
| 162 | tests.add('failures/unexpected/web-process-crash-with-stderr.html', web_process_crash=True, |
| 163 | error="mock-std-error-output") |
| 164 | tests.add('failures/unexpected/pass.html') |
| 165 | tests.add('failures/unexpected/text-checksum.html', |
| 166 | actual_text='text-checksum_fail-txt', |
| 167 | actual_checksum='text-checksum_fail-checksum') |
| 168 | tests.add('failures/unexpected/text-image-checksum.html', |
| 169 | actual_text='text-image-checksum_fail-txt', |
| 170 | actual_image='text-image-checksum_fail-pngtEXtchecksum\x00checksum_fail', |
| 171 | actual_checksum='text-image-checksum_fail-checksum') |
| 172 | tests.add('failures/unexpected/checksum-with-matching-image.html', |
| 173 | actual_checksum='text-image-checksum_fail-checksum') |
| 174 | tests.add('failures/unexpected/skip_pass.html') |
| 175 | tests.add('failures/unexpected/text.html', actual_text='text_fail-txt') |
| 176 | tests.add('failures/unexpected/timeout.html', timeout=True) |
| 177 | tests.add('http/tests/passes/text.html') |
| 178 | tests.add('http/tests/passes/image.html') |
| 179 | tests.add('http/tests/ssl/text.html') |
| 180 | tests.add('passes/args.html') |
| 181 | tests.add('passes/error.html', error='stuff going to stderr') |
| 182 | tests.add('passes/image.html') |
| 183 | tests.add('passes/audio.html', |
| 184 | actual_audio=base64.b64encode('audio-wav'), expected_audio='audio-wav', |
| 185 | actual_text=None, expected_text=None, |
| 186 | actual_image=None, expected_image=None, |
| 187 | actual_checksum=None) |
| 188 | tests.add('passes/platform_image.html') |
| 189 | tests.add('passes/checksum_in_image.html', |
| 190 | expected_image='tEXtchecksum\x00checksum_in_image-checksum') |
| 191 | tests.add('passes/skipped/skip.html') |
| 192 | |
| 193 | # Note that here the checksums don't match but the images do, so this test passes "unexpectedly". |
| 194 | # See https://bugs.webkit.org/show_bug.cgi?id=69444 . |
| 195 | tests.add('failures/unexpected/checksum.html', actual_checksum='checksum_fail-checksum') |
| 196 | |
| 197 | # Text output files contain "\r\n" on Windows. This may be |
| 198 | # helpfully filtered to "\r\r\n" by our Python/Cygwin tooling. |
| 199 | tests.add('passes/text.html', |
| 200 | expected_text='\nfoo\n\n', actual_text='\nfoo\r\n\r\r\n') |
| 201 | |
| 202 | # For reftests. |
| 203 | tests.add_reftest('passes/reftest.html', 'passes/reftest-expected.html', same_image=True) |
| 204 | tests.add_reftest('passes/mismatch.html', 'passes/mismatch-expected-mismatch.html', same_image=False) |
| 205 | tests.add_reftest('passes/svgreftest.svg', 'passes/svgreftest-expected.svg', same_image=True) |
| 206 | tests.add_reftest('passes/xhtreftest.xht', 'passes/xhtreftest-expected.html', same_image=True) |
| 207 | tests.add_reftest('passes/phpreftest.php', 'passes/phpreftest-expected-mismatch.svg', same_image=False) |
| 208 | tests.add_reftest('failures/expected/reftest.html', 'failures/expected/reftest-expected.html', same_image=False) |
| 209 | tests.add_reftest('failures/expected/mismatch.html', 'failures/expected/mismatch-expected-mismatch.html', same_image=True) |
| 210 | tests.add_reftest('failures/unexpected/reftest.html', 'failures/unexpected/reftest-expected.html', same_image=False) |
| 211 | tests.add_reftest('failures/unexpected/mismatch.html', 'failures/unexpected/mismatch-expected-mismatch.html', same_image=True) |
| 212 | tests.add('failures/unexpected/reftest-nopixel.html', actual_checksum=None, actual_image=None, is_reftest=True) |
| 213 | tests.add('failures/unexpected/reftest-nopixel-expected.html', actual_checksum=None, actual_image=None, is_reftest=True) |
| 214 | # FIXME: Add a reftest which crashes. |
| 215 | tests.add('reftests/foo/test.html') |
| 216 | tests.add('reftests/foo/test-ref.html') |
| 217 | |
| 218 | tests.add('reftests/foo/multiple-match-success.html', actual_checksum='abc', actual_image='abc') |
| 219 | tests.add('reftests/foo/multiple-match-failure.html', actual_checksum='abc', actual_image='abc') |
| 220 | tests.add('reftests/foo/multiple-mismatch-success.html', actual_checksum='abc', actual_image='abc') |
| 221 | tests.add('reftests/foo/multiple-mismatch-failure.html', actual_checksum='abc', actual_image='abc') |
| 222 | tests.add('reftests/foo/multiple-both-success.html', actual_checksum='abc', actual_image='abc') |
| 223 | tests.add('reftests/foo/multiple-both-failure.html', actual_checksum='abc', actual_image='abc') |
| 224 | |
| 225 | tests.add('reftests/foo/matching-ref.html', actual_checksum='abc', actual_image='abc') |
| 226 | tests.add('reftests/foo/mismatching-ref.html', actual_checksum='def', actual_image='def') |
| 227 | tests.add('reftests/foo/second-mismatching-ref.html', actual_checksum='ghi', actual_image='ghi') |
| 228 | |
| 229 | # The following files shouldn't be treated as reftests |
| 230 | tests.add_reftest('reftests/foo/unlistedtest.html', 'reftests/foo/unlistedtest-expected.html', same_image=True) |
| 231 | tests.add('reftests/foo/reference/bar/common.html') |
| 232 | tests.add('reftests/foo/reftest/bar/shared.html') |
| 233 | |
| 234 | tests.add('websocket/tests/passes/text.html') |
| 235 | |
Torne (Richard Coles) | 93ac45c | 2013-05-29 14:40:20 +0100 | [diff] [blame] | 236 | # For testing that we don't run tests under platform/. Note that these don't contribute to TOTAL_TESTS. |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 237 | tests.add('platform/test-mac-leopard/http/test.html') |
| 238 | tests.add('platform/test-win-win7/http/test.html') |
| 239 | |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 240 | # For testing if perf tests are running in a locked shard. |
| 241 | tests.add('perf/foo/test.html') |
| 242 | tests.add('perf/foo/test-ref.html') |
| 243 | |
| 244 | # For testing --pixel-test-directories. |
| 245 | tests.add('failures/unexpected/pixeldir/image_in_pixeldir.html', |
| 246 | actual_image='image_in_pixeldir-pngtEXtchecksum\x00checksum_fail', |
| 247 | expected_image='image_in_pixeldir-pngtEXtchecksum\x00checksum-png') |
| 248 | tests.add('failures/unexpected/image_not_in_pixeldir.html', |
| 249 | actual_image='image_not_in_pixeldir-pngtEXtchecksum\x00checksum_fail', |
| 250 | expected_image='image_not_in_pixeldir-pngtEXtchecksum\x00checksum-png') |
| 251 | |
| 252 | # For testing that virtual test suites don't expand names containing themselves |
| 253 | # See webkit.org/b/97925 and base_unittest.PortTest.test_tests(). |
| 254 | tests.add('passes/test-virtual-passes.html') |
| 255 | tests.add('passes/passes/test-virtual-passes.html') |
| 256 | |
| 257 | return tests |
| 258 | |
| 259 | |
| 260 | # Here we use a non-standard location for the layout tests, to ensure that |
| 261 | # this works. The path contains a '.' in the name because we've seen bugs |
| 262 | # related to this before. |
| 263 | |
| 264 | LAYOUT_TEST_DIR = '/test.checkout/LayoutTests' |
| 265 | PERF_TEST_DIR = '/test.checkout/PerformanceTests' |
| 266 | |
| 267 | |
| 268 | # Here we synthesize an in-memory filesystem from the test list |
| 269 | # in order to fully control the test output and to demonstrate that |
| 270 | # we don't need a real filesystem to run the tests. |
| 271 | def add_unit_tests_to_mock_filesystem(filesystem): |
| 272 | # Add the test_expectations file. |
| 273 | filesystem.maybe_make_directory(LAYOUT_TEST_DIR + '/platform/test') |
| 274 | if not filesystem.exists(LAYOUT_TEST_DIR + '/platform/test/TestExpectations'): |
| 275 | filesystem.write_text_file(LAYOUT_TEST_DIR + '/platform/test/TestExpectations', """ |
| 276 | Bug(test) failures/expected/crash.html [ Crash ] |
| 277 | Bug(test) failures/expected/image.html [ ImageOnlyFailure ] |
Ben Murdoch | 591b958 | 2013-07-10 11:41:44 +0100 | [diff] [blame] | 278 | Bug(test) failures/expected/needsrebaseline.html [ NeedsRebaseline ] |
Ben Murdoch | e69819b | 2013-07-17 14:56:49 +0100 | [diff] [blame] | 279 | Bug(test) failures/expected/needsmanualrebaseline.html [ NeedsManualRebaseline ] |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 280 | Bug(test) failures/expected/audio.html [ Failure ] |
| 281 | Bug(test) failures/expected/image_checksum.html [ ImageOnlyFailure ] |
| 282 | Bug(test) failures/expected/mismatch.html [ ImageOnlyFailure ] |
| 283 | Bug(test) failures/expected/missing_check.html [ Missing Pass ] |
| 284 | Bug(test) failures/expected/missing_image.html [ Missing Pass ] |
| 285 | Bug(test) failures/expected/missing_audio.html [ Missing Pass ] |
| 286 | Bug(test) failures/expected/missing_text.html [ Missing Pass ] |
| 287 | Bug(test) failures/expected/newlines_leading.html [ Failure ] |
| 288 | Bug(test) failures/expected/newlines_trailing.html [ Failure ] |
| 289 | Bug(test) failures/expected/newlines_with_excess_CR.html [ Failure ] |
| 290 | Bug(test) failures/expected/reftest.html [ ImageOnlyFailure ] |
| 291 | Bug(test) failures/expected/text.html [ Failure ] |
| 292 | Bug(test) failures/expected/timeout.html [ Timeout ] |
| 293 | Bug(test) failures/expected/hang.html [ WontFix ] |
| 294 | Bug(test) failures/expected/keyboard.html [ WontFix ] |
| 295 | Bug(test) failures/expected/exception.html [ WontFix ] |
| 296 | Bug(test) failures/unexpected/pass.html [ Failure ] |
| 297 | Bug(test) passes/skipped/skip.html [ Skip ] |
Torne (Richard Coles) | 81a5157 | 2013-05-13 16:52:28 +0100 | [diff] [blame] | 298 | Bug(test) passes/text.html [ Pass ] |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 299 | """) |
| 300 | |
| 301 | filesystem.maybe_make_directory(LAYOUT_TEST_DIR + '/reftests/foo') |
| 302 | filesystem.write_text_file(LAYOUT_TEST_DIR + '/reftests/foo/reftest.list', """ |
| 303 | == test.html test-ref.html |
| 304 | |
| 305 | == multiple-match-success.html mismatching-ref.html |
| 306 | == multiple-match-success.html matching-ref.html |
| 307 | == multiple-match-failure.html mismatching-ref.html |
| 308 | == multiple-match-failure.html second-mismatching-ref.html |
| 309 | != multiple-mismatch-success.html mismatching-ref.html |
| 310 | != multiple-mismatch-success.html second-mismatching-ref.html |
| 311 | != multiple-mismatch-failure.html mismatching-ref.html |
| 312 | != multiple-mismatch-failure.html matching-ref.html |
| 313 | == multiple-both-success.html matching-ref.html |
| 314 | == multiple-both-success.html mismatching-ref.html |
| 315 | != multiple-both-success.html second-mismatching-ref.html |
| 316 | == multiple-both-failure.html matching-ref.html |
| 317 | != multiple-both-failure.html second-mismatching-ref.html |
| 318 | != multiple-both-failure.html matching-ref.html |
| 319 | """) |
| 320 | |
| 321 | # FIXME: This test was only being ignored because of missing a leading '/'. |
| 322 | # Fixing the typo causes several tests to assert, so disabling the test entirely. |
| 323 | # Add in a file should be ignored by port.find_test_files(). |
| 324 | #files[LAYOUT_TEST_DIR + '/userscripts/resources/iframe.html'] = 'iframe' |
| 325 | |
| 326 | def add_file(test, suffix, contents): |
| 327 | dirname = filesystem.join(LAYOUT_TEST_DIR, test.name[0:test.name.rfind('/')]) |
| 328 | base = test.base |
| 329 | filesystem.maybe_make_directory(dirname) |
| 330 | filesystem.write_binary_file(filesystem.join(dirname, base + suffix), contents) |
| 331 | |
| 332 | # Add each test and the expected output, if any. |
| 333 | test_list = unit_test_list() |
| 334 | for test in test_list.tests.values(): |
| 335 | add_file(test, test.name[test.name.rfind('.'):], '') |
| 336 | if test.is_reftest: |
| 337 | continue |
| 338 | if test.actual_audio: |
| 339 | add_file(test, '-expected.wav', test.expected_audio) |
| 340 | continue |
| 341 | add_file(test, '-expected.txt', test.expected_text) |
| 342 | add_file(test, '-expected.png', test.expected_image) |
| 343 | |
| 344 | filesystem.write_text_file(filesystem.join(LAYOUT_TEST_DIR, 'virtual', 'passes', 'args-expected.txt'), 'args-txt --virtual-arg') |
| 345 | # Clear the list of written files so that we can watch what happens during testing. |
| 346 | filesystem.clear_written_files() |
| 347 | |
| 348 | |
| 349 | class TestPort(Port): |
| 350 | port_name = 'test' |
Torne (Richard Coles) | 926b001 | 2013-03-28 15:32:48 +0000 | [diff] [blame] | 351 | default_port_name = 'test-mac-leopard' |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 352 | |
| 353 | """Test implementation of the Port interface.""" |
| 354 | ALL_BASELINE_VARIANTS = ( |
| 355 | 'test-linux-x86_64', |
| 356 | 'test-mac-snowleopard', 'test-mac-leopard', |
Torne (Richard Coles) | 521d96e | 2013-06-19 11:58:24 +0100 | [diff] [blame] | 357 | 'test-win-win7', 'test-win-xp', |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 358 | ) |
| 359 | |
| 360 | @classmethod |
| 361 | def determine_full_port_name(cls, host, options, port_name): |
| 362 | if port_name == 'test': |
Torne (Richard Coles) | 926b001 | 2013-03-28 15:32:48 +0000 | [diff] [blame] | 363 | return TestPort.default_port_name |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 364 | return port_name |
| 365 | |
| 366 | def __init__(self, host, port_name=None, **kwargs): |
Torne (Richard Coles) | 926b001 | 2013-03-28 15:32:48 +0000 | [diff] [blame] | 367 | Port.__init__(self, host, port_name or TestPort.default_port_name, **kwargs) |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 368 | self._tests = unit_test_list() |
| 369 | self._flakes = set() |
Torne (Richard Coles) | 53e740f | 2013-05-09 18:38:43 +0100 | [diff] [blame] | 370 | self._generic_expectations_path = LAYOUT_TEST_DIR + '/TestExpectations' |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 371 | self._results_directory = None |
| 372 | |
| 373 | self._operating_system = 'mac' |
Torne (Richard Coles) | 926b001 | 2013-03-28 15:32:48 +0000 | [diff] [blame] | 374 | if self._name.startswith('test-win'): |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 375 | self._operating_system = 'win' |
Torne (Richard Coles) | 926b001 | 2013-03-28 15:32:48 +0000 | [diff] [blame] | 376 | elif self._name.startswith('test-linux'): |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 377 | self._operating_system = 'linux' |
| 378 | |
| 379 | version_map = { |
| 380 | 'test-win-xp': 'xp', |
| 381 | 'test-win-win7': 'win7', |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 382 | 'test-mac-leopard': 'leopard', |
| 383 | 'test-mac-snowleopard': 'snowleopard', |
| 384 | 'test-linux-x86_64': 'lucid', |
| 385 | } |
Torne (Richard Coles) | 926b001 | 2013-03-28 15:32:48 +0000 | [diff] [blame] | 386 | self._version = version_map[self._name] |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 387 | |
Torne (Richard Coles) | 53e740f | 2013-05-09 18:38:43 +0100 | [diff] [blame] | 388 | def buildbot_archives_baselines(self): |
| 389 | return self._name != 'test-win-xp' |
| 390 | |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 391 | def default_pixel_tests(self): |
| 392 | return True |
| 393 | |
| 394 | def _path_to_driver(self): |
| 395 | # This routine shouldn't normally be called, but it is called by |
| 396 | # the mock_drt Driver. We return something, but make sure it's useless. |
| 397 | return 'MOCK _path_to_driver' |
| 398 | |
| 399 | def baseline_search_path(self): |
| 400 | search_paths = { |
| 401 | 'test-mac-snowleopard': ['test-mac-snowleopard'], |
| 402 | 'test-mac-leopard': ['test-mac-leopard', 'test-mac-snowleopard'], |
| 403 | 'test-win-win7': ['test-win-win7'], |
Torne (Richard Coles) | 521d96e | 2013-06-19 11:58:24 +0100 | [diff] [blame] | 404 | 'test-win-xp': ['test-win-xp', 'test-win-win7'], |
Torne (Richard Coles) | 53e740f | 2013-05-09 18:38:43 +0100 | [diff] [blame] | 405 | 'test-linux-x86_64': ['test-linux-x86_64', 'test-win-win7'], |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 406 | } |
| 407 | return [self._webkit_baseline_path(d) for d in search_paths[self.name()]] |
| 408 | |
| 409 | def default_child_processes(self): |
| 410 | return 1 |
| 411 | |
| 412 | def worker_startup_delay_secs(self): |
| 413 | return 0 |
| 414 | |
| 415 | def check_build(self, needs_http): |
| 416 | return True |
| 417 | |
| 418 | def check_sys_deps(self, needs_http): |
| 419 | return True |
| 420 | |
| 421 | def default_configuration(self): |
| 422 | return 'Release' |
| 423 | |
Ben Murdoch | 591b958 | 2013-07-10 11:41:44 +0100 | [diff] [blame] | 424 | def diff_image(self, expected_contents, actual_contents): |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 425 | diffed = actual_contents != expected_contents |
| 426 | if not actual_contents and not expected_contents: |
Ben Murdoch | 591b958 | 2013-07-10 11:41:44 +0100 | [diff] [blame] | 427 | return (None, None) |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 428 | if not actual_contents or not expected_contents: |
Ben Murdoch | 591b958 | 2013-07-10 11:41:44 +0100 | [diff] [blame] | 429 | return (True, None) |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 430 | if diffed: |
Ben Murdoch | 591b958 | 2013-07-10 11:41:44 +0100 | [diff] [blame] | 431 | return ("< %s\n---\n> %s\n" % (expected_contents, actual_contents), None) |
| 432 | return (None, None) |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 433 | |
| 434 | def layout_tests_dir(self): |
| 435 | return LAYOUT_TEST_DIR |
| 436 | |
| 437 | def perf_tests_dir(self): |
| 438 | return PERF_TEST_DIR |
| 439 | |
| 440 | def webkit_base(self): |
| 441 | return '/test.checkout' |
| 442 | |
Torne (Richard Coles) | 926b001 | 2013-03-28 15:32:48 +0000 | [diff] [blame] | 443 | def _skipped_tests_for_unsupported_features(self, test_list): |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 444 | return set(['failures/expected/skip_text.html', |
| 445 | 'failures/unexpected/skip_pass.html', |
| 446 | 'virtual/skipped']) |
| 447 | |
| 448 | def name(self): |
| 449 | return self._name |
| 450 | |
| 451 | def operating_system(self): |
| 452 | return self._operating_system |
| 453 | |
| 454 | def _path_to_wdiff(self): |
| 455 | return None |
| 456 | |
| 457 | def default_results_directory(self): |
| 458 | return '/tmp/layout-test-results' |
| 459 | |
| 460 | def setup_test_run(self): |
| 461 | pass |
| 462 | |
| 463 | def _driver_class(self): |
| 464 | return TestDriver |
| 465 | |
| 466 | def start_http_server(self, additional_dirs=None, number_of_servers=None): |
| 467 | pass |
| 468 | |
| 469 | def start_websocket_server(self): |
| 470 | pass |
| 471 | |
| 472 | def acquire_http_lock(self): |
| 473 | pass |
| 474 | |
| 475 | def stop_http_server(self): |
| 476 | pass |
| 477 | |
| 478 | def stop_websocket_server(self): |
| 479 | pass |
| 480 | |
| 481 | def release_http_lock(self): |
| 482 | pass |
| 483 | |
| 484 | def _path_to_lighttpd(self): |
| 485 | return "/usr/sbin/lighttpd" |
| 486 | |
| 487 | def _path_to_lighttpd_modules(self): |
| 488 | return "/usr/lib/lighttpd" |
| 489 | |
| 490 | def _path_to_lighttpd_php(self): |
| 491 | return "/usr/bin/php-cgi" |
| 492 | |
| 493 | def _path_to_apache(self): |
| 494 | return "/usr/sbin/httpd" |
| 495 | |
| 496 | def _path_to_apache_config_file(self): |
| 497 | return self._filesystem.join(self.layout_tests_dir(), 'http', 'conf', 'httpd.conf') |
| 498 | |
Torne (Richard Coles) | 53e740f | 2013-05-09 18:38:43 +0100 | [diff] [blame] | 499 | def path_to_generic_test_expectations_file(self): |
| 500 | return self._generic_expectations_path |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 501 | |
| 502 | def all_test_configurations(self): |
| 503 | """Returns a sequence of the TestConfigurations the port supports.""" |
| 504 | # By default, we assume we want to test every graphics type in |
| 505 | # every configuration on every system. |
| 506 | test_configurations = [] |
| 507 | for version, architecture in self._all_systems(): |
| 508 | for build_type in self._all_build_types(): |
| 509 | test_configurations.append(TestConfiguration( |
| 510 | version=version, |
| 511 | architecture=architecture, |
| 512 | build_type=build_type)) |
| 513 | return test_configurations |
| 514 | |
| 515 | def _all_systems(self): |
| 516 | return (('leopard', 'x86'), |
| 517 | ('snowleopard', 'x86'), |
| 518 | ('xp', 'x86'), |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 519 | ('win7', 'x86'), |
| 520 | ('lucid', 'x86'), |
| 521 | ('lucid', 'x86_64')) |
| 522 | |
| 523 | def _all_build_types(self): |
| 524 | return ('debug', 'release') |
| 525 | |
| 526 | def configuration_specifier_macros(self): |
| 527 | """To avoid surprises when introducing new macros, these are intentionally fixed in time.""" |
Torne (Richard Coles) | 521d96e | 2013-06-19 11:58:24 +0100 | [diff] [blame] | 528 | return {'mac': ['leopard', 'snowleopard'], 'win': ['xp', 'win7'], 'linux': ['lucid']} |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 529 | |
| 530 | def all_baseline_variants(self): |
| 531 | return self.ALL_BASELINE_VARIANTS |
| 532 | |
| 533 | def virtual_test_suites(self): |
| 534 | return [ |
| 535 | VirtualTestSuite('virtual/passes', 'passes', ['--virtual-arg']), |
| 536 | VirtualTestSuite('virtual/skipped', 'failures/expected', ['--virtual-arg2']), |
| 537 | ] |
| 538 | |
| 539 | |
| 540 | class TestDriver(Driver): |
Ben Murdoch | 591b958 | 2013-07-10 11:41:44 +0100 | [diff] [blame] | 541 | """Test/Dummy implementation of the driver interface.""" |
Torne (Richard Coles) | 926b001 | 2013-03-28 15:32:48 +0000 | [diff] [blame] | 542 | next_pid = 1 |
| 543 | |
| 544 | def __init__(self, *args, **kwargs): |
| 545 | super(TestDriver, self).__init__(*args, **kwargs) |
| 546 | self.started = False |
| 547 | self.pid = 0 |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 548 | |
| 549 | def cmd_line(self, pixel_tests, per_test_args): |
| 550 | pixel_tests_flag = '-p' if pixel_tests else '' |
| 551 | return [self._port._path_to_driver()] + [pixel_tests_flag] + self._port.get_option('additional_drt_flag', []) + per_test_args |
| 552 | |
Torne (Richard Coles) | 93ac45c | 2013-05-29 14:40:20 +0100 | [diff] [blame] | 553 | def run_test(self, driver_input, stop_when_done): |
| 554 | base = self._port.lookup_virtual_test_base(driver_input.test_name) |
| 555 | if base: |
| 556 | virtual_driver_input = copy.copy(driver_input) |
| 557 | virtual_driver_input.test_name = base |
| 558 | virtual_driver_input.args = self._port.lookup_virtual_test_args(driver_input.test_name) |
| 559 | return self.run_test(virtual_driver_input, stop_when_done) |
| 560 | |
Torne (Richard Coles) | 926b001 | 2013-03-28 15:32:48 +0000 | [diff] [blame] | 561 | if not self.started: |
| 562 | self.started = True |
| 563 | self.pid = TestDriver.next_pid |
| 564 | TestDriver.next_pid += 1 |
| 565 | |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 566 | start_time = time.time() |
Torne (Richard Coles) | 93ac45c | 2013-05-29 14:40:20 +0100 | [diff] [blame] | 567 | test_name = driver_input.test_name |
| 568 | test_args = driver_input.args or [] |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 569 | test = self._port._tests[test_name] |
| 570 | if test.keyboard: |
| 571 | raise KeyboardInterrupt |
| 572 | if test.exception: |
| 573 | raise ValueError('exception from ' + test_name) |
| 574 | if test.hang: |
Torne (Richard Coles) | 93ac45c | 2013-05-29 14:40:20 +0100 | [diff] [blame] | 575 | time.sleep((float(driver_input.timeout) * 4) / 1000.0 + 1.0) # The 1.0 comes from thread_padding_sec in layout_test_runnery. |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 576 | |
| 577 | audio = None |
| 578 | actual_text = test.actual_text |
| 579 | |
| 580 | if 'flaky' in test_name and not test_name in self._port._flakes: |
| 581 | self._port._flakes.add(test_name) |
| 582 | actual_text = 'flaky text failure' |
| 583 | |
| 584 | if actual_text and test_args and test_name == 'passes/args.html': |
| 585 | actual_text = actual_text + ' ' + ' '.join(test_args) |
| 586 | |
| 587 | if test.actual_audio: |
| 588 | audio = base64.b64decode(test.actual_audio) |
| 589 | crashed_process_name = None |
| 590 | crashed_pid = None |
| 591 | if test.crash: |
| 592 | crashed_process_name = self._port.driver_name() |
| 593 | crashed_pid = 1 |
| 594 | elif test.web_process_crash: |
| 595 | crashed_process_name = 'WebProcess' |
| 596 | crashed_pid = 2 |
| 597 | |
| 598 | crash_log = '' |
| 599 | if crashed_process_name: |
| 600 | crash_logs = CrashLogs(self._port.host) |
| 601 | crash_log = crash_logs.find_newest_log(crashed_process_name, None) or '' |
| 602 | |
| 603 | if stop_when_done: |
| 604 | self.stop() |
| 605 | |
Torne (Richard Coles) | 93ac45c | 2013-05-29 14:40:20 +0100 | [diff] [blame] | 606 | if test.actual_checksum == driver_input.image_hash: |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 607 | image = None |
| 608 | else: |
| 609 | image = test.actual_image |
| 610 | return DriverOutput(actual_text, image, test.actual_checksum, audio, |
| 611 | crash=test.crash or test.web_process_crash, crashed_process_name=crashed_process_name, |
| 612 | crashed_pid=crashed_pid, crash_log=crash_log, |
Torne (Richard Coles) | 926b001 | 2013-03-28 15:32:48 +0000 | [diff] [blame] | 613 | test_time=time.time() - start_time, timeout=test.timeout, error=test.error, pid=self.pid) |
Torne (Richard Coles) | 5c87bf8 | 2012-11-14 11:46:17 +0000 | [diff] [blame] | 614 | |
| 615 | def stop(self): |
Torne (Richard Coles) | 926b001 | 2013-03-28 15:32:48 +0000 | [diff] [blame] | 616 | self.started = False |