Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 1 | # Copyright 2013 the V8 project authors. All rights reserved. |
| 2 | # Redistribution and use in source and binary forms, with or without |
| 3 | # modification, are permitted provided that the following conditions are |
| 4 | # met: |
| 5 | # |
| 6 | # * Redistributions of source code must retain the above copyright |
| 7 | # notice, this list of conditions and the following disclaimer. |
| 8 | # * Redistributions in binary form must reproduce the above |
| 9 | # copyright notice, this list of conditions and the following |
| 10 | # disclaimer in the documentation and/or other materials provided |
| 11 | # with the distribution. |
| 12 | # * Neither the name of Google Inc. nor the names of its |
| 13 | # contributors may be used to endorse or promote products derived |
| 14 | # from this software without specific prior written permission. |
| 15 | # |
| 16 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 17 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 18 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 19 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 20 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 21 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 22 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 23 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 24 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 25 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 26 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 27 | |
| 28 | import itertools |
| 29 | import os |
| 30 | import re |
| 31 | |
| 32 | from testrunner.local import testsuite |
| 33 | from testrunner.objects import testcase |
| 34 | |
| 35 | FLAGS_PATTERN = re.compile(r"//\s+Flags:(.*)") |
| 36 | FILES_PATTERN = re.compile(r"//\s+Files:(.*)") |
| 37 | SELF_SCRIPT_PATTERN = re.compile(r"//\s+Env: TEST_FILE_NAME") |
| 38 | |
| 39 | |
| 40 | # TODO (machenbach): Share commonalities with mjstest. |
| 41 | class WebkitTestSuite(testsuite.TestSuite): |
| 42 | |
| 43 | def __init__(self, name, root): |
| 44 | super(WebkitTestSuite, self).__init__(name, root) |
| 45 | |
| 46 | def ListTests(self, context): |
| 47 | tests = [] |
| 48 | for dirname, dirs, files in os.walk(self.root): |
| 49 | for dotted in [x for x in dirs if x.startswith('.')]: |
| 50 | dirs.remove(dotted) |
| 51 | if 'resources' in dirs: |
| 52 | dirs.remove('resources') |
| 53 | |
| 54 | dirs.sort() |
| 55 | files.sort() |
| 56 | for filename in files: |
| 57 | if filename.endswith(".js"): |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 58 | fullpath = os.path.join(dirname, filename) |
| 59 | relpath = fullpath[len(self.root) + 1 : -3] |
| 60 | testname = relpath.replace(os.path.sep, "/") |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 61 | test = testcase.TestCase(self, testname) |
| 62 | tests.append(test) |
| 63 | return tests |
| 64 | |
| 65 | def GetFlagsForTestCase(self, testcase, context): |
| 66 | source = self.GetSourceForTest(testcase) |
| 67 | flags = [] + context.mode_flags |
| 68 | flags_match = re.findall(FLAGS_PATTERN, source) |
| 69 | for match in flags_match: |
| 70 | flags += match.strip().split() |
| 71 | |
| 72 | files_list = [] # List of file names to append to command arguments. |
| 73 | files_match = FILES_PATTERN.search(source); |
| 74 | # Accept several lines of 'Files:'. |
| 75 | while True: |
| 76 | if files_match: |
| 77 | files_list += files_match.group(1).strip().split() |
| 78 | files_match = FILES_PATTERN.search(source, files_match.end()) |
| 79 | else: |
| 80 | break |
| 81 | files = [ os.path.normpath(os.path.join(self.root, '..', '..', f)) |
| 82 | for f in files_list ] |
| 83 | testfilename = os.path.join(self.root, testcase.path + self.suffix()) |
| 84 | if SELF_SCRIPT_PATTERN.search(source): |
| 85 | env = ["-e", "TEST_FILE_NAME=\"%s\"" % testfilename.replace("\\", "\\\\")] |
| 86 | files = env + files |
| 87 | files.append(os.path.join(self.root, "resources/standalone-pre.js")) |
| 88 | files.append(testfilename) |
| 89 | files.append(os.path.join(self.root, "resources/standalone-post.js")) |
| 90 | |
| 91 | flags += files |
| 92 | if context.isolates: |
| 93 | flags.append("--isolate") |
| 94 | flags += files |
| 95 | |
| 96 | return testcase.flags + flags |
| 97 | |
| 98 | def GetSourceForTest(self, testcase): |
| 99 | filename = os.path.join(self.root, testcase.path + self.suffix()) |
| 100 | with open(filename) as f: |
| 101 | return f.read() |
| 102 | |
| 103 | # TODO(machenbach): Share with test/message/testcfg.py |
| 104 | def _IgnoreLine(self, string): |
| 105 | """Ignore empty lines, valgrind output and Android output.""" |
| 106 | if not string: return True |
| 107 | return (string.startswith("==") or string.startswith("**") or |
| 108 | string.startswith("ANDROID") or |
| 109 | # These five patterns appear in normal Native Client output. |
| 110 | string.startswith("DEBUG MODE ENABLED") or |
| 111 | string.startswith("tools/nacl-run.py") or |
| 112 | string.find("BYPASSING ALL ACL CHECKS") > 0 or |
| 113 | string.find("Native Client module will be loaded") > 0 or |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 114 | string.find("NaClHostDescOpen:") > 0 or |
| 115 | # FIXME(machenbach): The test driver shouldn't try to use slow |
| 116 | # asserts if they weren't compiled. This fails in optdebug=2. |
| 117 | string == "Warning: unknown flag --enable-slow-asserts." or |
| 118 | string == "Try --help for options") |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 119 | |
Ben Murdoch | da12d29 | 2016-06-02 14:46:10 +0100 | [diff] [blame] | 120 | def IsFailureOutput(self, testcase): |
| 121 | if super(WebkitTestSuite, self).IsFailureOutput(testcase): |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 122 | return True |
Ben Murdoch | da12d29 | 2016-06-02 14:46:10 +0100 | [diff] [blame] | 123 | file_name = os.path.join(self.root, testcase.path) + "-expected.txt" |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 124 | with file(file_name, "r") as expected: |
| 125 | expected_lines = expected.readlines() |
| 126 | |
| 127 | def ExpIterator(): |
| 128 | for line in expected_lines: |
| 129 | if line.startswith("#") or not line.strip(): continue |
| 130 | yield line.strip() |
| 131 | |
| 132 | def ActIterator(lines): |
| 133 | for line in lines: |
| 134 | if self._IgnoreLine(line.strip()): continue |
| 135 | yield line.strip() |
| 136 | |
| 137 | def ActBlockIterator(): |
| 138 | """Iterates over blocks of actual output lines.""" |
Ben Murdoch | da12d29 | 2016-06-02 14:46:10 +0100 | [diff] [blame] | 139 | lines = testcase.output.stdout.splitlines() |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 140 | start_index = 0 |
| 141 | found_eqeq = False |
| 142 | for index, line in enumerate(lines): |
| 143 | # If a stress test separator is found: |
| 144 | if line.startswith("=="): |
| 145 | # Iterate over all lines before a separator except the first. |
| 146 | if not found_eqeq: |
| 147 | found_eqeq = True |
| 148 | else: |
| 149 | yield ActIterator(lines[start_index:index]) |
Ben Murdoch | da12d29 | 2016-06-02 14:46:10 +0100 | [diff] [blame] | 150 | # The next block of output lines starts after the separator. |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 151 | start_index = index + 1 |
| 152 | # Iterate over complete output if no separator was found. |
| 153 | if not found_eqeq: |
| 154 | yield ActIterator(lines) |
| 155 | |
| 156 | for act_iterator in ActBlockIterator(): |
| 157 | for (expected, actual) in itertools.izip_longest( |
| 158 | ExpIterator(), act_iterator, fillvalue=''): |
| 159 | if expected != actual: |
| 160 | return True |
| 161 | return False |
| 162 | |
| 163 | |
| 164 | def GetSuite(name, root): |
| 165 | return WebkitTestSuite(name, root) |