Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 1 | # Copyright 2012 the V8 project authors. All rights reserved. |
| 2 | # Redistribution and use in source and binary forms, with or without |
| 3 | # modification, are permitted provided that the following conditions are |
| 4 | # met: |
| 5 | # |
| 6 | # * Redistributions of source code must retain the above copyright |
| 7 | # notice, this list of conditions and the following disclaimer. |
| 8 | # * Redistributions in binary form must reproduce the above |
| 9 | # copyright notice, this list of conditions and the following |
| 10 | # disclaimer in the documentation and/or other materials provided |
| 11 | # with the distribution. |
| 12 | # * Neither the name of Google Inc. nor the names of its |
| 13 | # contributors may be used to endorse or promote products derived |
| 14 | # from this software without specific prior written permission. |
| 15 | # |
| 16 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 17 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 18 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 19 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 20 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 21 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 22 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 23 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 24 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 25 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 26 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 27 | |
| 28 | |
| 29 | import imp |
| 30 | import os |
| 31 | |
| 32 | from . import commands |
| 33 | from . import statusfile |
| 34 | from . import utils |
| 35 | from ..objects import testcase |
| 36 | |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 37 | # Use this to run several variants of the tests. |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 38 | ALL_VARIANT_FLAGS = { |
| 39 | "default": [[]], |
| 40 | "stress": [["--stress-opt", "--always-opt"]], |
| 41 | "turbofan": [["--turbo"]], |
| 42 | "turbofan_opt": [["--turbo", "--always-opt"]], |
| 43 | "nocrankshaft": [["--nocrankshaft"]], |
Ben Murdoch | 097c5b2 | 2016-05-18 11:27:45 +0100 | [diff] [blame] | 44 | "ignition": [["--ignition", "--turbo"]], |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 45 | "preparser": [["--min-preparse-length=0"]], |
| 46 | } |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 47 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 48 | # FAST_VARIANTS implies no --always-opt. |
| 49 | FAST_VARIANT_FLAGS = { |
| 50 | "default": [[]], |
| 51 | "stress": [["--stress-opt"]], |
| 52 | "turbofan": [["--turbo"]], |
| 53 | "nocrankshaft": [["--nocrankshaft"]], |
Ben Murdoch | 097c5b2 | 2016-05-18 11:27:45 +0100 | [diff] [blame] | 54 | "ignition": [["--ignition", "--turbo"]], |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 55 | "preparser": [["--min-preparse-length=0"]], |
| 56 | } |
| 57 | |
| 58 | ALL_VARIANTS = set(["default", "stress", "turbofan", "turbofan_opt", |
| 59 | "nocrankshaft", "ignition", "preparser"]) |
| 60 | FAST_VARIANTS = set(["default", "turbofan"]) |
| 61 | STANDARD_VARIANT = set(["default"]) |
| 62 | |
| 63 | |
| 64 | class VariantGenerator(object): |
| 65 | def __init__(self, suite, variants): |
| 66 | self.suite = suite |
| 67 | self.all_variants = ALL_VARIANTS & variants |
| 68 | self.fast_variants = FAST_VARIANTS & variants |
| 69 | self.standard_variant = STANDARD_VARIANT & variants |
| 70 | |
| 71 | def FilterVariantsByTest(self, testcase): |
| 72 | if testcase.outcomes and statusfile.OnlyStandardVariant( |
| 73 | testcase.outcomes): |
| 74 | return self.standard_variant |
| 75 | if testcase.outcomes and statusfile.OnlyFastVariants(testcase.outcomes): |
| 76 | return self.fast_variants |
| 77 | return self.all_variants |
| 78 | |
| 79 | def GetFlagSets(self, testcase, variant): |
| 80 | if testcase.outcomes and statusfile.OnlyFastVariants(testcase.outcomes): |
| 81 | return FAST_VARIANT_FLAGS[variant] |
| 82 | else: |
| 83 | return ALL_VARIANT_FLAGS[variant] |
| 84 | |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 85 | |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 86 | class TestSuite(object): |
| 87 | |
| 88 | @staticmethod |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 89 | def LoadTestSuite(root, global_init=True): |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 90 | name = root.split(os.path.sep)[-1] |
| 91 | f = None |
| 92 | try: |
| 93 | (f, pathname, description) = imp.find_module("testcfg", [root]) |
| 94 | module = imp.load_module("testcfg", f, pathname, description) |
| 95 | return module.GetSuite(name, root) |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 96 | except ImportError: |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 97 | # Use default if no testcfg is present. |
| 98 | return GoogleTestSuite(name, root) |
| 99 | finally: |
| 100 | if f: |
| 101 | f.close() |
| 102 | |
| 103 | def __init__(self, name, root): |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 104 | # Note: This might be called concurrently from different processes. |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 105 | self.name = name # string |
| 106 | self.root = root # string containing path |
| 107 | self.tests = None # list of TestCase objects |
| 108 | self.rules = None # dictionary mapping test path to list of outcomes |
| 109 | self.wildcards = None # dictionary mapping test paths to list of outcomes |
| 110 | self.total_duration = None # float, assigned on demand |
| 111 | |
| 112 | def shell(self): |
| 113 | return "d8" |
| 114 | |
| 115 | def suffix(self): |
| 116 | return ".js" |
| 117 | |
| 118 | def status_file(self): |
| 119 | return "%s/%s.status" % (self.root, self.name) |
| 120 | |
| 121 | # Used in the status file and for stdout printing. |
| 122 | def CommonTestName(self, testcase): |
| 123 | if utils.IsWindows(): |
| 124 | return testcase.path.replace("\\", "/") |
| 125 | else: |
| 126 | return testcase.path |
| 127 | |
| 128 | def ListTests(self, context): |
| 129 | raise NotImplementedError |
| 130 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 131 | def _VariantGeneratorFactory(self): |
| 132 | """The variant generator class to be used.""" |
| 133 | return VariantGenerator |
| 134 | |
| 135 | def CreateVariantGenerator(self, variants): |
| 136 | """Return a generator for the testing variants of this suite. |
| 137 | |
| 138 | Args: |
| 139 | variants: List of variant names to be run as specified by the test |
| 140 | runner. |
| 141 | Returns: An object of type VariantGenerator. |
| 142 | """ |
| 143 | return self._VariantGeneratorFactory()(self, set(variants)) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 144 | |
| 145 | def DownloadData(self): |
| 146 | pass |
| 147 | |
| 148 | def ReadStatusFile(self, variables): |
| 149 | (self.rules, self.wildcards) = \ |
| 150 | statusfile.ReadStatusFile(self.status_file(), variables) |
| 151 | |
| 152 | def ReadTestCases(self, context): |
| 153 | self.tests = self.ListTests(context) |
| 154 | |
| 155 | @staticmethod |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 156 | def _FilterSlow(slow, mode): |
| 157 | return (mode == "run" and not slow) or (mode == "skip" and slow) |
| 158 | |
| 159 | @staticmethod |
| 160 | def _FilterPassFail(pass_fail, mode): |
| 161 | return (mode == "run" and not pass_fail) or (mode == "skip" and pass_fail) |
| 162 | |
| 163 | def FilterTestCasesByStatus(self, warn_unused_rules, |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 164 | slow_tests="dontcare", |
| 165 | pass_fail_tests="dontcare"): |
| 166 | filtered = [] |
| 167 | used_rules = set() |
| 168 | for t in self.tests: |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 169 | slow = False |
| 170 | pass_fail = False |
| 171 | testname = self.CommonTestName(t) |
| 172 | if testname in self.rules: |
| 173 | used_rules.add(testname) |
| 174 | # Even for skipped tests, as the TestCase object stays around and |
| 175 | # PrintReport() uses it. |
| 176 | t.outcomes = self.rules[testname] |
| 177 | if statusfile.DoSkip(t.outcomes): |
| 178 | continue # Don't add skipped tests to |filtered|. |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 179 | for outcome in t.outcomes: |
| 180 | if outcome.startswith('Flags: '): |
| 181 | t.flags += outcome[7:].split() |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 182 | slow = statusfile.IsSlow(t.outcomes) |
| 183 | pass_fail = statusfile.IsPassOrFail(t.outcomes) |
| 184 | skip = False |
| 185 | for rule in self.wildcards: |
| 186 | assert rule[-1] == '*' |
| 187 | if testname.startswith(rule[:-1]): |
| 188 | used_rules.add(rule) |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 189 | t.outcomes |= self.wildcards[rule] |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 190 | if statusfile.DoSkip(t.outcomes): |
| 191 | skip = True |
| 192 | break # "for rule in self.wildcards" |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 193 | slow = slow or statusfile.IsSlow(t.outcomes) |
| 194 | pass_fail = pass_fail or statusfile.IsPassOrFail(t.outcomes) |
Ben Murdoch | da12d29 | 2016-06-02 14:46:10 +0100 | [diff] [blame^] | 195 | if (skip |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 196 | or self._FilterSlow(slow, slow_tests) |
| 197 | or self._FilterPassFail(pass_fail, pass_fail_tests)): |
| 198 | continue # "for t in self.tests" |
| 199 | filtered.append(t) |
| 200 | self.tests = filtered |
| 201 | |
| 202 | if not warn_unused_rules: |
| 203 | return |
| 204 | |
| 205 | for rule in self.rules: |
| 206 | if rule not in used_rules: |
| 207 | print("Unused rule: %s -> %s" % (rule, self.rules[rule])) |
| 208 | for rule in self.wildcards: |
| 209 | if rule not in used_rules: |
| 210 | print("Unused rule: %s -> %s" % (rule, self.wildcards[rule])) |
| 211 | |
| 212 | def FilterTestCasesByArgs(self, args): |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 213 | """Filter test cases based on command-line arguments. |
| 214 | |
| 215 | An argument with an asterisk in the end will match all test cases |
| 216 | that have the argument as a prefix. Without asterisk, only exact matches |
| 217 | will be used with the exeption of the test-suite name as argument. |
| 218 | """ |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 219 | filtered = [] |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 220 | globs = [] |
| 221 | exact_matches = [] |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 222 | for a in args: |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 223 | argpath = a.split('/') |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 224 | if argpath[0] != self.name: |
| 225 | continue |
| 226 | if len(argpath) == 1 or (len(argpath) == 2 and argpath[1] == '*'): |
| 227 | return # Don't filter, run all tests in this suite. |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 228 | path = '/'.join(argpath[1:]) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 229 | if path[-1] == '*': |
| 230 | path = path[:-1] |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 231 | globs.append(path) |
| 232 | else: |
| 233 | exact_matches.append(path) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 234 | for t in self.tests: |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 235 | for a in globs: |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 236 | if t.path.startswith(a): |
| 237 | filtered.append(t) |
| 238 | break |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 239 | for a in exact_matches: |
| 240 | if t.path == a: |
| 241 | filtered.append(t) |
| 242 | break |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 243 | self.tests = filtered |
| 244 | |
| 245 | def GetFlagsForTestCase(self, testcase, context): |
| 246 | raise NotImplementedError |
| 247 | |
| 248 | def GetSourceForTest(self, testcase): |
| 249 | return "(no source available)" |
| 250 | |
Ben Murdoch | da12d29 | 2016-06-02 14:46:10 +0100 | [diff] [blame^] | 251 | def IsFailureOutput(self, testcase): |
| 252 | return testcase.output.exit_code != 0 |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 253 | |
| 254 | def IsNegativeTest(self, testcase): |
| 255 | return False |
| 256 | |
| 257 | def HasFailed(self, testcase): |
Ben Murdoch | da12d29 | 2016-06-02 14:46:10 +0100 | [diff] [blame^] | 258 | execution_failed = self.IsFailureOutput(testcase) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 259 | if self.IsNegativeTest(testcase): |
| 260 | return not execution_failed |
| 261 | else: |
| 262 | return execution_failed |
| 263 | |
| 264 | def GetOutcome(self, testcase): |
| 265 | if testcase.output.HasCrashed(): |
| 266 | return statusfile.CRASH |
| 267 | elif testcase.output.HasTimedOut(): |
| 268 | return statusfile.TIMEOUT |
| 269 | elif self.HasFailed(testcase): |
| 270 | return statusfile.FAIL |
| 271 | else: |
| 272 | return statusfile.PASS |
| 273 | |
| 274 | def HasUnexpectedOutput(self, testcase): |
| 275 | outcome = self.GetOutcome(testcase) |
| 276 | return not outcome in (testcase.outcomes or [statusfile.PASS]) |
| 277 | |
| 278 | def StripOutputForTransmit(self, testcase): |
| 279 | if not self.HasUnexpectedOutput(testcase): |
| 280 | testcase.output.stdout = "" |
| 281 | testcase.output.stderr = "" |
| 282 | |
| 283 | def CalculateTotalDuration(self): |
| 284 | self.total_duration = 0.0 |
| 285 | for t in self.tests: |
| 286 | self.total_duration += t.duration |
| 287 | return self.total_duration |
| 288 | |
| 289 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 290 | class StandardVariantGenerator(VariantGenerator): |
| 291 | def FilterVariantsByTest(self, testcase): |
| 292 | return self.standard_variant |
| 293 | |
| 294 | |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 295 | class GoogleTestSuite(TestSuite): |
| 296 | def __init__(self, name, root): |
| 297 | super(GoogleTestSuite, self).__init__(name, root) |
| 298 | |
| 299 | def ListTests(self, context): |
| 300 | shell = os.path.abspath(os.path.join(context.shell_dir, self.shell())) |
| 301 | if utils.IsWindows(): |
| 302 | shell += ".exe" |
| 303 | output = commands.Execute(context.command_prefix + |
| 304 | [shell, "--gtest_list_tests"] + |
| 305 | context.extra_flags) |
| 306 | if output.exit_code != 0: |
| 307 | print output.stdout |
| 308 | print output.stderr |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 309 | raise Exception("Test executable failed to list the tests.") |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 310 | tests = [] |
| 311 | test_case = '' |
| 312 | for line in output.stdout.splitlines(): |
| 313 | test_desc = line.strip().split()[0] |
| 314 | if test_desc.endswith('.'): |
| 315 | test_case = test_desc |
| 316 | elif test_case and test_desc: |
Ben Murdoch | da12d29 | 2016-06-02 14:46:10 +0100 | [diff] [blame^] | 317 | test = testcase.TestCase(self, test_case + test_desc) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 318 | tests.append(test) |
Ben Murdoch | da12d29 | 2016-06-02 14:46:10 +0100 | [diff] [blame^] | 319 | tests.sort(key=lambda t: t.path) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 320 | return tests |
| 321 | |
| 322 | def GetFlagsForTestCase(self, testcase, context): |
| 323 | return (testcase.flags + ["--gtest_filter=" + testcase.path] + |
| 324 | ["--gtest_random_seed=%s" % context.random_seed] + |
| 325 | ["--gtest_print_time=0"] + |
| 326 | context.mode_flags) |
| 327 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 328 | def _VariantGeneratorFactory(self): |
| 329 | return StandardVariantGenerator |
| 330 | |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 331 | def shell(self): |
| 332 | return self.name |