Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 1 | # Copyright 2012 the V8 project authors. All rights reserved. |
| 2 | # Redistribution and use in source and binary forms, with or without |
| 3 | # modification, are permitted provided that the following conditions are |
| 4 | # met: |
| 5 | # |
| 6 | # * Redistributions of source code must retain the above copyright |
| 7 | # notice, this list of conditions and the following disclaimer. |
| 8 | # * Redistributions in binary form must reproduce the above |
| 9 | # copyright notice, this list of conditions and the following |
| 10 | # disclaimer in the documentation and/or other materials provided |
| 11 | # with the distribution. |
| 12 | # * Neither the name of Google Inc. nor the names of its |
| 13 | # contributors may be used to endorse or promote products derived |
| 14 | # from this software without specific prior written permission. |
| 15 | # |
| 16 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 17 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 18 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 19 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 20 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 21 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 22 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 23 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 24 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 25 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 26 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 27 | |
| 28 | |
| 29 | import imp |
| 30 | import os |
| 31 | |
| 32 | from . import commands |
| 33 | from . import statusfile |
| 34 | from . import utils |
| 35 | from ..objects import testcase |
| 36 | |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 37 | # Use this to run several variants of the tests. |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame^] | 38 | ALL_VARIANT_FLAGS = { |
| 39 | "default": [[]], |
| 40 | "stress": [["--stress-opt", "--always-opt"]], |
| 41 | "turbofan": [["--turbo"]], |
| 42 | "turbofan_opt": [["--turbo", "--always-opt"]], |
| 43 | "nocrankshaft": [["--nocrankshaft"]], |
| 44 | "ignition": [["--ignition", "--turbo", "--ignition-fake-try-catch", |
| 45 | "--ignition-fallback-on-eval-and-catch"]], |
| 46 | "preparser": [["--min-preparse-length=0"]], |
| 47 | } |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 48 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame^] | 49 | # FAST_VARIANTS implies no --always-opt. |
| 50 | FAST_VARIANT_FLAGS = { |
| 51 | "default": [[]], |
| 52 | "stress": [["--stress-opt"]], |
| 53 | "turbofan": [["--turbo"]], |
| 54 | "nocrankshaft": [["--nocrankshaft"]], |
| 55 | "ignition": [["--ignition", "--turbo", "--ignition-fake-try-catch", |
| 56 | "--ignition-fallback-on-eval-and-catch"]], |
| 57 | "preparser": [["--min-preparse-length=0"]], |
| 58 | } |
| 59 | |
| 60 | ALL_VARIANTS = set(["default", "stress", "turbofan", "turbofan_opt", |
| 61 | "nocrankshaft", "ignition", "preparser"]) |
| 62 | FAST_VARIANTS = set(["default", "turbofan"]) |
| 63 | STANDARD_VARIANT = set(["default"]) |
| 64 | |
| 65 | |
| 66 | class VariantGenerator(object): |
| 67 | def __init__(self, suite, variants): |
| 68 | self.suite = suite |
| 69 | self.all_variants = ALL_VARIANTS & variants |
| 70 | self.fast_variants = FAST_VARIANTS & variants |
| 71 | self.standard_variant = STANDARD_VARIANT & variants |
| 72 | |
| 73 | def FilterVariantsByTest(self, testcase): |
| 74 | if testcase.outcomes and statusfile.OnlyStandardVariant( |
| 75 | testcase.outcomes): |
| 76 | return self.standard_variant |
| 77 | if testcase.outcomes and statusfile.OnlyFastVariants(testcase.outcomes): |
| 78 | return self.fast_variants |
| 79 | return self.all_variants |
| 80 | |
| 81 | def GetFlagSets(self, testcase, variant): |
| 82 | if testcase.outcomes and statusfile.OnlyFastVariants(testcase.outcomes): |
| 83 | return FAST_VARIANT_FLAGS[variant] |
| 84 | else: |
| 85 | return ALL_VARIANT_FLAGS[variant] |
| 86 | |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 87 | |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 88 | class TestSuite(object): |
| 89 | |
| 90 | @staticmethod |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame^] | 91 | def LoadTestSuite(root, global_init=True): |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 92 | name = root.split(os.path.sep)[-1] |
| 93 | f = None |
| 94 | try: |
| 95 | (f, pathname, description) = imp.find_module("testcfg", [root]) |
| 96 | module = imp.load_module("testcfg", f, pathname, description) |
| 97 | return module.GetSuite(name, root) |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame^] | 98 | except ImportError: |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 99 | # Use default if no testcfg is present. |
| 100 | return GoogleTestSuite(name, root) |
| 101 | finally: |
| 102 | if f: |
| 103 | f.close() |
| 104 | |
| 105 | def __init__(self, name, root): |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame^] | 106 | # Note: This might be called concurrently from different processes. |
| 107 | # Changing harddisk state should be done in 'SetupWorkingDirectory' below. |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 108 | self.name = name # string |
| 109 | self.root = root # string containing path |
| 110 | self.tests = None # list of TestCase objects |
| 111 | self.rules = None # dictionary mapping test path to list of outcomes |
| 112 | self.wildcards = None # dictionary mapping test paths to list of outcomes |
| 113 | self.total_duration = None # float, assigned on demand |
| 114 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame^] | 115 | def SetupWorkingDirectory(self): |
| 116 | # This is called once per test suite object in a multi-process setting. |
| 117 | # Multi-process-unsafe work-directory setup can go here. |
| 118 | pass |
| 119 | |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 120 | def shell(self): |
| 121 | return "d8" |
| 122 | |
| 123 | def suffix(self): |
| 124 | return ".js" |
| 125 | |
| 126 | def status_file(self): |
| 127 | return "%s/%s.status" % (self.root, self.name) |
| 128 | |
| 129 | # Used in the status file and for stdout printing. |
| 130 | def CommonTestName(self, testcase): |
| 131 | if utils.IsWindows(): |
| 132 | return testcase.path.replace("\\", "/") |
| 133 | else: |
| 134 | return testcase.path |
| 135 | |
| 136 | def ListTests(self, context): |
| 137 | raise NotImplementedError |
| 138 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame^] | 139 | def _VariantGeneratorFactory(self): |
| 140 | """The variant generator class to be used.""" |
| 141 | return VariantGenerator |
| 142 | |
| 143 | def CreateVariantGenerator(self, variants): |
| 144 | """Return a generator for the testing variants of this suite. |
| 145 | |
| 146 | Args: |
| 147 | variants: List of variant names to be run as specified by the test |
| 148 | runner. |
| 149 | Returns: An object of type VariantGenerator. |
| 150 | """ |
| 151 | return self._VariantGeneratorFactory()(self, set(variants)) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 152 | |
| 153 | def DownloadData(self): |
| 154 | pass |
| 155 | |
| 156 | def ReadStatusFile(self, variables): |
| 157 | (self.rules, self.wildcards) = \ |
| 158 | statusfile.ReadStatusFile(self.status_file(), variables) |
| 159 | |
| 160 | def ReadTestCases(self, context): |
| 161 | self.tests = self.ListTests(context) |
| 162 | |
| 163 | @staticmethod |
| 164 | def _FilterFlaky(flaky, mode): |
| 165 | return (mode == "run" and not flaky) or (mode == "skip" and flaky) |
| 166 | |
| 167 | @staticmethod |
| 168 | def _FilterSlow(slow, mode): |
| 169 | return (mode == "run" and not slow) or (mode == "skip" and slow) |
| 170 | |
| 171 | @staticmethod |
| 172 | def _FilterPassFail(pass_fail, mode): |
| 173 | return (mode == "run" and not pass_fail) or (mode == "skip" and pass_fail) |
| 174 | |
| 175 | def FilterTestCasesByStatus(self, warn_unused_rules, |
| 176 | flaky_tests="dontcare", |
| 177 | slow_tests="dontcare", |
| 178 | pass_fail_tests="dontcare"): |
| 179 | filtered = [] |
| 180 | used_rules = set() |
| 181 | for t in self.tests: |
| 182 | flaky = False |
| 183 | slow = False |
| 184 | pass_fail = False |
| 185 | testname = self.CommonTestName(t) |
| 186 | if testname in self.rules: |
| 187 | used_rules.add(testname) |
| 188 | # Even for skipped tests, as the TestCase object stays around and |
| 189 | # PrintReport() uses it. |
| 190 | t.outcomes = self.rules[testname] |
| 191 | if statusfile.DoSkip(t.outcomes): |
| 192 | continue # Don't add skipped tests to |filtered|. |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 193 | for outcome in t.outcomes: |
| 194 | if outcome.startswith('Flags: '): |
| 195 | t.flags += outcome[7:].split() |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 196 | flaky = statusfile.IsFlaky(t.outcomes) |
| 197 | slow = statusfile.IsSlow(t.outcomes) |
| 198 | pass_fail = statusfile.IsPassOrFail(t.outcomes) |
| 199 | skip = False |
| 200 | for rule in self.wildcards: |
| 201 | assert rule[-1] == '*' |
| 202 | if testname.startswith(rule[:-1]): |
| 203 | used_rules.add(rule) |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame^] | 204 | t.outcomes |= self.wildcards[rule] |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 205 | if statusfile.DoSkip(t.outcomes): |
| 206 | skip = True |
| 207 | break # "for rule in self.wildcards" |
| 208 | flaky = flaky or statusfile.IsFlaky(t.outcomes) |
| 209 | slow = slow or statusfile.IsSlow(t.outcomes) |
| 210 | pass_fail = pass_fail or statusfile.IsPassOrFail(t.outcomes) |
| 211 | if (skip or self._FilterFlaky(flaky, flaky_tests) |
| 212 | or self._FilterSlow(slow, slow_tests) |
| 213 | or self._FilterPassFail(pass_fail, pass_fail_tests)): |
| 214 | continue # "for t in self.tests" |
| 215 | filtered.append(t) |
| 216 | self.tests = filtered |
| 217 | |
| 218 | if not warn_unused_rules: |
| 219 | return |
| 220 | |
| 221 | for rule in self.rules: |
| 222 | if rule not in used_rules: |
| 223 | print("Unused rule: %s -> %s" % (rule, self.rules[rule])) |
| 224 | for rule in self.wildcards: |
| 225 | if rule not in used_rules: |
| 226 | print("Unused rule: %s -> %s" % (rule, self.wildcards[rule])) |
| 227 | |
| 228 | def FilterTestCasesByArgs(self, args): |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame^] | 229 | """Filter test cases based on command-line arguments. |
| 230 | |
| 231 | An argument with an asterisk in the end will match all test cases |
| 232 | that have the argument as a prefix. Without asterisk, only exact matches |
| 233 | will be used with the exeption of the test-suite name as argument. |
| 234 | """ |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 235 | filtered = [] |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame^] | 236 | globs = [] |
| 237 | exact_matches = [] |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 238 | for a in args: |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame^] | 239 | argpath = a.split('/') |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 240 | if argpath[0] != self.name: |
| 241 | continue |
| 242 | if len(argpath) == 1 or (len(argpath) == 2 and argpath[1] == '*'): |
| 243 | return # Don't filter, run all tests in this suite. |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame^] | 244 | path = '/'.join(argpath[1:]) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 245 | if path[-1] == '*': |
| 246 | path = path[:-1] |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame^] | 247 | globs.append(path) |
| 248 | else: |
| 249 | exact_matches.append(path) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 250 | for t in self.tests: |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame^] | 251 | for a in globs: |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 252 | if t.path.startswith(a): |
| 253 | filtered.append(t) |
| 254 | break |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame^] | 255 | for a in exact_matches: |
| 256 | if t.path == a: |
| 257 | filtered.append(t) |
| 258 | break |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 259 | self.tests = filtered |
| 260 | |
| 261 | def GetFlagsForTestCase(self, testcase, context): |
| 262 | raise NotImplementedError |
| 263 | |
| 264 | def GetSourceForTest(self, testcase): |
| 265 | return "(no source available)" |
| 266 | |
| 267 | def IsFailureOutput(self, output, testpath): |
| 268 | return output.exit_code != 0 |
| 269 | |
| 270 | def IsNegativeTest(self, testcase): |
| 271 | return False |
| 272 | |
| 273 | def HasFailed(self, testcase): |
| 274 | execution_failed = self.IsFailureOutput(testcase.output, testcase.path) |
| 275 | if self.IsNegativeTest(testcase): |
| 276 | return not execution_failed |
| 277 | else: |
| 278 | return execution_failed |
| 279 | |
| 280 | def GetOutcome(self, testcase): |
| 281 | if testcase.output.HasCrashed(): |
| 282 | return statusfile.CRASH |
| 283 | elif testcase.output.HasTimedOut(): |
| 284 | return statusfile.TIMEOUT |
| 285 | elif self.HasFailed(testcase): |
| 286 | return statusfile.FAIL |
| 287 | else: |
| 288 | return statusfile.PASS |
| 289 | |
| 290 | def HasUnexpectedOutput(self, testcase): |
| 291 | outcome = self.GetOutcome(testcase) |
| 292 | return not outcome in (testcase.outcomes or [statusfile.PASS]) |
| 293 | |
| 294 | def StripOutputForTransmit(self, testcase): |
| 295 | if not self.HasUnexpectedOutput(testcase): |
| 296 | testcase.output.stdout = "" |
| 297 | testcase.output.stderr = "" |
| 298 | |
| 299 | def CalculateTotalDuration(self): |
| 300 | self.total_duration = 0.0 |
| 301 | for t in self.tests: |
| 302 | self.total_duration += t.duration |
| 303 | return self.total_duration |
| 304 | |
| 305 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame^] | 306 | class StandardVariantGenerator(VariantGenerator): |
| 307 | def FilterVariantsByTest(self, testcase): |
| 308 | return self.standard_variant |
| 309 | |
| 310 | |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 311 | class GoogleTestSuite(TestSuite): |
| 312 | def __init__(self, name, root): |
| 313 | super(GoogleTestSuite, self).__init__(name, root) |
| 314 | |
| 315 | def ListTests(self, context): |
| 316 | shell = os.path.abspath(os.path.join(context.shell_dir, self.shell())) |
| 317 | if utils.IsWindows(): |
| 318 | shell += ".exe" |
| 319 | output = commands.Execute(context.command_prefix + |
| 320 | [shell, "--gtest_list_tests"] + |
| 321 | context.extra_flags) |
| 322 | if output.exit_code != 0: |
| 323 | print output.stdout |
| 324 | print output.stderr |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 325 | raise Exception("Test executable failed to list the tests.") |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 326 | tests = [] |
| 327 | test_case = '' |
| 328 | for line in output.stdout.splitlines(): |
| 329 | test_desc = line.strip().split()[0] |
| 330 | if test_desc.endswith('.'): |
| 331 | test_case = test_desc |
| 332 | elif test_case and test_desc: |
| 333 | test = testcase.TestCase(self, test_case + test_desc, dependency=None) |
| 334 | tests.append(test) |
| 335 | tests.sort() |
| 336 | return tests |
| 337 | |
| 338 | def GetFlagsForTestCase(self, testcase, context): |
| 339 | return (testcase.flags + ["--gtest_filter=" + testcase.path] + |
| 340 | ["--gtest_random_seed=%s" % context.random_seed] + |
| 341 | ["--gtest_print_time=0"] + |
| 342 | context.mode_flags) |
| 343 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame^] | 344 | def _VariantGeneratorFactory(self): |
| 345 | return StandardVariantGenerator |
| 346 | |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 347 | def shell(self): |
| 348 | return self.name |