Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | # |
| 3 | # Copyright 2012 the V8 project authors. All rights reserved. |
| 4 | # Redistribution and use in source and binary forms, with or without |
| 5 | # modification, are permitted provided that the following conditions are |
| 6 | # met: |
| 7 | # |
| 8 | # * Redistributions of source code must retain the above copyright |
| 9 | # notice, this list of conditions and the following disclaimer. |
| 10 | # * Redistributions in binary form must reproduce the above |
| 11 | # copyright notice, this list of conditions and the following |
| 12 | # disclaimer in the documentation and/or other materials provided |
| 13 | # with the distribution. |
| 14 | # * Neither the name of Google Inc. nor the names of its |
| 15 | # contributors may be used to endorse or promote products derived |
| 16 | # from this software without specific prior written permission. |
| 17 | # |
| 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 19 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 20 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 21 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 22 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 23 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 24 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 25 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 26 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 27 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 28 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 29 | |
| 30 | |
| 31 | from collections import OrderedDict |
| 32 | import itertools |
| 33 | import multiprocessing |
| 34 | import optparse |
| 35 | import os |
| 36 | from os.path import join |
| 37 | import platform |
| 38 | import random |
| 39 | import shlex |
| 40 | import subprocess |
| 41 | import sys |
| 42 | import time |
| 43 | |
| 44 | from testrunner.local import execution |
| 45 | from testrunner.local import progress |
| 46 | from testrunner.local import testsuite |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 47 | from testrunner.local.testsuite import ALL_VARIANTS |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 48 | from testrunner.local import utils |
| 49 | from testrunner.local import verbose |
| 50 | from testrunner.network import network_execution |
| 51 | from testrunner.objects import context |
| 52 | |
| 53 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 54 | # Base dir of the v8 checkout to be used as cwd. |
| 55 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) |
| 56 | |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 57 | ARCH_GUESS = utils.DefaultArch() |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 58 | |
| 59 | # Map of test name synonyms to lists of test suites. Should be ordered by |
| 60 | # expected runtimes (suites with slow test cases first). These groups are |
| 61 | # invoked in seperate steps on the bots. |
| 62 | TEST_MAP = { |
Ben Murdoch | 097c5b2 | 2016-05-18 11:27:45 +0100 | [diff] [blame] | 63 | # This needs to stay in sync with test/bot_default.isolate. |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 64 | "bot_default": [ |
| 65 | "mjsunit", |
| 66 | "cctest", |
| 67 | "webkit", |
Ben Murdoch | 097c5b2 | 2016-05-18 11:27:45 +0100 | [diff] [blame] | 68 | "fuzzer", |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 69 | "message", |
| 70 | "preparser", |
| 71 | "intl", |
| 72 | "unittests", |
| 73 | ], |
Ben Murdoch | 097c5b2 | 2016-05-18 11:27:45 +0100 | [diff] [blame] | 74 | # This needs to stay in sync with test/default.isolate. |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 75 | "default": [ |
| 76 | "mjsunit", |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 77 | "cctest", |
Ben Murdoch | 097c5b2 | 2016-05-18 11:27:45 +0100 | [diff] [blame] | 78 | "fuzzer", |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 79 | "message", |
| 80 | "preparser", |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 81 | "intl", |
| 82 | "unittests", |
| 83 | ], |
Ben Murdoch | 097c5b2 | 2016-05-18 11:27:45 +0100 | [diff] [blame] | 84 | # This needs to stay in sync with test/ignition.isolate. |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 85 | "ignition": [ |
| 86 | "mjsunit", |
| 87 | "cctest", |
Ben Murdoch | da12d29 | 2016-06-02 14:46:10 +0100 | [diff] [blame] | 88 | "webkit", |
| 89 | "message", |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 90 | ], |
Ben Murdoch | 097c5b2 | 2016-05-18 11:27:45 +0100 | [diff] [blame] | 91 | # This needs to stay in sync with test/optimize_for_size.isolate. |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 92 | "optimize_for_size": [ |
| 93 | "mjsunit", |
| 94 | "cctest", |
| 95 | "webkit", |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 96 | "intl", |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 97 | ], |
| 98 | "unittests": [ |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 99 | "unittests", |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 100 | ], |
| 101 | } |
| 102 | |
| 103 | TIMEOUT_DEFAULT = 60 |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 104 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 105 | VARIANTS = ["default", "stress", "turbofan"] |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 106 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 107 | EXHAUSTIVE_VARIANTS = VARIANTS + [ |
Ben Murdoch | c561043 | 2016-08-08 18:44:38 +0100 | [diff] [blame] | 108 | "ignition", |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 109 | "nocrankshaft", |
| 110 | "turbofan_opt", |
| 111 | ] |
| 112 | |
| 113 | DEBUG_FLAGS = ["--nohard-abort", "--nodead-code-elimination", |
| 114 | "--nofold-constants", "--enable-slow-asserts", |
| 115 | "--debug-code", "--verify-heap"] |
| 116 | RELEASE_FLAGS = ["--nohard-abort", "--nodead-code-elimination", |
| 117 | "--nofold-constants"] |
| 118 | |
| 119 | MODES = { |
| 120 | "debug": { |
| 121 | "flags": DEBUG_FLAGS, |
| 122 | "timeout_scalefactor": 4, |
| 123 | "status_mode": "debug", |
| 124 | "execution_mode": "debug", |
| 125 | "output_folder": "debug", |
| 126 | }, |
| 127 | "optdebug": { |
| 128 | "flags": DEBUG_FLAGS, |
| 129 | "timeout_scalefactor": 4, |
| 130 | "status_mode": "debug", |
| 131 | "execution_mode": "debug", |
| 132 | "output_folder": "optdebug", |
| 133 | }, |
| 134 | "release": { |
| 135 | "flags": RELEASE_FLAGS, |
| 136 | "timeout_scalefactor": 1, |
| 137 | "status_mode": "release", |
| 138 | "execution_mode": "release", |
| 139 | "output_folder": "release", |
| 140 | }, |
| 141 | # Normal trybot release configuration. There, dchecks are always on which |
| 142 | # implies debug is set. Hence, the status file needs to assume debug-like |
| 143 | # behavior/timeouts. |
| 144 | "tryrelease": { |
| 145 | "flags": RELEASE_FLAGS, |
| 146 | "timeout_scalefactor": 1, |
| 147 | "status_mode": "debug", |
| 148 | "execution_mode": "release", |
| 149 | "output_folder": "release", |
| 150 | }, |
| 151 | # This mode requires v8 to be compiled with dchecks and slow dchecks. |
| 152 | "slowrelease": { |
| 153 | "flags": RELEASE_FLAGS + ["--enable-slow-asserts"], |
| 154 | "timeout_scalefactor": 2, |
| 155 | "status_mode": "debug", |
| 156 | "execution_mode": "release", |
| 157 | "output_folder": "release", |
| 158 | }, |
| 159 | } |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 160 | |
| 161 | GC_STRESS_FLAGS = ["--gc-interval=500", "--stress-compaction", |
| 162 | "--concurrent-recompilation-queue-length=64", |
| 163 | "--concurrent-recompilation-delay=500", |
| 164 | "--concurrent-recompilation"] |
| 165 | |
| 166 | SUPPORTED_ARCHS = ["android_arm", |
| 167 | "android_arm64", |
| 168 | "android_ia32", |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 169 | "android_x64", |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 170 | "arm", |
| 171 | "ia32", |
| 172 | "x87", |
| 173 | "mips", |
| 174 | "mipsel", |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 175 | "mips64", |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 176 | "mips64el", |
| 177 | "nacl_ia32", |
| 178 | "nacl_x64", |
Ben Murdoch | da12d29 | 2016-06-02 14:46:10 +0100 | [diff] [blame] | 179 | "s390", |
| 180 | "s390x", |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 181 | "ppc", |
| 182 | "ppc64", |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 183 | "x64", |
| 184 | "x32", |
| 185 | "arm64"] |
| 186 | # Double the timeout for these: |
| 187 | SLOW_ARCHS = ["android_arm", |
| 188 | "android_arm64", |
| 189 | "android_ia32", |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 190 | "android_x64", |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 191 | "arm", |
| 192 | "mips", |
| 193 | "mipsel", |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 194 | "mips64", |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 195 | "mips64el", |
| 196 | "nacl_ia32", |
| 197 | "nacl_x64", |
Ben Murdoch | c561043 | 2016-08-08 18:44:38 +0100 | [diff] [blame] | 198 | "s390", |
| 199 | "s390x", |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 200 | "x87", |
| 201 | "arm64"] |
| 202 | |
| 203 | |
| 204 | def BuildOptions(): |
| 205 | result = optparse.OptionParser() |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 206 | result.usage = '%prog [options] [tests]' |
| 207 | result.description = """TESTS: %s""" % (TEST_MAP["default"]) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 208 | result.add_option("--arch", |
| 209 | help=("The architecture to run tests for, " |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 210 | "'auto' or 'native' for auto-detect: %s" % SUPPORTED_ARCHS), |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 211 | default="ia32,x64,arm") |
| 212 | result.add_option("--arch-and-mode", |
| 213 | help="Architecture and mode in the format 'arch.mode'", |
| 214 | default=None) |
| 215 | result.add_option("--asan", |
| 216 | help="Regard test expectations for ASAN", |
| 217 | default=False, action="store_true") |
Ben Murdoch | da12d29 | 2016-06-02 14:46:10 +0100 | [diff] [blame] | 218 | result.add_option("--sancov-dir", |
| 219 | help="Directory where to collect coverage data") |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 220 | result.add_option("--cfi-vptr", |
| 221 | help="Run tests with UBSAN cfi_vptr option.", |
| 222 | default=False, action="store_true") |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 223 | result.add_option("--buildbot", |
| 224 | help="Adapt to path structure used on buildbots", |
| 225 | default=False, action="store_true") |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 226 | result.add_option("--dcheck-always-on", |
| 227 | help="Indicates that V8 was compiled with DCHECKs enabled", |
| 228 | default=False, action="store_true") |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 229 | result.add_option("--novfp3", |
| 230 | help="Indicates that V8 was compiled without VFP3 support", |
| 231 | default=False, action="store_true") |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 232 | result.add_option("--cat", help="Print the source of the tests", |
| 233 | default=False, action="store_true") |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 234 | result.add_option("--slow-tests", |
| 235 | help="Regard slow tests (run|skip|dontcare)", |
| 236 | default="dontcare") |
| 237 | result.add_option("--pass-fail-tests", |
| 238 | help="Regard pass|fail tests (run|skip|dontcare)", |
| 239 | default="dontcare") |
| 240 | result.add_option("--gc-stress", |
| 241 | help="Switch on GC stress mode", |
| 242 | default=False, action="store_true") |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 243 | result.add_option("--gcov-coverage", |
| 244 | help="Uses executables instrumented for gcov coverage", |
| 245 | default=False, action="store_true") |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 246 | result.add_option("--command-prefix", |
| 247 | help="Prepended to each shell command used to run a test", |
| 248 | default="") |
| 249 | result.add_option("--download-data", help="Download missing test suite data", |
| 250 | default=False, action="store_true") |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 251 | result.add_option("--download-data-only", |
| 252 | help="Download missing test suite data and exit", |
| 253 | default=False, action="store_true") |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 254 | result.add_option("--extra-flags", |
| 255 | help="Additional flags to pass to each test command", |
| 256 | default="") |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 257 | result.add_option("--ignition", help="Skip tests which don't run in ignition", |
| 258 | default=False, action="store_true") |
Ben Murdoch | c561043 | 2016-08-08 18:44:38 +0100 | [diff] [blame] | 259 | result.add_option("--ignition-turbofan", |
| 260 | help="Skip tests which don't run in ignition_turbofan", |
| 261 | default=False, action="store_true") |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 262 | result.add_option("--isolates", help="Whether to test isolates", |
| 263 | default=False, action="store_true") |
| 264 | result.add_option("-j", help="The number of parallel tasks to run", |
| 265 | default=0, type="int") |
| 266 | result.add_option("-m", "--mode", |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 267 | help="The test modes in which to run (comma-separated," |
| 268 | " uppercase for ninja and buildbot builds): %s" % MODES.keys(), |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 269 | default="release,debug") |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 270 | result.add_option("--no-harness", "--noharness", |
| 271 | help="Run without test harness of a given suite", |
| 272 | default=False, action="store_true") |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 273 | result.add_option("--no-i18n", "--noi18n", |
| 274 | help="Skip internationalization tests", |
| 275 | default=False, action="store_true") |
| 276 | result.add_option("--no-network", "--nonetwork", |
| 277 | help="Don't distribute tests on the network", |
| 278 | default=(utils.GuessOS() != "linux"), |
| 279 | dest="no_network", action="store_true") |
| 280 | result.add_option("--no-presubmit", "--nopresubmit", |
| 281 | help='Skip presubmit checks', |
| 282 | default=False, dest="no_presubmit", action="store_true") |
| 283 | result.add_option("--no-snap", "--nosnap", |
| 284 | help='Test a build compiled without snapshot.', |
| 285 | default=False, dest="no_snap", action="store_true") |
| 286 | result.add_option("--no-sorting", "--nosorting", |
| 287 | help="Don't sort tests according to duration of last run.", |
| 288 | default=False, dest="no_sorting", action="store_true") |
| 289 | result.add_option("--no-stress", "--nostress", |
| 290 | help="Don't run crankshaft --always-opt --stress-op test", |
| 291 | default=False, dest="no_stress", action="store_true") |
| 292 | result.add_option("--no-variants", "--novariants", |
| 293 | help="Don't run any testing variants", |
| 294 | default=False, dest="no_variants", action="store_true") |
| 295 | result.add_option("--variants", |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 296 | help="Comma-separated list of testing variants: %s" % VARIANTS) |
| 297 | result.add_option("--exhaustive-variants", |
| 298 | default=False, action="store_true", |
| 299 | help="Use exhaustive set of default variants.") |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 300 | result.add_option("--outdir", help="Base directory with compile output", |
| 301 | default="out") |
| 302 | result.add_option("--predictable", |
| 303 | help="Compare output of several reruns of each test", |
| 304 | default=False, action="store_true") |
| 305 | result.add_option("-p", "--progress", |
| 306 | help=("The style of progress indicator" |
| 307 | " (verbose, dots, color, mono)"), |
| 308 | choices=progress.PROGRESS_INDICATORS.keys(), default="mono") |
| 309 | result.add_option("--quickcheck", default=False, action="store_true", |
Ben Murdoch | da12d29 | 2016-06-02 14:46:10 +0100 | [diff] [blame] | 310 | help=("Quick check mode (skip slow tests)")) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 311 | result.add_option("--report", help="Print a summary of the tests to be run", |
| 312 | default=False, action="store_true") |
| 313 | result.add_option("--json-test-results", |
| 314 | help="Path to a file for storing json results.") |
| 315 | result.add_option("--rerun-failures-count", |
| 316 | help=("Number of times to rerun each failing test case. " |
| 317 | "Very slow tests will be rerun only once."), |
| 318 | default=0, type="int") |
| 319 | result.add_option("--rerun-failures-max", |
| 320 | help="Maximum number of failing test cases to rerun.", |
| 321 | default=100, type="int") |
| 322 | result.add_option("--shard-count", |
| 323 | help="Split testsuites into this number of shards", |
| 324 | default=1, type="int") |
| 325 | result.add_option("--shard-run", |
| 326 | help="Run this shard from the split up tests.", |
| 327 | default=1, type="int") |
| 328 | result.add_option("--shell", help="DEPRECATED! use --shell-dir", default="") |
| 329 | result.add_option("--shell-dir", help="Directory containing executables", |
| 330 | default="") |
| 331 | result.add_option("--dont-skip-slow-simulator-tests", |
| 332 | help="Don't skip more slow tests when using a simulator.", |
| 333 | default=False, action="store_true", |
| 334 | dest="dont_skip_simulator_slow_tests") |
| 335 | result.add_option("--stress-only", |
| 336 | help="Only run tests with --always-opt --stress-opt", |
| 337 | default=False, action="store_true") |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 338 | result.add_option("--swarming", |
| 339 | help="Indicates running test driver on swarming.", |
| 340 | default=False, action="store_true") |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 341 | result.add_option("--time", help="Print timing information after running", |
| 342 | default=False, action="store_true") |
| 343 | result.add_option("-t", "--timeout", help="Timeout in seconds", |
Ben Murdoch | c561043 | 2016-08-08 18:44:38 +0100 | [diff] [blame] | 344 | default=TIMEOUT_DEFAULT, type="int") |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 345 | result.add_option("--tsan", |
| 346 | help="Regard test expectations for TSAN", |
| 347 | default=False, action="store_true") |
| 348 | result.add_option("-v", "--verbose", help="Verbose output", |
| 349 | default=False, action="store_true") |
| 350 | result.add_option("--valgrind", help="Run tests through valgrind", |
| 351 | default=False, action="store_true") |
| 352 | result.add_option("--warn-unused", help="Report unused rules", |
| 353 | default=False, action="store_true") |
| 354 | result.add_option("--junitout", help="File name of the JUnit output") |
| 355 | result.add_option("--junittestsuite", |
| 356 | help="The testsuite name in the JUnit output file", |
| 357 | default="v8tests") |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 358 | result.add_option("--random-seed", default=0, dest="random_seed", type="int", |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 359 | help="Default seed for initializing random generator") |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 360 | result.add_option("--random-seed-stress-count", default=1, type="int", |
| 361 | dest="random_seed_stress_count", |
| 362 | help="Number of runs with different random seeds") |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 363 | result.add_option("--msan", |
| 364 | help="Regard test expectations for MSAN", |
| 365 | default=False, action="store_true") |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 366 | return result |
| 367 | |
| 368 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 369 | def RandomSeed(): |
| 370 | seed = 0 |
| 371 | while not seed: |
| 372 | seed = random.SystemRandom().randint(-2147483648, 2147483647) |
| 373 | return seed |
| 374 | |
| 375 | |
| 376 | def BuildbotToV8Mode(config): |
| 377 | """Convert buildbot build configs to configs understood by the v8 runner. |
| 378 | |
| 379 | V8 configs are always lower case and without the additional _x64 suffix for |
| 380 | 64 bit builds on windows with ninja. |
| 381 | """ |
| 382 | mode = config[:-4] if config.endswith('_x64') else config |
| 383 | return mode.lower() |
| 384 | |
| 385 | def SetupEnvironment(options): |
| 386 | """Setup additional environment variables.""" |
Ben Murdoch | c561043 | 2016-08-08 18:44:38 +0100 | [diff] [blame] | 387 | |
| 388 | # Many tests assume an English interface. |
| 389 | os.environ['LANG'] = 'en_US.UTF-8' |
| 390 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 391 | symbolizer = 'external_symbolizer_path=%s' % ( |
| 392 | os.path.join( |
| 393 | BASE_DIR, 'third_party', 'llvm-build', 'Release+Asserts', 'bin', |
| 394 | 'llvm-symbolizer', |
| 395 | ) |
| 396 | ) |
| 397 | |
| 398 | if options.asan: |
| 399 | os.environ['ASAN_OPTIONS'] = symbolizer |
| 400 | |
Ben Murdoch | da12d29 | 2016-06-02 14:46:10 +0100 | [diff] [blame] | 401 | if options.sancov_dir: |
| 402 | assert os.path.exists(options.sancov_dir) |
| 403 | os.environ['ASAN_OPTIONS'] = ":".join([ |
| 404 | 'coverage=1', |
| 405 | 'coverage_dir=%s' % options.sancov_dir, |
| 406 | symbolizer, |
| 407 | ]) |
| 408 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 409 | if options.cfi_vptr: |
| 410 | os.environ['UBSAN_OPTIONS'] = ":".join([ |
| 411 | 'print_stacktrace=1', |
| 412 | 'print_summary=1', |
| 413 | 'symbolize=1', |
| 414 | symbolizer, |
| 415 | ]) |
| 416 | |
| 417 | if options.msan: |
| 418 | os.environ['MSAN_OPTIONS'] = symbolizer |
| 419 | |
| 420 | if options.tsan: |
| 421 | suppressions_file = os.path.join( |
| 422 | BASE_DIR, 'tools', 'sanitizers', 'tsan_suppressions.txt') |
| 423 | os.environ['TSAN_OPTIONS'] = " ".join([ |
| 424 | symbolizer, |
| 425 | 'suppressions=%s' % suppressions_file, |
| 426 | 'exit_code=0', |
| 427 | 'report_thread_leaks=0', |
| 428 | 'history_size=7', |
| 429 | 'report_destroy_locked=0', |
| 430 | ]) |
| 431 | |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 432 | def ProcessOptions(options): |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 433 | global ALL_VARIANTS |
| 434 | global EXHAUSTIVE_VARIANTS |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 435 | global VARIANTS |
| 436 | |
| 437 | # Architecture and mode related stuff. |
| 438 | if options.arch_and_mode: |
| 439 | options.arch_and_mode = [arch_and_mode.split(".") |
| 440 | for arch_and_mode in options.arch_and_mode.split(",")] |
| 441 | options.arch = ",".join([tokens[0] for tokens in options.arch_and_mode]) |
| 442 | options.mode = ",".join([tokens[1] for tokens in options.arch_and_mode]) |
| 443 | options.mode = options.mode.split(",") |
| 444 | for mode in options.mode: |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 445 | if not BuildbotToV8Mode(mode) in MODES: |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 446 | print "Unknown mode %s" % mode |
| 447 | return False |
| 448 | if options.arch in ["auto", "native"]: |
| 449 | options.arch = ARCH_GUESS |
| 450 | options.arch = options.arch.split(",") |
| 451 | for arch in options.arch: |
| 452 | if not arch in SUPPORTED_ARCHS: |
| 453 | print "Unknown architecture %s" % arch |
| 454 | return False |
| 455 | |
| 456 | # Store the final configuration in arch_and_mode list. Don't overwrite |
| 457 | # predefined arch_and_mode since it is more expressive than arch and mode. |
| 458 | if not options.arch_and_mode: |
| 459 | options.arch_and_mode = itertools.product(options.arch, options.mode) |
| 460 | |
| 461 | # Special processing of other options, sorted alphabetically. |
| 462 | |
| 463 | if options.buildbot: |
| 464 | # Buildbots run presubmit tests as a separate step. |
| 465 | options.no_presubmit = True |
| 466 | options.no_network = True |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 467 | if options.download_data_only: |
| 468 | options.no_presubmit = True |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 469 | if options.command_prefix: |
| 470 | print("Specifying --command-prefix disables network distribution, " |
| 471 | "running tests locally.") |
| 472 | options.no_network = True |
| 473 | options.command_prefix = shlex.split(options.command_prefix) |
| 474 | options.extra_flags = shlex.split(options.extra_flags) |
| 475 | |
| 476 | if options.gc_stress: |
| 477 | options.extra_flags += GC_STRESS_FLAGS |
| 478 | |
| 479 | if options.asan: |
| 480 | options.extra_flags.append("--invoke-weak-callbacks") |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 481 | options.extra_flags.append("--omit-quit") |
| 482 | |
| 483 | if options.novfp3: |
| 484 | options.extra_flags.append("--noenable-vfp3") |
| 485 | |
| 486 | if options.exhaustive_variants: |
| 487 | # This is used on many bots. It includes a larger set of default variants. |
| 488 | # Other options for manipulating variants still apply afterwards. |
| 489 | VARIANTS = EXHAUSTIVE_VARIANTS |
| 490 | |
| 491 | if options.msan: |
| 492 | VARIANTS = ["default"] |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 493 | |
| 494 | if options.tsan: |
| 495 | VARIANTS = ["default"] |
| 496 | |
| 497 | if options.j == 0: |
| 498 | options.j = multiprocessing.cpu_count() |
| 499 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 500 | if options.random_seed_stress_count <= 1 and options.random_seed == 0: |
| 501 | options.random_seed = RandomSeed() |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 502 | |
| 503 | def excl(*args): |
| 504 | """Returns true if zero or one of multiple arguments are true.""" |
| 505 | return reduce(lambda x, y: x + y, args) <= 1 |
| 506 | |
| 507 | if not excl(options.no_stress, options.stress_only, options.no_variants, |
| 508 | bool(options.variants)): |
| 509 | print("Use only one of --no-stress, --stress-only, --no-variants, " |
| 510 | "or --variants.") |
| 511 | return False |
| 512 | if options.quickcheck: |
| 513 | VARIANTS = ["default", "stress"] |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 514 | options.slow_tests = "skip" |
| 515 | options.pass_fail_tests = "skip" |
| 516 | if options.no_stress: |
| 517 | VARIANTS = ["default", "nocrankshaft"] |
| 518 | if options.no_variants: |
| 519 | VARIANTS = ["default"] |
| 520 | if options.stress_only: |
| 521 | VARIANTS = ["stress"] |
| 522 | if options.variants: |
| 523 | VARIANTS = options.variants.split(",") |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 524 | if not set(VARIANTS).issubset(ALL_VARIANTS): |
| 525 | print "All variants must be in %s" % str(ALL_VARIANTS) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 526 | return False |
| 527 | if options.predictable: |
| 528 | VARIANTS = ["default"] |
| 529 | options.extra_flags.append("--predictable") |
| 530 | options.extra_flags.append("--verify_predictable") |
| 531 | options.extra_flags.append("--no-inline-new") |
| 532 | |
| 533 | if not options.shell_dir: |
| 534 | if options.shell: |
| 535 | print "Warning: --shell is deprecated, use --shell-dir instead." |
| 536 | options.shell_dir = os.path.dirname(options.shell) |
| 537 | if options.valgrind: |
| 538 | run_valgrind = os.path.join("tools", "run-valgrind.py") |
| 539 | # This is OK for distributed running, so we don't need to set no_network. |
| 540 | options.command_prefix = (["python", "-u", run_valgrind] + |
| 541 | options.command_prefix) |
| 542 | def CheckTestMode(name, option): |
| 543 | if not option in ["run", "skip", "dontcare"]: |
| 544 | print "Unknown %s mode %s" % (name, option) |
| 545 | return False |
| 546 | return True |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 547 | if not CheckTestMode("slow test", options.slow_tests): |
| 548 | return False |
| 549 | if not CheckTestMode("pass|fail test", options.pass_fail_tests): |
| 550 | return False |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 551 | if options.no_i18n: |
| 552 | TEST_MAP["bot_default"].remove("intl") |
| 553 | TEST_MAP["default"].remove("intl") |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 554 | return True |
| 555 | |
| 556 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 557 | def ShardTests(tests, options): |
| 558 | # Read gtest shard configuration from environment (e.g. set by swarming). |
| 559 | # If none is present, use values passed on the command line. |
| 560 | shard_count = int(os.environ.get('GTEST_TOTAL_SHARDS', options.shard_count)) |
| 561 | shard_run = os.environ.get('GTEST_SHARD_INDEX') |
| 562 | if shard_run is not None: |
| 563 | # The v8 shard_run starts at 1, while GTEST_SHARD_INDEX starts at 0. |
| 564 | shard_run = int(shard_run) + 1 |
| 565 | else: |
| 566 | shard_run = options.shard_run |
| 567 | |
| 568 | if options.shard_count > 1: |
| 569 | # Log if a value was passed on the cmd line and it differs from the |
| 570 | # environment variables. |
| 571 | if options.shard_count != shard_count: |
| 572 | print("shard_count from cmd line differs from environment variable " |
| 573 | "GTEST_TOTAL_SHARDS") |
| 574 | if options.shard_run > 1 and options.shard_run != shard_run: |
| 575 | print("shard_run from cmd line differs from environment variable " |
| 576 | "GTEST_SHARD_INDEX") |
| 577 | |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 578 | if shard_count < 2: |
| 579 | return tests |
| 580 | if shard_run < 1 or shard_run > shard_count: |
| 581 | print "shard-run not a valid number, should be in [1:shard-count]" |
| 582 | print "defaulting back to running all tests" |
| 583 | return tests |
| 584 | count = 0 |
| 585 | shard = [] |
| 586 | for test in tests: |
| 587 | if count % shard_count == shard_run - 1: |
| 588 | shard.append(test) |
| 589 | count += 1 |
| 590 | return shard |
| 591 | |
| 592 | |
| 593 | def Main(): |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 594 | # Use the v8 root as cwd as some test cases use "load" with relative paths. |
| 595 | os.chdir(BASE_DIR) |
| 596 | |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 597 | parser = BuildOptions() |
| 598 | (options, args) = parser.parse_args() |
| 599 | if not ProcessOptions(options): |
| 600 | parser.print_help() |
| 601 | return 1 |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 602 | SetupEnvironment(options) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 603 | |
Ben Murdoch | c561043 | 2016-08-08 18:44:38 +0100 | [diff] [blame] | 604 | if options.swarming: |
| 605 | # Swarming doesn't print how isolated commands are called. Lets make this |
| 606 | # less cryptic by printing it ourselves. |
| 607 | print ' '.join(sys.argv) |
| 608 | |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 609 | exit_code = 0 |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 610 | if not options.no_presubmit: |
| 611 | print ">>> running presubmit tests" |
| 612 | exit_code = subprocess.call( |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 613 | [sys.executable, join(BASE_DIR, "tools", "presubmit.py")]) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 614 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 615 | suite_paths = utils.GetSuitePaths(join(BASE_DIR, "test")) |
| 616 | |
| 617 | # Use default tests if no test configuration was provided at the cmd line. |
| 618 | if len(args) == 0: |
| 619 | args = ["default"] |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 620 | |
| 621 | # Expand arguments with grouped tests. The args should reflect the list of |
| 622 | # suites as otherwise filters would break. |
| 623 | def ExpandTestGroups(name): |
| 624 | if name in TEST_MAP: |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 625 | return [suite for suite in TEST_MAP[name]] |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 626 | else: |
| 627 | return [name] |
| 628 | args = reduce(lambda x, y: x + y, |
| 629 | [ExpandTestGroups(arg) for arg in args], |
| 630 | []) |
| 631 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 632 | args_suites = OrderedDict() # Used as set |
| 633 | for arg in args: |
| 634 | args_suites[arg.split('/')[0]] = True |
| 635 | suite_paths = [ s for s in args_suites if s in suite_paths ] |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 636 | |
| 637 | suites = [] |
| 638 | for root in suite_paths: |
| 639 | suite = testsuite.TestSuite.LoadTestSuite( |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 640 | os.path.join(BASE_DIR, "test", root)) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 641 | if suite: |
| 642 | suites.append(suite) |
| 643 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 644 | if options.download_data or options.download_data_only: |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 645 | for s in suites: |
| 646 | s.DownloadData() |
| 647 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 648 | if options.download_data_only: |
| 649 | return exit_code |
| 650 | |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 651 | for (arch, mode) in options.arch_and_mode: |
| 652 | try: |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 653 | code = Execute(arch, mode, args, options, suites) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 654 | except KeyboardInterrupt: |
| 655 | return 2 |
| 656 | exit_code = exit_code or code |
| 657 | return exit_code |
| 658 | |
| 659 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 660 | def Execute(arch, mode, args, options, suites): |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 661 | print(">>> Running tests for %s.%s" % (arch, mode)) |
| 662 | |
| 663 | shell_dir = options.shell_dir |
| 664 | if not shell_dir: |
| 665 | if options.buildbot: |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 666 | # TODO(machenbach): Get rid of different output folder location on |
| 667 | # buildbot. Currently this is capitalized Release and Debug. |
| 668 | shell_dir = os.path.join(BASE_DIR, options.outdir, mode) |
| 669 | mode = BuildbotToV8Mode(mode) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 670 | else: |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 671 | shell_dir = os.path.join( |
| 672 | BASE_DIR, |
| 673 | options.outdir, |
| 674 | "%s.%s" % (arch, MODES[mode]["output_folder"]), |
| 675 | ) |
| 676 | if not os.path.exists(shell_dir): |
| 677 | raise Exception('Could not find shell_dir: "%s"' % shell_dir) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 678 | |
| 679 | # Populate context object. |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 680 | mode_flags = MODES[mode]["flags"] |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 681 | |
Ben Murdoch | c561043 | 2016-08-08 18:44:38 +0100 | [diff] [blame] | 682 | # Simulators are slow, therefore allow a longer timeout. |
| 683 | if arch in SLOW_ARCHS: |
| 684 | options.timeout *= 2 |
| 685 | |
| 686 | options.timeout *= MODES[mode]["timeout_scalefactor"] |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 687 | |
| 688 | if options.predictable: |
| 689 | # Predictable mode is slower. |
Ben Murdoch | c561043 | 2016-08-08 18:44:38 +0100 | [diff] [blame] | 690 | options.timeout *= 2 |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 691 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 692 | # TODO(machenbach): Remove temporary verbose output on windows after |
| 693 | # debugging driver-hung-up on XP. |
| 694 | verbose_output = ( |
| 695 | options.verbose or |
| 696 | utils.IsWindows() and options.progress == "verbose" |
| 697 | ) |
| 698 | ctx = context.Context(arch, MODES[mode]["execution_mode"], shell_dir, |
| 699 | mode_flags, verbose_output, |
Ben Murdoch | c561043 | 2016-08-08 18:44:38 +0100 | [diff] [blame] | 700 | options.timeout, |
| 701 | options.isolates, |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 702 | options.command_prefix, |
| 703 | options.extra_flags, |
| 704 | options.no_i18n, |
| 705 | options.random_seed, |
| 706 | options.no_sorting, |
| 707 | options.rerun_failures_count, |
| 708 | options.rerun_failures_max, |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 709 | options.predictable, |
| 710 | options.no_harness, |
Ben Murdoch | da12d29 | 2016-06-02 14:46:10 +0100 | [diff] [blame] | 711 | use_perf_data=not options.swarming, |
| 712 | sancov_dir=options.sancov_dir) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 713 | |
| 714 | # TODO(all): Combine "simulator" and "simulator_run". |
| 715 | simulator_run = not options.dont_skip_simulator_slow_tests and \ |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 716 | arch in ['arm64', 'arm', 'mipsel', 'mips', 'mips64', 'mips64el', \ |
| 717 | 'ppc', 'ppc64'] and \ |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 718 | ARCH_GUESS and arch != ARCH_GUESS |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 719 | # Find available test suites and read test cases from them. |
| 720 | variables = { |
| 721 | "arch": arch, |
| 722 | "asan": options.asan, |
| 723 | "deopt_fuzzer": False, |
| 724 | "gc_stress": options.gc_stress, |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 725 | "gcov_coverage": options.gcov_coverage, |
| 726 | "ignition": options.ignition, |
Ben Murdoch | c561043 | 2016-08-08 18:44:38 +0100 | [diff] [blame] | 727 | "ignition_turbofan": options.ignition_turbofan, |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 728 | "isolates": options.isolates, |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 729 | "mode": MODES[mode]["status_mode"], |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 730 | "no_i18n": options.no_i18n, |
| 731 | "no_snap": options.no_snap, |
| 732 | "simulator_run": simulator_run, |
| 733 | "simulator": utils.UseSimulator(arch), |
| 734 | "system": utils.GuessOS(), |
| 735 | "tsan": options.tsan, |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 736 | "msan": options.msan, |
| 737 | "dcheck_always_on": options.dcheck_always_on, |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 738 | "novfp3": options.novfp3, |
| 739 | "predictable": options.predictable, |
| 740 | "byteorder": sys.byteorder, |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 741 | } |
| 742 | all_tests = [] |
| 743 | num_tests = 0 |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 744 | for s in suites: |
| 745 | s.ReadStatusFile(variables) |
| 746 | s.ReadTestCases(ctx) |
| 747 | if len(args) > 0: |
| 748 | s.FilterTestCasesByArgs(args) |
| 749 | all_tests += s.tests |
Ben Murdoch | da12d29 | 2016-06-02 14:46:10 +0100 | [diff] [blame] | 750 | s.FilterTestCasesByStatus(options.warn_unused, options.slow_tests, |
| 751 | options.pass_fail_tests) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 752 | if options.cat: |
| 753 | verbose.PrintTestSource(s.tests) |
| 754 | continue |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 755 | variant_gen = s.CreateVariantGenerator(VARIANTS) |
| 756 | variant_tests = [ t.CopyAddingFlags(v, flags) |
| 757 | for t in s.tests |
| 758 | for v in variant_gen.FilterVariantsByTest(t) |
| 759 | for flags in variant_gen.GetFlagSets(t, v) ] |
| 760 | |
| 761 | if options.random_seed_stress_count > 1: |
| 762 | # Duplicate test for random seed stress mode. |
| 763 | def iter_seed_flags(): |
| 764 | for i in range(0, options.random_seed_stress_count): |
| 765 | # Use given random seed for all runs (set by default in execution.py) |
| 766 | # or a new random seed if none is specified. |
| 767 | if options.random_seed: |
| 768 | yield [] |
| 769 | else: |
| 770 | yield ["--random-seed=%d" % RandomSeed()] |
| 771 | s.tests = [ |
| 772 | t.CopyAddingFlags(t.variant, flags) |
| 773 | for t in variant_tests |
| 774 | for flags in iter_seed_flags() |
| 775 | ] |
| 776 | else: |
| 777 | s.tests = variant_tests |
| 778 | |
| 779 | s.tests = ShardTests(s.tests, options) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 780 | num_tests += len(s.tests) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 781 | |
| 782 | if options.cat: |
| 783 | return 0 # We're done here. |
| 784 | |
| 785 | if options.report: |
| 786 | verbose.PrintReport(all_tests) |
| 787 | |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 788 | # Run the tests, either locally or distributed on the network. |
| 789 | start_time = time.time() |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 790 | progress_indicator = progress.IndicatorNotifier() |
| 791 | progress_indicator.Register(progress.PROGRESS_INDICATORS[options.progress]()) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 792 | if options.junitout: |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 793 | progress_indicator.Register(progress.JUnitTestProgressIndicator( |
| 794 | options.junitout, options.junittestsuite)) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 795 | if options.json_test_results: |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 796 | progress_indicator.Register(progress.JsonTestProgressIndicator( |
| 797 | options.json_test_results, arch, MODES[mode]["execution_mode"], |
| 798 | ctx.random_seed)) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 799 | |
| 800 | run_networked = not options.no_network |
| 801 | if not run_networked: |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 802 | if verbose_output: |
| 803 | print("Network distribution disabled, running tests locally.") |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 804 | elif utils.GuessOS() != "linux": |
| 805 | print("Network distribution is only supported on Linux, sorry!") |
| 806 | run_networked = False |
| 807 | peers = [] |
| 808 | if run_networked: |
| 809 | peers = network_execution.GetPeers() |
| 810 | if not peers: |
| 811 | print("No connection to distribution server; running tests locally.") |
| 812 | run_networked = False |
| 813 | elif len(peers) == 1: |
| 814 | print("No other peers on the network; running tests locally.") |
| 815 | run_networked = False |
| 816 | elif num_tests <= 100: |
| 817 | print("Less than 100 tests, running them locally.") |
| 818 | run_networked = False |
| 819 | |
| 820 | if run_networked: |
| 821 | runner = network_execution.NetworkedRunner(suites, progress_indicator, |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 822 | ctx, peers, BASE_DIR) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 823 | else: |
| 824 | runner = execution.Runner(suites, progress_indicator, ctx) |
| 825 | |
| 826 | exit_code = runner.Run(options.j) |
| 827 | overall_duration = time.time() - start_time |
| 828 | |
| 829 | if options.time: |
| 830 | verbose.PrintTestDurations(suites, overall_duration) |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 831 | |
| 832 | if num_tests == 0: |
| 833 | print("Warning: no tests were run!") |
| 834 | |
| 835 | if exit_code == 1 and options.json_test_results: |
| 836 | print("Force exit code 0 after failures. Json test results file generated " |
| 837 | "with failure information.") |
| 838 | exit_code = 0 |
| 839 | |
Ben Murdoch | da12d29 | 2016-06-02 14:46:10 +0100 | [diff] [blame] | 840 | if options.sancov_dir: |
| 841 | # If tests ran with sanitizer coverage, merge coverage files in the end. |
| 842 | try: |
| 843 | print "Merging sancov files." |
| 844 | subprocess.check_call([ |
| 845 | sys.executable, |
| 846 | join(BASE_DIR, "tools", "sanitizers", "sancov_merger.py"), |
| 847 | "--coverage-dir=%s" % options.sancov_dir]) |
| 848 | except: |
| 849 | print >> sys.stderr, "Error: Merging sancov files failed." |
| 850 | exit_code = 1 |
| 851 | |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 852 | return exit_code |
| 853 | |
| 854 | |
| 855 | if __name__ == "__main__": |
| 856 | sys.exit(Main()) |