Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | # |
| 3 | # Copyright 2012 the V8 project authors. All rights reserved. |
| 4 | # Redistribution and use in source and binary forms, with or without |
| 5 | # modification, are permitted provided that the following conditions are |
| 6 | # met: |
| 7 | # |
| 8 | # * Redistributions of source code must retain the above copyright |
| 9 | # notice, this list of conditions and the following disclaimer. |
| 10 | # * Redistributions in binary form must reproduce the above |
| 11 | # copyright notice, this list of conditions and the following |
| 12 | # disclaimer in the documentation and/or other materials provided |
| 13 | # with the distribution. |
| 14 | # * Neither the name of Google Inc. nor the names of its |
| 15 | # contributors may be used to endorse or promote products derived |
| 16 | # from this software without specific prior written permission. |
| 17 | # |
| 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 19 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 20 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 21 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 22 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 23 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 24 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 25 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 26 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 27 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 28 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 29 | |
| 30 | |
| 31 | from collections import OrderedDict |
| 32 | import itertools |
| 33 | import multiprocessing |
| 34 | import optparse |
| 35 | import os |
| 36 | from os.path import join |
| 37 | import platform |
| 38 | import random |
| 39 | import shlex |
| 40 | import subprocess |
| 41 | import sys |
| 42 | import time |
| 43 | |
| 44 | from testrunner.local import execution |
| 45 | from testrunner.local import progress |
| 46 | from testrunner.local import testsuite |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 47 | from testrunner.local.testsuite import ALL_VARIANTS |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 48 | from testrunner.local import utils |
| 49 | from testrunner.local import verbose |
| 50 | from testrunner.network import network_execution |
| 51 | from testrunner.objects import context |
| 52 | |
| 53 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 54 | # Base dir of the v8 checkout to be used as cwd. |
| 55 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) |
| 56 | |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 57 | ARCH_GUESS = utils.DefaultArch() |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 58 | |
| 59 | # Map of test name synonyms to lists of test suites. Should be ordered by |
| 60 | # expected runtimes (suites with slow test cases first). These groups are |
| 61 | # invoked in seperate steps on the bots. |
| 62 | TEST_MAP = { |
Ben Murdoch | 097c5b2 | 2016-05-18 11:27:45 +0100 | [diff] [blame] | 63 | # This needs to stay in sync with test/bot_default.isolate. |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 64 | "bot_default": [ |
| 65 | "mjsunit", |
| 66 | "cctest", |
| 67 | "webkit", |
Ben Murdoch | 097c5b2 | 2016-05-18 11:27:45 +0100 | [diff] [blame] | 68 | "fuzzer", |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 69 | "message", |
| 70 | "preparser", |
| 71 | "intl", |
| 72 | "unittests", |
| 73 | ], |
Ben Murdoch | 097c5b2 | 2016-05-18 11:27:45 +0100 | [diff] [blame] | 74 | # This needs to stay in sync with test/default.isolate. |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 75 | "default": [ |
| 76 | "mjsunit", |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 77 | "cctest", |
Ben Murdoch | 097c5b2 | 2016-05-18 11:27:45 +0100 | [diff] [blame] | 78 | "fuzzer", |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 79 | "message", |
| 80 | "preparser", |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 81 | "intl", |
| 82 | "unittests", |
| 83 | ], |
Ben Murdoch | 097c5b2 | 2016-05-18 11:27:45 +0100 | [diff] [blame] | 84 | # This needs to stay in sync with test/ignition.isolate. |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 85 | "ignition": [ |
| 86 | "mjsunit", |
| 87 | "cctest", |
Ben Murdoch | da12d29 | 2016-06-02 14:46:10 +0100 | [diff] [blame^] | 88 | "webkit", |
| 89 | "message", |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 90 | ], |
Ben Murdoch | 097c5b2 | 2016-05-18 11:27:45 +0100 | [diff] [blame] | 91 | # This needs to stay in sync with test/optimize_for_size.isolate. |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 92 | "optimize_for_size": [ |
| 93 | "mjsunit", |
| 94 | "cctest", |
| 95 | "webkit", |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 96 | "intl", |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 97 | ], |
| 98 | "unittests": [ |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 99 | "unittests", |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 100 | ], |
| 101 | } |
| 102 | |
| 103 | TIMEOUT_DEFAULT = 60 |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 104 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 105 | VARIANTS = ["default", "stress", "turbofan"] |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 106 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 107 | EXHAUSTIVE_VARIANTS = VARIANTS + [ |
| 108 | "nocrankshaft", |
| 109 | "turbofan_opt", |
| 110 | ] |
| 111 | |
| 112 | DEBUG_FLAGS = ["--nohard-abort", "--nodead-code-elimination", |
| 113 | "--nofold-constants", "--enable-slow-asserts", |
| 114 | "--debug-code", "--verify-heap"] |
| 115 | RELEASE_FLAGS = ["--nohard-abort", "--nodead-code-elimination", |
| 116 | "--nofold-constants"] |
| 117 | |
| 118 | MODES = { |
| 119 | "debug": { |
| 120 | "flags": DEBUG_FLAGS, |
| 121 | "timeout_scalefactor": 4, |
| 122 | "status_mode": "debug", |
| 123 | "execution_mode": "debug", |
| 124 | "output_folder": "debug", |
| 125 | }, |
| 126 | "optdebug": { |
| 127 | "flags": DEBUG_FLAGS, |
| 128 | "timeout_scalefactor": 4, |
| 129 | "status_mode": "debug", |
| 130 | "execution_mode": "debug", |
| 131 | "output_folder": "optdebug", |
| 132 | }, |
| 133 | "release": { |
| 134 | "flags": RELEASE_FLAGS, |
| 135 | "timeout_scalefactor": 1, |
| 136 | "status_mode": "release", |
| 137 | "execution_mode": "release", |
| 138 | "output_folder": "release", |
| 139 | }, |
| 140 | # Normal trybot release configuration. There, dchecks are always on which |
| 141 | # implies debug is set. Hence, the status file needs to assume debug-like |
| 142 | # behavior/timeouts. |
| 143 | "tryrelease": { |
| 144 | "flags": RELEASE_FLAGS, |
| 145 | "timeout_scalefactor": 1, |
| 146 | "status_mode": "debug", |
| 147 | "execution_mode": "release", |
| 148 | "output_folder": "release", |
| 149 | }, |
| 150 | # This mode requires v8 to be compiled with dchecks and slow dchecks. |
| 151 | "slowrelease": { |
| 152 | "flags": RELEASE_FLAGS + ["--enable-slow-asserts"], |
| 153 | "timeout_scalefactor": 2, |
| 154 | "status_mode": "debug", |
| 155 | "execution_mode": "release", |
| 156 | "output_folder": "release", |
| 157 | }, |
| 158 | } |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 159 | |
| 160 | GC_STRESS_FLAGS = ["--gc-interval=500", "--stress-compaction", |
| 161 | "--concurrent-recompilation-queue-length=64", |
| 162 | "--concurrent-recompilation-delay=500", |
| 163 | "--concurrent-recompilation"] |
| 164 | |
| 165 | SUPPORTED_ARCHS = ["android_arm", |
| 166 | "android_arm64", |
| 167 | "android_ia32", |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 168 | "android_x64", |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 169 | "arm", |
| 170 | "ia32", |
| 171 | "x87", |
| 172 | "mips", |
| 173 | "mipsel", |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 174 | "mips64", |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 175 | "mips64el", |
| 176 | "nacl_ia32", |
| 177 | "nacl_x64", |
Ben Murdoch | da12d29 | 2016-06-02 14:46:10 +0100 | [diff] [blame^] | 178 | "s390", |
| 179 | "s390x", |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 180 | "ppc", |
| 181 | "ppc64", |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 182 | "x64", |
| 183 | "x32", |
| 184 | "arm64"] |
| 185 | # Double the timeout for these: |
| 186 | SLOW_ARCHS = ["android_arm", |
| 187 | "android_arm64", |
| 188 | "android_ia32", |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 189 | "android_x64", |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 190 | "arm", |
| 191 | "mips", |
| 192 | "mipsel", |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 193 | "mips64", |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 194 | "mips64el", |
| 195 | "nacl_ia32", |
| 196 | "nacl_x64", |
| 197 | "x87", |
| 198 | "arm64"] |
| 199 | |
| 200 | |
| 201 | def BuildOptions(): |
| 202 | result = optparse.OptionParser() |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 203 | result.usage = '%prog [options] [tests]' |
| 204 | result.description = """TESTS: %s""" % (TEST_MAP["default"]) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 205 | result.add_option("--arch", |
| 206 | help=("The architecture to run tests for, " |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 207 | "'auto' or 'native' for auto-detect: %s" % SUPPORTED_ARCHS), |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 208 | default="ia32,x64,arm") |
| 209 | result.add_option("--arch-and-mode", |
| 210 | help="Architecture and mode in the format 'arch.mode'", |
| 211 | default=None) |
| 212 | result.add_option("--asan", |
| 213 | help="Regard test expectations for ASAN", |
| 214 | default=False, action="store_true") |
Ben Murdoch | da12d29 | 2016-06-02 14:46:10 +0100 | [diff] [blame^] | 215 | result.add_option("--sancov-dir", |
| 216 | help="Directory where to collect coverage data") |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 217 | result.add_option("--cfi-vptr", |
| 218 | help="Run tests with UBSAN cfi_vptr option.", |
| 219 | default=False, action="store_true") |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 220 | result.add_option("--buildbot", |
| 221 | help="Adapt to path structure used on buildbots", |
| 222 | default=False, action="store_true") |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 223 | result.add_option("--dcheck-always-on", |
| 224 | help="Indicates that V8 was compiled with DCHECKs enabled", |
| 225 | default=False, action="store_true") |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 226 | result.add_option("--novfp3", |
| 227 | help="Indicates that V8 was compiled without VFP3 support", |
| 228 | default=False, action="store_true") |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 229 | result.add_option("--cat", help="Print the source of the tests", |
| 230 | default=False, action="store_true") |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 231 | result.add_option("--slow-tests", |
| 232 | help="Regard slow tests (run|skip|dontcare)", |
| 233 | default="dontcare") |
| 234 | result.add_option("--pass-fail-tests", |
| 235 | help="Regard pass|fail tests (run|skip|dontcare)", |
| 236 | default="dontcare") |
| 237 | result.add_option("--gc-stress", |
| 238 | help="Switch on GC stress mode", |
| 239 | default=False, action="store_true") |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 240 | result.add_option("--gcov-coverage", |
| 241 | help="Uses executables instrumented for gcov coverage", |
| 242 | default=False, action="store_true") |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 243 | result.add_option("--command-prefix", |
| 244 | help="Prepended to each shell command used to run a test", |
| 245 | default="") |
| 246 | result.add_option("--download-data", help="Download missing test suite data", |
| 247 | default=False, action="store_true") |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 248 | result.add_option("--download-data-only", |
| 249 | help="Download missing test suite data and exit", |
| 250 | default=False, action="store_true") |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 251 | result.add_option("--extra-flags", |
| 252 | help="Additional flags to pass to each test command", |
| 253 | default="") |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 254 | result.add_option("--ignition", help="Skip tests which don't run in ignition", |
| 255 | default=False, action="store_true") |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 256 | result.add_option("--isolates", help="Whether to test isolates", |
| 257 | default=False, action="store_true") |
| 258 | result.add_option("-j", help="The number of parallel tasks to run", |
| 259 | default=0, type="int") |
| 260 | result.add_option("-m", "--mode", |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 261 | help="The test modes in which to run (comma-separated," |
| 262 | " uppercase for ninja and buildbot builds): %s" % MODES.keys(), |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 263 | default="release,debug") |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 264 | result.add_option("--no-harness", "--noharness", |
| 265 | help="Run without test harness of a given suite", |
| 266 | default=False, action="store_true") |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 267 | result.add_option("--no-i18n", "--noi18n", |
| 268 | help="Skip internationalization tests", |
| 269 | default=False, action="store_true") |
| 270 | result.add_option("--no-network", "--nonetwork", |
| 271 | help="Don't distribute tests on the network", |
| 272 | default=(utils.GuessOS() != "linux"), |
| 273 | dest="no_network", action="store_true") |
| 274 | result.add_option("--no-presubmit", "--nopresubmit", |
| 275 | help='Skip presubmit checks', |
| 276 | default=False, dest="no_presubmit", action="store_true") |
| 277 | result.add_option("--no-snap", "--nosnap", |
| 278 | help='Test a build compiled without snapshot.', |
| 279 | default=False, dest="no_snap", action="store_true") |
| 280 | result.add_option("--no-sorting", "--nosorting", |
| 281 | help="Don't sort tests according to duration of last run.", |
| 282 | default=False, dest="no_sorting", action="store_true") |
| 283 | result.add_option("--no-stress", "--nostress", |
| 284 | help="Don't run crankshaft --always-opt --stress-op test", |
| 285 | default=False, dest="no_stress", action="store_true") |
| 286 | result.add_option("--no-variants", "--novariants", |
| 287 | help="Don't run any testing variants", |
| 288 | default=False, dest="no_variants", action="store_true") |
| 289 | result.add_option("--variants", |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 290 | help="Comma-separated list of testing variants: %s" % VARIANTS) |
| 291 | result.add_option("--exhaustive-variants", |
| 292 | default=False, action="store_true", |
| 293 | help="Use exhaustive set of default variants.") |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 294 | result.add_option("--outdir", help="Base directory with compile output", |
| 295 | default="out") |
| 296 | result.add_option("--predictable", |
| 297 | help="Compare output of several reruns of each test", |
| 298 | default=False, action="store_true") |
| 299 | result.add_option("-p", "--progress", |
| 300 | help=("The style of progress indicator" |
| 301 | " (verbose, dots, color, mono)"), |
| 302 | choices=progress.PROGRESS_INDICATORS.keys(), default="mono") |
| 303 | result.add_option("--quickcheck", default=False, action="store_true", |
Ben Murdoch | da12d29 | 2016-06-02 14:46:10 +0100 | [diff] [blame^] | 304 | help=("Quick check mode (skip slow tests)")) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 305 | result.add_option("--report", help="Print a summary of the tests to be run", |
| 306 | default=False, action="store_true") |
| 307 | result.add_option("--json-test-results", |
| 308 | help="Path to a file for storing json results.") |
| 309 | result.add_option("--rerun-failures-count", |
| 310 | help=("Number of times to rerun each failing test case. " |
| 311 | "Very slow tests will be rerun only once."), |
| 312 | default=0, type="int") |
| 313 | result.add_option("--rerun-failures-max", |
| 314 | help="Maximum number of failing test cases to rerun.", |
| 315 | default=100, type="int") |
| 316 | result.add_option("--shard-count", |
| 317 | help="Split testsuites into this number of shards", |
| 318 | default=1, type="int") |
| 319 | result.add_option("--shard-run", |
| 320 | help="Run this shard from the split up tests.", |
| 321 | default=1, type="int") |
| 322 | result.add_option("--shell", help="DEPRECATED! use --shell-dir", default="") |
| 323 | result.add_option("--shell-dir", help="Directory containing executables", |
| 324 | default="") |
| 325 | result.add_option("--dont-skip-slow-simulator-tests", |
| 326 | help="Don't skip more slow tests when using a simulator.", |
| 327 | default=False, action="store_true", |
| 328 | dest="dont_skip_simulator_slow_tests") |
| 329 | result.add_option("--stress-only", |
| 330 | help="Only run tests with --always-opt --stress-opt", |
| 331 | default=False, action="store_true") |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 332 | result.add_option("--swarming", |
| 333 | help="Indicates running test driver on swarming.", |
| 334 | default=False, action="store_true") |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 335 | result.add_option("--time", help="Print timing information after running", |
| 336 | default=False, action="store_true") |
| 337 | result.add_option("-t", "--timeout", help="Timeout in seconds", |
| 338 | default= -1, type="int") |
| 339 | result.add_option("--tsan", |
| 340 | help="Regard test expectations for TSAN", |
| 341 | default=False, action="store_true") |
| 342 | result.add_option("-v", "--verbose", help="Verbose output", |
| 343 | default=False, action="store_true") |
| 344 | result.add_option("--valgrind", help="Run tests through valgrind", |
| 345 | default=False, action="store_true") |
| 346 | result.add_option("--warn-unused", help="Report unused rules", |
| 347 | default=False, action="store_true") |
| 348 | result.add_option("--junitout", help="File name of the JUnit output") |
| 349 | result.add_option("--junittestsuite", |
| 350 | help="The testsuite name in the JUnit output file", |
| 351 | default="v8tests") |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 352 | result.add_option("--random-seed", default=0, dest="random_seed", type="int", |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 353 | help="Default seed for initializing random generator") |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 354 | result.add_option("--random-seed-stress-count", default=1, type="int", |
| 355 | dest="random_seed_stress_count", |
| 356 | help="Number of runs with different random seeds") |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 357 | result.add_option("--msan", |
| 358 | help="Regard test expectations for MSAN", |
| 359 | default=False, action="store_true") |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 360 | return result |
| 361 | |
| 362 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 363 | def RandomSeed(): |
| 364 | seed = 0 |
| 365 | while not seed: |
| 366 | seed = random.SystemRandom().randint(-2147483648, 2147483647) |
| 367 | return seed |
| 368 | |
| 369 | |
| 370 | def BuildbotToV8Mode(config): |
| 371 | """Convert buildbot build configs to configs understood by the v8 runner. |
| 372 | |
| 373 | V8 configs are always lower case and without the additional _x64 suffix for |
| 374 | 64 bit builds on windows with ninja. |
| 375 | """ |
| 376 | mode = config[:-4] if config.endswith('_x64') else config |
| 377 | return mode.lower() |
| 378 | |
| 379 | def SetupEnvironment(options): |
| 380 | """Setup additional environment variables.""" |
| 381 | symbolizer = 'external_symbolizer_path=%s' % ( |
| 382 | os.path.join( |
| 383 | BASE_DIR, 'third_party', 'llvm-build', 'Release+Asserts', 'bin', |
| 384 | 'llvm-symbolizer', |
| 385 | ) |
| 386 | ) |
| 387 | |
| 388 | if options.asan: |
| 389 | os.environ['ASAN_OPTIONS'] = symbolizer |
| 390 | |
Ben Murdoch | da12d29 | 2016-06-02 14:46:10 +0100 | [diff] [blame^] | 391 | if options.sancov_dir: |
| 392 | assert os.path.exists(options.sancov_dir) |
| 393 | os.environ['ASAN_OPTIONS'] = ":".join([ |
| 394 | 'coverage=1', |
| 395 | 'coverage_dir=%s' % options.sancov_dir, |
| 396 | symbolizer, |
| 397 | ]) |
| 398 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 399 | if options.cfi_vptr: |
| 400 | os.environ['UBSAN_OPTIONS'] = ":".join([ |
| 401 | 'print_stacktrace=1', |
| 402 | 'print_summary=1', |
| 403 | 'symbolize=1', |
| 404 | symbolizer, |
| 405 | ]) |
| 406 | |
| 407 | if options.msan: |
| 408 | os.environ['MSAN_OPTIONS'] = symbolizer |
| 409 | |
| 410 | if options.tsan: |
| 411 | suppressions_file = os.path.join( |
| 412 | BASE_DIR, 'tools', 'sanitizers', 'tsan_suppressions.txt') |
| 413 | os.environ['TSAN_OPTIONS'] = " ".join([ |
| 414 | symbolizer, |
| 415 | 'suppressions=%s' % suppressions_file, |
| 416 | 'exit_code=0', |
| 417 | 'report_thread_leaks=0', |
| 418 | 'history_size=7', |
| 419 | 'report_destroy_locked=0', |
| 420 | ]) |
| 421 | |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 422 | def ProcessOptions(options): |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 423 | global ALL_VARIANTS |
| 424 | global EXHAUSTIVE_VARIANTS |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 425 | global VARIANTS |
| 426 | |
| 427 | # Architecture and mode related stuff. |
| 428 | if options.arch_and_mode: |
| 429 | options.arch_and_mode = [arch_and_mode.split(".") |
| 430 | for arch_and_mode in options.arch_and_mode.split(",")] |
| 431 | options.arch = ",".join([tokens[0] for tokens in options.arch_and_mode]) |
| 432 | options.mode = ",".join([tokens[1] for tokens in options.arch_and_mode]) |
| 433 | options.mode = options.mode.split(",") |
| 434 | for mode in options.mode: |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 435 | if not BuildbotToV8Mode(mode) in MODES: |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 436 | print "Unknown mode %s" % mode |
| 437 | return False |
| 438 | if options.arch in ["auto", "native"]: |
| 439 | options.arch = ARCH_GUESS |
| 440 | options.arch = options.arch.split(",") |
| 441 | for arch in options.arch: |
| 442 | if not arch in SUPPORTED_ARCHS: |
| 443 | print "Unknown architecture %s" % arch |
| 444 | return False |
| 445 | |
| 446 | # Store the final configuration in arch_and_mode list. Don't overwrite |
| 447 | # predefined arch_and_mode since it is more expressive than arch and mode. |
| 448 | if not options.arch_and_mode: |
| 449 | options.arch_and_mode = itertools.product(options.arch, options.mode) |
| 450 | |
| 451 | # Special processing of other options, sorted alphabetically. |
| 452 | |
| 453 | if options.buildbot: |
| 454 | # Buildbots run presubmit tests as a separate step. |
| 455 | options.no_presubmit = True |
| 456 | options.no_network = True |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 457 | if options.download_data_only: |
| 458 | options.no_presubmit = True |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 459 | if options.command_prefix: |
| 460 | print("Specifying --command-prefix disables network distribution, " |
| 461 | "running tests locally.") |
| 462 | options.no_network = True |
| 463 | options.command_prefix = shlex.split(options.command_prefix) |
| 464 | options.extra_flags = shlex.split(options.extra_flags) |
| 465 | |
| 466 | if options.gc_stress: |
| 467 | options.extra_flags += GC_STRESS_FLAGS |
| 468 | |
| 469 | if options.asan: |
| 470 | options.extra_flags.append("--invoke-weak-callbacks") |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 471 | options.extra_flags.append("--omit-quit") |
| 472 | |
| 473 | if options.novfp3: |
| 474 | options.extra_flags.append("--noenable-vfp3") |
| 475 | |
| 476 | if options.exhaustive_variants: |
| 477 | # This is used on many bots. It includes a larger set of default variants. |
| 478 | # Other options for manipulating variants still apply afterwards. |
| 479 | VARIANTS = EXHAUSTIVE_VARIANTS |
| 480 | |
| 481 | if options.msan: |
| 482 | VARIANTS = ["default"] |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 483 | |
| 484 | if options.tsan: |
| 485 | VARIANTS = ["default"] |
| 486 | |
| 487 | if options.j == 0: |
| 488 | options.j = multiprocessing.cpu_count() |
| 489 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 490 | if options.random_seed_stress_count <= 1 and options.random_seed == 0: |
| 491 | options.random_seed = RandomSeed() |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 492 | |
| 493 | def excl(*args): |
| 494 | """Returns true if zero or one of multiple arguments are true.""" |
| 495 | return reduce(lambda x, y: x + y, args) <= 1 |
| 496 | |
| 497 | if not excl(options.no_stress, options.stress_only, options.no_variants, |
| 498 | bool(options.variants)): |
| 499 | print("Use only one of --no-stress, --stress-only, --no-variants, " |
| 500 | "or --variants.") |
| 501 | return False |
| 502 | if options.quickcheck: |
| 503 | VARIANTS = ["default", "stress"] |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 504 | options.slow_tests = "skip" |
| 505 | options.pass_fail_tests = "skip" |
| 506 | if options.no_stress: |
| 507 | VARIANTS = ["default", "nocrankshaft"] |
| 508 | if options.no_variants: |
| 509 | VARIANTS = ["default"] |
| 510 | if options.stress_only: |
| 511 | VARIANTS = ["stress"] |
| 512 | if options.variants: |
| 513 | VARIANTS = options.variants.split(",") |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 514 | if not set(VARIANTS).issubset(ALL_VARIANTS): |
| 515 | print "All variants must be in %s" % str(ALL_VARIANTS) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 516 | return False |
| 517 | if options.predictable: |
| 518 | VARIANTS = ["default"] |
| 519 | options.extra_flags.append("--predictable") |
| 520 | options.extra_flags.append("--verify_predictable") |
| 521 | options.extra_flags.append("--no-inline-new") |
| 522 | |
| 523 | if not options.shell_dir: |
| 524 | if options.shell: |
| 525 | print "Warning: --shell is deprecated, use --shell-dir instead." |
| 526 | options.shell_dir = os.path.dirname(options.shell) |
| 527 | if options.valgrind: |
| 528 | run_valgrind = os.path.join("tools", "run-valgrind.py") |
| 529 | # This is OK for distributed running, so we don't need to set no_network. |
| 530 | options.command_prefix = (["python", "-u", run_valgrind] + |
| 531 | options.command_prefix) |
| 532 | def CheckTestMode(name, option): |
| 533 | if not option in ["run", "skip", "dontcare"]: |
| 534 | print "Unknown %s mode %s" % (name, option) |
| 535 | return False |
| 536 | return True |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 537 | if not CheckTestMode("slow test", options.slow_tests): |
| 538 | return False |
| 539 | if not CheckTestMode("pass|fail test", options.pass_fail_tests): |
| 540 | return False |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 541 | if options.no_i18n: |
| 542 | TEST_MAP["bot_default"].remove("intl") |
| 543 | TEST_MAP["default"].remove("intl") |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 544 | return True |
| 545 | |
| 546 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 547 | def ShardTests(tests, options): |
| 548 | # Read gtest shard configuration from environment (e.g. set by swarming). |
| 549 | # If none is present, use values passed on the command line. |
| 550 | shard_count = int(os.environ.get('GTEST_TOTAL_SHARDS', options.shard_count)) |
| 551 | shard_run = os.environ.get('GTEST_SHARD_INDEX') |
| 552 | if shard_run is not None: |
| 553 | # The v8 shard_run starts at 1, while GTEST_SHARD_INDEX starts at 0. |
| 554 | shard_run = int(shard_run) + 1 |
| 555 | else: |
| 556 | shard_run = options.shard_run |
| 557 | |
| 558 | if options.shard_count > 1: |
| 559 | # Log if a value was passed on the cmd line and it differs from the |
| 560 | # environment variables. |
| 561 | if options.shard_count != shard_count: |
| 562 | print("shard_count from cmd line differs from environment variable " |
| 563 | "GTEST_TOTAL_SHARDS") |
| 564 | if options.shard_run > 1 and options.shard_run != shard_run: |
| 565 | print("shard_run from cmd line differs from environment variable " |
| 566 | "GTEST_SHARD_INDEX") |
| 567 | |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 568 | if shard_count < 2: |
| 569 | return tests |
| 570 | if shard_run < 1 or shard_run > shard_count: |
| 571 | print "shard-run not a valid number, should be in [1:shard-count]" |
| 572 | print "defaulting back to running all tests" |
| 573 | return tests |
| 574 | count = 0 |
| 575 | shard = [] |
| 576 | for test in tests: |
| 577 | if count % shard_count == shard_run - 1: |
| 578 | shard.append(test) |
| 579 | count += 1 |
| 580 | return shard |
| 581 | |
| 582 | |
| 583 | def Main(): |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 584 | # Use the v8 root as cwd as some test cases use "load" with relative paths. |
| 585 | os.chdir(BASE_DIR) |
| 586 | |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 587 | parser = BuildOptions() |
| 588 | (options, args) = parser.parse_args() |
| 589 | if not ProcessOptions(options): |
| 590 | parser.print_help() |
| 591 | return 1 |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 592 | SetupEnvironment(options) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 593 | |
| 594 | exit_code = 0 |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 595 | if not options.no_presubmit: |
| 596 | print ">>> running presubmit tests" |
| 597 | exit_code = subprocess.call( |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 598 | [sys.executable, join(BASE_DIR, "tools", "presubmit.py")]) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 599 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 600 | suite_paths = utils.GetSuitePaths(join(BASE_DIR, "test")) |
| 601 | |
| 602 | # Use default tests if no test configuration was provided at the cmd line. |
| 603 | if len(args) == 0: |
| 604 | args = ["default"] |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 605 | |
| 606 | # Expand arguments with grouped tests. The args should reflect the list of |
| 607 | # suites as otherwise filters would break. |
| 608 | def ExpandTestGroups(name): |
| 609 | if name in TEST_MAP: |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 610 | return [suite for suite in TEST_MAP[name]] |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 611 | else: |
| 612 | return [name] |
| 613 | args = reduce(lambda x, y: x + y, |
| 614 | [ExpandTestGroups(arg) for arg in args], |
| 615 | []) |
| 616 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 617 | args_suites = OrderedDict() # Used as set |
| 618 | for arg in args: |
| 619 | args_suites[arg.split('/')[0]] = True |
| 620 | suite_paths = [ s for s in args_suites if s in suite_paths ] |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 621 | |
| 622 | suites = [] |
| 623 | for root in suite_paths: |
| 624 | suite = testsuite.TestSuite.LoadTestSuite( |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 625 | os.path.join(BASE_DIR, "test", root)) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 626 | if suite: |
| 627 | suites.append(suite) |
| 628 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 629 | if options.download_data or options.download_data_only: |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 630 | for s in suites: |
| 631 | s.DownloadData() |
| 632 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 633 | if options.download_data_only: |
| 634 | return exit_code |
| 635 | |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 636 | for (arch, mode) in options.arch_and_mode: |
| 637 | try: |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 638 | code = Execute(arch, mode, args, options, suites) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 639 | except KeyboardInterrupt: |
| 640 | return 2 |
| 641 | exit_code = exit_code or code |
| 642 | return exit_code |
| 643 | |
| 644 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 645 | def Execute(arch, mode, args, options, suites): |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 646 | print(">>> Running tests for %s.%s" % (arch, mode)) |
| 647 | |
| 648 | shell_dir = options.shell_dir |
| 649 | if not shell_dir: |
| 650 | if options.buildbot: |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 651 | # TODO(machenbach): Get rid of different output folder location on |
| 652 | # buildbot. Currently this is capitalized Release and Debug. |
| 653 | shell_dir = os.path.join(BASE_DIR, options.outdir, mode) |
| 654 | mode = BuildbotToV8Mode(mode) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 655 | else: |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 656 | shell_dir = os.path.join( |
| 657 | BASE_DIR, |
| 658 | options.outdir, |
| 659 | "%s.%s" % (arch, MODES[mode]["output_folder"]), |
| 660 | ) |
| 661 | if not os.path.exists(shell_dir): |
| 662 | raise Exception('Could not find shell_dir: "%s"' % shell_dir) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 663 | |
| 664 | # Populate context object. |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 665 | mode_flags = MODES[mode]["flags"] |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 666 | timeout = options.timeout |
| 667 | if timeout == -1: |
| 668 | # Simulators are slow, therefore allow a longer default timeout. |
| 669 | if arch in SLOW_ARCHS: |
| 670 | timeout = 2 * TIMEOUT_DEFAULT; |
| 671 | else: |
| 672 | timeout = TIMEOUT_DEFAULT; |
| 673 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 674 | timeout *= MODES[mode]["timeout_scalefactor"] |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 675 | |
| 676 | if options.predictable: |
| 677 | # Predictable mode is slower. |
| 678 | timeout *= 2 |
| 679 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 680 | # TODO(machenbach): Remove temporary verbose output on windows after |
| 681 | # debugging driver-hung-up on XP. |
| 682 | verbose_output = ( |
| 683 | options.verbose or |
| 684 | utils.IsWindows() and options.progress == "verbose" |
| 685 | ) |
| 686 | ctx = context.Context(arch, MODES[mode]["execution_mode"], shell_dir, |
| 687 | mode_flags, verbose_output, |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 688 | timeout, options.isolates, |
| 689 | options.command_prefix, |
| 690 | options.extra_flags, |
| 691 | options.no_i18n, |
| 692 | options.random_seed, |
| 693 | options.no_sorting, |
| 694 | options.rerun_failures_count, |
| 695 | options.rerun_failures_max, |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 696 | options.predictable, |
| 697 | options.no_harness, |
Ben Murdoch | da12d29 | 2016-06-02 14:46:10 +0100 | [diff] [blame^] | 698 | use_perf_data=not options.swarming, |
| 699 | sancov_dir=options.sancov_dir) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 700 | |
| 701 | # TODO(all): Combine "simulator" and "simulator_run". |
| 702 | simulator_run = not options.dont_skip_simulator_slow_tests and \ |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 703 | arch in ['arm64', 'arm', 'mipsel', 'mips', 'mips64', 'mips64el', \ |
| 704 | 'ppc', 'ppc64'] and \ |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 705 | ARCH_GUESS and arch != ARCH_GUESS |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 706 | # Find available test suites and read test cases from them. |
| 707 | variables = { |
| 708 | "arch": arch, |
| 709 | "asan": options.asan, |
| 710 | "deopt_fuzzer": False, |
| 711 | "gc_stress": options.gc_stress, |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 712 | "gcov_coverage": options.gcov_coverage, |
| 713 | "ignition": options.ignition, |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 714 | "isolates": options.isolates, |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 715 | "mode": MODES[mode]["status_mode"], |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 716 | "no_i18n": options.no_i18n, |
| 717 | "no_snap": options.no_snap, |
| 718 | "simulator_run": simulator_run, |
| 719 | "simulator": utils.UseSimulator(arch), |
| 720 | "system": utils.GuessOS(), |
| 721 | "tsan": options.tsan, |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 722 | "msan": options.msan, |
| 723 | "dcheck_always_on": options.dcheck_always_on, |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 724 | "novfp3": options.novfp3, |
| 725 | "predictable": options.predictable, |
| 726 | "byteorder": sys.byteorder, |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 727 | } |
| 728 | all_tests = [] |
| 729 | num_tests = 0 |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 730 | for s in suites: |
| 731 | s.ReadStatusFile(variables) |
| 732 | s.ReadTestCases(ctx) |
| 733 | if len(args) > 0: |
| 734 | s.FilterTestCasesByArgs(args) |
| 735 | all_tests += s.tests |
Ben Murdoch | da12d29 | 2016-06-02 14:46:10 +0100 | [diff] [blame^] | 736 | s.FilterTestCasesByStatus(options.warn_unused, options.slow_tests, |
| 737 | options.pass_fail_tests) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 738 | if options.cat: |
| 739 | verbose.PrintTestSource(s.tests) |
| 740 | continue |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 741 | variant_gen = s.CreateVariantGenerator(VARIANTS) |
| 742 | variant_tests = [ t.CopyAddingFlags(v, flags) |
| 743 | for t in s.tests |
| 744 | for v in variant_gen.FilterVariantsByTest(t) |
| 745 | for flags in variant_gen.GetFlagSets(t, v) ] |
| 746 | |
| 747 | if options.random_seed_stress_count > 1: |
| 748 | # Duplicate test for random seed stress mode. |
| 749 | def iter_seed_flags(): |
| 750 | for i in range(0, options.random_seed_stress_count): |
| 751 | # Use given random seed for all runs (set by default in execution.py) |
| 752 | # or a new random seed if none is specified. |
| 753 | if options.random_seed: |
| 754 | yield [] |
| 755 | else: |
| 756 | yield ["--random-seed=%d" % RandomSeed()] |
| 757 | s.tests = [ |
| 758 | t.CopyAddingFlags(t.variant, flags) |
| 759 | for t in variant_tests |
| 760 | for flags in iter_seed_flags() |
| 761 | ] |
| 762 | else: |
| 763 | s.tests = variant_tests |
| 764 | |
| 765 | s.tests = ShardTests(s.tests, options) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 766 | num_tests += len(s.tests) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 767 | |
| 768 | if options.cat: |
| 769 | return 0 # We're done here. |
| 770 | |
| 771 | if options.report: |
| 772 | verbose.PrintReport(all_tests) |
| 773 | |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 774 | # Run the tests, either locally or distributed on the network. |
| 775 | start_time = time.time() |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 776 | progress_indicator = progress.IndicatorNotifier() |
| 777 | progress_indicator.Register(progress.PROGRESS_INDICATORS[options.progress]()) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 778 | if options.junitout: |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 779 | progress_indicator.Register(progress.JUnitTestProgressIndicator( |
| 780 | options.junitout, options.junittestsuite)) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 781 | if options.json_test_results: |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 782 | progress_indicator.Register(progress.JsonTestProgressIndicator( |
| 783 | options.json_test_results, arch, MODES[mode]["execution_mode"], |
| 784 | ctx.random_seed)) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 785 | |
| 786 | run_networked = not options.no_network |
| 787 | if not run_networked: |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 788 | if verbose_output: |
| 789 | print("Network distribution disabled, running tests locally.") |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 790 | elif utils.GuessOS() != "linux": |
| 791 | print("Network distribution is only supported on Linux, sorry!") |
| 792 | run_networked = False |
| 793 | peers = [] |
| 794 | if run_networked: |
| 795 | peers = network_execution.GetPeers() |
| 796 | if not peers: |
| 797 | print("No connection to distribution server; running tests locally.") |
| 798 | run_networked = False |
| 799 | elif len(peers) == 1: |
| 800 | print("No other peers on the network; running tests locally.") |
| 801 | run_networked = False |
| 802 | elif num_tests <= 100: |
| 803 | print("Less than 100 tests, running them locally.") |
| 804 | run_networked = False |
| 805 | |
| 806 | if run_networked: |
| 807 | runner = network_execution.NetworkedRunner(suites, progress_indicator, |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 808 | ctx, peers, BASE_DIR) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 809 | else: |
| 810 | runner = execution.Runner(suites, progress_indicator, ctx) |
| 811 | |
| 812 | exit_code = runner.Run(options.j) |
| 813 | overall_duration = time.time() - start_time |
| 814 | |
| 815 | if options.time: |
| 816 | verbose.PrintTestDurations(suites, overall_duration) |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 817 | |
| 818 | if num_tests == 0: |
| 819 | print("Warning: no tests were run!") |
| 820 | |
| 821 | if exit_code == 1 and options.json_test_results: |
| 822 | print("Force exit code 0 after failures. Json test results file generated " |
| 823 | "with failure information.") |
| 824 | exit_code = 0 |
| 825 | |
Ben Murdoch | da12d29 | 2016-06-02 14:46:10 +0100 | [diff] [blame^] | 826 | if options.sancov_dir: |
| 827 | # If tests ran with sanitizer coverage, merge coverage files in the end. |
| 828 | try: |
| 829 | print "Merging sancov files." |
| 830 | subprocess.check_call([ |
| 831 | sys.executable, |
| 832 | join(BASE_DIR, "tools", "sanitizers", "sancov_merger.py"), |
| 833 | "--coverage-dir=%s" % options.sancov_dir]) |
| 834 | except: |
| 835 | print >> sys.stderr, "Error: Merging sancov files failed." |
| 836 | exit_code = 1 |
| 837 | |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 838 | return exit_code |
| 839 | |
| 840 | |
| 841 | if __name__ == "__main__": |
| 842 | sys.exit(Main()) |