Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 1 | /* Copyright (c) 2015, Google Inc. |
| 2 | * |
| 3 | * Permission to use, copy, modify, and/or distribute this software for any |
| 4 | * purpose with or without fee is hereby granted, provided that the above |
| 5 | * copyright notice and this permission notice appear in all copies. |
| 6 | * |
| 7 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES |
| 8 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF |
| 9 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY |
| 10 | * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES |
| 11 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION |
| 12 | * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN |
| 13 | * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ |
| 14 | |
| 15 | package main |
| 16 | |
| 17 | import ( |
Robert Sloan | 8ff0355 | 2017-06-14 12:40:58 -0700 | [diff] [blame] | 18 | "bufio" |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 19 | "bytes" |
| 20 | "encoding/json" |
Robert Sloan | c9abfe4 | 2018-11-26 12:19:07 -0800 | [diff] [blame] | 21 | "errors" |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 22 | "flag" |
| 23 | "fmt" |
Robert Sloan | 8ff0355 | 2017-06-14 12:40:58 -0700 | [diff] [blame] | 24 | "math/rand" |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 25 | "os" |
| 26 | "os/exec" |
| 27 | "path" |
Robert Sloan | 572a4e2 | 2017-04-17 10:52:19 -0700 | [diff] [blame] | 28 | "runtime" |
Adam Langley | f4e4272 | 2015-06-04 17:45:09 -0700 | [diff] [blame] | 29 | "strconv" |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 30 | "strings" |
David Benjamin | 4969cc9 | 2016-04-22 15:02:23 -0400 | [diff] [blame] | 31 | "sync" |
Adam Langley | f4e4272 | 2015-06-04 17:45:09 -0700 | [diff] [blame] | 32 | "syscall" |
Robert Sloan | c9abfe4 | 2018-11-26 12:19:07 -0800 | [diff] [blame] | 33 | |
| 34 | "boringssl.googlesource.com/boringssl/util/testresult" |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 35 | ) |
| 36 | |
| 37 | // TODO(davidben): Link tests with the malloc shim and port -malloc-test to this runner. |
| 38 | |
| 39 | var ( |
Adam Langley | f4e4272 | 2015-06-04 17:45:09 -0700 | [diff] [blame] | 40 | useValgrind = flag.Bool("valgrind", false, "If true, run code under valgrind") |
David Benjamin | 4969cc9 | 2016-04-22 15:02:23 -0400 | [diff] [blame] | 41 | useCallgrind = flag.Bool("callgrind", false, "If true, run code under valgrind to generate callgrind traces.") |
Adam Langley | f4e4272 | 2015-06-04 17:45:09 -0700 | [diff] [blame] | 42 | useGDB = flag.Bool("gdb", false, "If true, run BoringSSL code under gdb") |
Robert Sloan | 4d1ac50 | 2017-02-06 08:36:14 -0800 | [diff] [blame] | 43 | useSDE = flag.Bool("sde", false, "If true, run BoringSSL code under Intel's SDE for each supported chip") |
Robert Sloan | 8ff0355 | 2017-06-14 12:40:58 -0700 | [diff] [blame] | 44 | sdePath = flag.String("sde-path", "sde", "The path to find the sde binary.") |
Adam Langley | f4e4272 | 2015-06-04 17:45:09 -0700 | [diff] [blame] | 45 | buildDir = flag.String("build-dir", "build", "The build directory to run the tests from.") |
Robert Sloan | 572a4e2 | 2017-04-17 10:52:19 -0700 | [diff] [blame] | 46 | numWorkers = flag.Int("num-workers", runtime.NumCPU(), "Runs the given number of workers when testing.") |
Adam Langley | f4e4272 | 2015-06-04 17:45:09 -0700 | [diff] [blame] | 47 | jsonOutput = flag.String("json-output", "", "The file to output JSON results to.") |
| 48 | mallocTest = flag.Int64("malloc-test", -1, "If non-negative, run each test with each malloc in turn failing from the given number onwards.") |
| 49 | mallocTestDebug = flag.Bool("malloc-test-debug", false, "If true, ask each test to abort rather than fail a malloc. This can be used with a specific value for --malloc-test to identity the malloc failing that is causing problems.") |
Robert Sloan | c9abfe4 | 2018-11-26 12:19:07 -0800 | [diff] [blame] | 50 | simulateARMCPUs = flag.Bool("simulate-arm-cpus", simulateARMCPUsDefault(), "If true, runs tests simulating different ARM CPUs.") |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 51 | ) |
| 52 | |
Robert Sloan | c9abfe4 | 2018-11-26 12:19:07 -0800 | [diff] [blame] | 53 | func simulateARMCPUsDefault() bool { |
| 54 | return runtime.GOOS == "linux" && (runtime.GOARCH == "arm" || runtime.GOARCH == "arm64") |
| 55 | } |
| 56 | |
Robert Sloan | 4d1ac50 | 2017-02-06 08:36:14 -0800 | [diff] [blame] | 57 | type test struct { |
Robert Sloan | 4562e9d | 2017-10-02 10:26:51 -0700 | [diff] [blame] | 58 | args []string |
| 59 | shard, numShards int |
Robert Sloan | c9abfe4 | 2018-11-26 12:19:07 -0800 | [diff] [blame] | 60 | // cpu, if not empty, contains a code to simulate. For SDE, run `sde64 |
| 61 | // -help` to get a list of these codes. For ARM, see gtest_main.cc for |
| 62 | // the supported values. |
Robert Sloan | 4d1ac50 | 2017-02-06 08:36:14 -0800 | [diff] [blame] | 63 | cpu string |
| 64 | } |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 65 | |
David Benjamin | 4969cc9 | 2016-04-22 15:02:23 -0400 | [diff] [blame] | 66 | type result struct { |
| 67 | Test test |
| 68 | Passed bool |
| 69 | Error error |
| 70 | } |
| 71 | |
Robert Sloan | 4d1ac50 | 2017-02-06 08:36:14 -0800 | [diff] [blame] | 72 | // sdeCPUs contains a list of CPU code that we run all tests under when *useSDE |
| 73 | // is true. |
| 74 | var sdeCPUs = []string{ |
| 75 | "p4p", // Pentium4 Prescott |
| 76 | "mrm", // Merom |
| 77 | "pnr", // Penryn |
| 78 | "nhm", // Nehalem |
| 79 | "wsm", // Westmere |
| 80 | "snb", // Sandy Bridge |
| 81 | "ivb", // Ivy Bridge |
| 82 | "hsw", // Haswell |
| 83 | "bdw", // Broadwell |
| 84 | "skx", // Skylake Server |
| 85 | "skl", // Skylake Client |
| 86 | "cnl", // Cannonlake |
| 87 | "knl", // Knights Landing |
| 88 | "slt", // Saltwell |
| 89 | "slm", // Silvermont |
| 90 | "glm", // Goldmont |
Robert Sloan | fe7cd21 | 2017-08-07 09:03:39 -0700 | [diff] [blame] | 91 | "knm", // Knights Mill |
Robert Sloan | 4d1ac50 | 2017-02-06 08:36:14 -0800 | [diff] [blame] | 92 | } |
| 93 | |
Robert Sloan | c9abfe4 | 2018-11-26 12:19:07 -0800 | [diff] [blame] | 94 | var armCPUs = []string{ |
| 95 | "none", // No support for any ARM extensions. |
| 96 | "neon", // Support for NEON. |
| 97 | "crypto", // Support for NEON and crypto extensions. |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 98 | } |
| 99 | |
| 100 | func valgrindOf(dbAttach bool, path string, args ...string) *exec.Cmd { |
David Benjamin | 7c0d06c | 2016-08-11 13:26:41 -0400 | [diff] [blame] | 101 | valgrindArgs := []string{"--error-exitcode=99", "--track-origins=yes", "--leak-check=full", "--quiet"} |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 102 | if dbAttach { |
| 103 | valgrindArgs = append(valgrindArgs, "--db-attach=yes", "--db-command=xterm -e gdb -nw %f %p") |
| 104 | } |
| 105 | valgrindArgs = append(valgrindArgs, path) |
| 106 | valgrindArgs = append(valgrindArgs, args...) |
| 107 | |
| 108 | return exec.Command("valgrind", valgrindArgs...) |
| 109 | } |
| 110 | |
David Benjamin | 4969cc9 | 2016-04-22 15:02:23 -0400 | [diff] [blame] | 111 | func callgrindOf(path string, args ...string) *exec.Cmd { |
| 112 | valgrindArgs := []string{"-q", "--tool=callgrind", "--dump-instr=yes", "--collect-jumps=yes", "--callgrind-out-file=" + *buildDir + "/callgrind/callgrind.out.%p"} |
| 113 | valgrindArgs = append(valgrindArgs, path) |
| 114 | valgrindArgs = append(valgrindArgs, args...) |
| 115 | |
| 116 | return exec.Command("valgrind", valgrindArgs...) |
| 117 | } |
| 118 | |
Adam Langley | f4e4272 | 2015-06-04 17:45:09 -0700 | [diff] [blame] | 119 | func gdbOf(path string, args ...string) *exec.Cmd { |
| 120 | xtermArgs := []string{"-e", "gdb", "--args"} |
| 121 | xtermArgs = append(xtermArgs, path) |
| 122 | xtermArgs = append(xtermArgs, args...) |
| 123 | |
| 124 | return exec.Command("xterm", xtermArgs...) |
| 125 | } |
| 126 | |
Robert Sloan | 4d1ac50 | 2017-02-06 08:36:14 -0800 | [diff] [blame] | 127 | func sdeOf(cpu, path string, args ...string) *exec.Cmd { |
Robert Sloan | b6d070c | 2017-07-24 08:40:01 -0700 | [diff] [blame] | 128 | sdeArgs := []string{"-" + cpu} |
| 129 | // The kernel's vdso code for gettimeofday sometimes uses the RDTSCP |
| 130 | // instruction. Although SDE has a -chip_check_vsyscall flag that |
| 131 | // excludes such code by default, it does not seem to work. Instead, |
| 132 | // pass the -chip_check_exe_only flag which retains test coverage when |
| 133 | // statically linked and excludes the vdso. |
| 134 | if cpu == "p4p" || cpu == "pnr" || cpu == "mrm" || cpu == "slt" { |
| 135 | sdeArgs = append(sdeArgs, "-chip_check_exe_only") |
| 136 | } |
| 137 | sdeArgs = append(sdeArgs, "--", path) |
Robert Sloan | 4d1ac50 | 2017-02-06 08:36:14 -0800 | [diff] [blame] | 138 | sdeArgs = append(sdeArgs, args...) |
Robert Sloan | 8ff0355 | 2017-06-14 12:40:58 -0700 | [diff] [blame] | 139 | return exec.Command(*sdePath, sdeArgs...) |
Robert Sloan | 4d1ac50 | 2017-02-06 08:36:14 -0800 | [diff] [blame] | 140 | } |
| 141 | |
Robert Sloan | c9abfe4 | 2018-11-26 12:19:07 -0800 | [diff] [blame] | 142 | var ( |
| 143 | errMoreMallocs = errors.New("child process did not exhaust all allocation calls") |
| 144 | errTestSkipped = errors.New("test was skipped") |
| 145 | ) |
Adam Langley | f4e4272 | 2015-06-04 17:45:09 -0700 | [diff] [blame] | 146 | |
| 147 | func runTestOnce(test test, mallocNumToFail int64) (passed bool, err error) { |
Robert Sloan | 4d1ac50 | 2017-02-06 08:36:14 -0800 | [diff] [blame] | 148 | prog := path.Join(*buildDir, test.args[0]) |
Robert Sloan | 4c22c5f | 2019-03-01 15:53:37 -0800 | [diff] [blame] | 149 | args := append([]string{}, test.args[1:]...) |
Robert Sloan | c9abfe4 | 2018-11-26 12:19:07 -0800 | [diff] [blame] | 150 | if *simulateARMCPUs && test.cpu != "" { |
Robert Sloan | 4c22c5f | 2019-03-01 15:53:37 -0800 | [diff] [blame] | 151 | args = append(args, "--cpu=" + test.cpu) |
| 152 | } |
| 153 | if *useSDE { |
| 154 | // SDE is neither compatible with the unwind tester nor automatically |
| 155 | // detected. |
| 156 | args = append(args, "--no_unwind_tests") |
Robert Sloan | c9abfe4 | 2018-11-26 12:19:07 -0800 | [diff] [blame] | 157 | } |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 158 | var cmd *exec.Cmd |
| 159 | if *useValgrind { |
| 160 | cmd = valgrindOf(false, prog, args...) |
David Benjamin | 4969cc9 | 2016-04-22 15:02:23 -0400 | [diff] [blame] | 161 | } else if *useCallgrind { |
| 162 | cmd = callgrindOf(prog, args...) |
Adam Langley | f4e4272 | 2015-06-04 17:45:09 -0700 | [diff] [blame] | 163 | } else if *useGDB { |
| 164 | cmd = gdbOf(prog, args...) |
Robert Sloan | 4d1ac50 | 2017-02-06 08:36:14 -0800 | [diff] [blame] | 165 | } else if *useSDE { |
| 166 | cmd = sdeOf(test.cpu, prog, args...) |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 167 | } else { |
| 168 | cmd = exec.Command(prog, args...) |
| 169 | } |
Robert Sloan | 5d62578 | 2017-02-13 09:55:39 -0800 | [diff] [blame] | 170 | var outBuf bytes.Buffer |
| 171 | cmd.Stdout = &outBuf |
| 172 | cmd.Stderr = &outBuf |
Adam Langley | f4e4272 | 2015-06-04 17:45:09 -0700 | [diff] [blame] | 173 | if mallocNumToFail >= 0 { |
| 174 | cmd.Env = os.Environ() |
| 175 | cmd.Env = append(cmd.Env, "MALLOC_NUMBER_TO_FAIL="+strconv.FormatInt(mallocNumToFail, 10)) |
| 176 | if *mallocTestDebug { |
| 177 | cmd.Env = append(cmd.Env, "MALLOC_ABORT_ON_FAIL=1") |
| 178 | } |
| 179 | cmd.Env = append(cmd.Env, "_MALLOC_CHECK=1") |
| 180 | } |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 181 | |
| 182 | if err := cmd.Start(); err != nil { |
| 183 | return false, err |
| 184 | } |
| 185 | if err := cmd.Wait(); err != nil { |
Adam Langley | f4e4272 | 2015-06-04 17:45:09 -0700 | [diff] [blame] | 186 | if exitError, ok := err.(*exec.ExitError); ok { |
Robert Sloan | c9abfe4 | 2018-11-26 12:19:07 -0800 | [diff] [blame] | 187 | switch exitError.Sys().(syscall.WaitStatus).ExitStatus() { |
| 188 | case 88: |
Adam Langley | f4e4272 | 2015-06-04 17:45:09 -0700 | [diff] [blame] | 189 | return false, errMoreMallocs |
Robert Sloan | c9abfe4 | 2018-11-26 12:19:07 -0800 | [diff] [blame] | 190 | case 89: |
| 191 | fmt.Print(string(outBuf.Bytes())) |
| 192 | return false, errTestSkipped |
Adam Langley | f4e4272 | 2015-06-04 17:45:09 -0700 | [diff] [blame] | 193 | } |
| 194 | } |
Robert Sloan | 5d62578 | 2017-02-13 09:55:39 -0800 | [diff] [blame] | 195 | fmt.Print(string(outBuf.Bytes())) |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 196 | return false, err |
| 197 | } |
| 198 | |
| 199 | // Account for Windows line-endings. |
Robert Sloan | 5d62578 | 2017-02-13 09:55:39 -0800 | [diff] [blame] | 200 | stdout := bytes.Replace(outBuf.Bytes(), []byte("\r\n"), []byte("\n"), -1) |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 201 | |
| 202 | if bytes.HasSuffix(stdout, []byte("PASS\n")) && |
| 203 | (len(stdout) == 5 || stdout[len(stdout)-6] == '\n') { |
| 204 | return true, nil |
| 205 | } |
David Benjamin | f31229b | 2017-01-25 14:08:15 -0500 | [diff] [blame] | 206 | |
| 207 | // Also accept a googletest-style pass line. This is left here in |
| 208 | // transition until the tests are all converted and this script made |
| 209 | // unnecessary. |
| 210 | if bytes.Contains(stdout, []byte("\n[ PASSED ]")) { |
| 211 | return true, nil |
| 212 | } |
| 213 | |
Robert Sloan | 5d62578 | 2017-02-13 09:55:39 -0800 | [diff] [blame] | 214 | fmt.Print(string(outBuf.Bytes())) |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 215 | return false, nil |
| 216 | } |
| 217 | |
Adam Langley | f4e4272 | 2015-06-04 17:45:09 -0700 | [diff] [blame] | 218 | func runTest(test test) (bool, error) { |
| 219 | if *mallocTest < 0 { |
| 220 | return runTestOnce(test, -1) |
| 221 | } |
| 222 | |
| 223 | for mallocNumToFail := int64(*mallocTest); ; mallocNumToFail++ { |
| 224 | if passed, err := runTestOnce(test, mallocNumToFail); err != errMoreMallocs { |
| 225 | if err != nil { |
| 226 | err = fmt.Errorf("at malloc %d: %s", mallocNumToFail, err) |
| 227 | } |
| 228 | return passed, err |
| 229 | } |
| 230 | } |
| 231 | } |
| 232 | |
Kenny Root | b849459 | 2015-09-25 02:29:14 +0000 | [diff] [blame] | 233 | // setWorkingDirectory walks up directories as needed until the current working |
| 234 | // directory is the top of a BoringSSL checkout. |
| 235 | func setWorkingDirectory() { |
| 236 | for i := 0; i < 64; i++ { |
| 237 | if _, err := os.Stat("BUILDING.md"); err == nil { |
| 238 | return |
| 239 | } |
| 240 | os.Chdir("..") |
| 241 | } |
| 242 | |
| 243 | panic("Couldn't find BUILDING.md in a parent directory!") |
| 244 | } |
| 245 | |
| 246 | func parseTestConfig(filename string) ([]test, error) { |
| 247 | in, err := os.Open(filename) |
| 248 | if err != nil { |
| 249 | return nil, err |
| 250 | } |
| 251 | defer in.Close() |
| 252 | |
| 253 | decoder := json.NewDecoder(in) |
Robert Sloan | 4d1ac50 | 2017-02-06 08:36:14 -0800 | [diff] [blame] | 254 | var testArgs [][]string |
| 255 | if err := decoder.Decode(&testArgs); err != nil { |
Kenny Root | b849459 | 2015-09-25 02:29:14 +0000 | [diff] [blame] | 256 | return nil, err |
| 257 | } |
Robert Sloan | 4d1ac50 | 2017-02-06 08:36:14 -0800 | [diff] [blame] | 258 | |
| 259 | var result []test |
| 260 | for _, args := range testArgs { |
| 261 | result = append(result, test{args: args}) |
| 262 | } |
Kenny Root | b849459 | 2015-09-25 02:29:14 +0000 | [diff] [blame] | 263 | return result, nil |
| 264 | } |
| 265 | |
David Benjamin | 4969cc9 | 2016-04-22 15:02:23 -0400 | [diff] [blame] | 266 | func worker(tests <-chan test, results chan<- result, done *sync.WaitGroup) { |
| 267 | defer done.Done() |
| 268 | for test := range tests { |
| 269 | passed, err := runTest(test) |
| 270 | results <- result{test, passed, err} |
| 271 | } |
| 272 | } |
| 273 | |
Robert Sloan | 4562e9d | 2017-10-02 10:26:51 -0700 | [diff] [blame] | 274 | func (t test) shortName() string { |
| 275 | return t.args[0] + t.shardMsg() + t.cpuMsg() |
| 276 | } |
| 277 | |
| 278 | func (t test) longName() string { |
| 279 | return strings.Join(t.args, " ") + t.cpuMsg() |
| 280 | } |
| 281 | |
| 282 | func (t test) shardMsg() string { |
| 283 | if t.numShards == 0 { |
| 284 | return "" |
| 285 | } |
| 286 | |
| 287 | return fmt.Sprintf(" [shard %d/%d]", t.shard+1, t.numShards) |
| 288 | } |
| 289 | |
Robert Sloan | 4d1ac50 | 2017-02-06 08:36:14 -0800 | [diff] [blame] | 290 | func (t test) cpuMsg() string { |
| 291 | if len(t.cpu) == 0 { |
| 292 | return "" |
| 293 | } |
| 294 | |
| 295 | return fmt.Sprintf(" (for CPU %q)", t.cpu) |
| 296 | } |
| 297 | |
Robert Sloan | 8ff0355 | 2017-06-14 12:40:58 -0700 | [diff] [blame] | 298 | func (t test) getGTestShards() ([]test, error) { |
| 299 | if *numWorkers == 1 || len(t.args) != 1 { |
| 300 | return []test{t}, nil |
| 301 | } |
| 302 | |
| 303 | // Only shard the three GTest-based tests. |
| 304 | if t.args[0] != "crypto/crypto_test" && t.args[0] != "ssl/ssl_test" && t.args[0] != "decrepit/decrepit_test" { |
| 305 | return []test{t}, nil |
| 306 | } |
| 307 | |
| 308 | prog := path.Join(*buildDir, t.args[0]) |
| 309 | cmd := exec.Command(prog, "--gtest_list_tests") |
| 310 | var stdout bytes.Buffer |
| 311 | cmd.Stdout = &stdout |
| 312 | if err := cmd.Start(); err != nil { |
| 313 | return nil, err |
| 314 | } |
| 315 | if err := cmd.Wait(); err != nil { |
| 316 | return nil, err |
| 317 | } |
| 318 | |
| 319 | var group string |
| 320 | var tests []string |
| 321 | scanner := bufio.NewScanner(&stdout) |
| 322 | for scanner.Scan() { |
| 323 | line := scanner.Text() |
| 324 | |
| 325 | // Remove the parameter comment and trailing space. |
| 326 | if idx := strings.Index(line, "#"); idx >= 0 { |
| 327 | line = line[:idx] |
| 328 | } |
| 329 | line = strings.TrimSpace(line) |
| 330 | if len(line) == 0 { |
| 331 | continue |
| 332 | } |
| 333 | |
| 334 | if line[len(line)-1] == '.' { |
| 335 | group = line |
| 336 | continue |
| 337 | } |
| 338 | |
| 339 | if len(group) == 0 { |
| 340 | return nil, fmt.Errorf("found test case %q without group", line) |
| 341 | } |
| 342 | tests = append(tests, group+line) |
| 343 | } |
| 344 | |
| 345 | const testsPerShard = 20 |
| 346 | if len(tests) <= testsPerShard { |
| 347 | return []test{t}, nil |
| 348 | } |
| 349 | |
| 350 | // Slow tests which process large test vector files tend to be grouped |
| 351 | // together, so shuffle the order. |
| 352 | shuffled := make([]string, len(tests)) |
| 353 | perm := rand.Perm(len(tests)) |
| 354 | for i, j := range perm { |
| 355 | shuffled[i] = tests[j] |
| 356 | } |
| 357 | |
| 358 | var shards []test |
| 359 | for i := 0; i < len(shuffled); i += testsPerShard { |
| 360 | n := len(shuffled) - i |
| 361 | if n > testsPerShard { |
| 362 | n = testsPerShard |
| 363 | } |
| 364 | shard := t |
| 365 | shard.args = []string{shard.args[0], "--gtest_filter=" + strings.Join(shuffled[i:i+n], ":")} |
Robert Sloan | 4562e9d | 2017-10-02 10:26:51 -0700 | [diff] [blame] | 366 | shard.shard = len(shards) |
Robert Sloan | 8ff0355 | 2017-06-14 12:40:58 -0700 | [diff] [blame] | 367 | shards = append(shards, shard) |
| 368 | } |
| 369 | |
Robert Sloan | 4562e9d | 2017-10-02 10:26:51 -0700 | [diff] [blame] | 370 | for i := range shards { |
| 371 | shards[i].numShards = len(shards) |
| 372 | } |
| 373 | |
Robert Sloan | 8ff0355 | 2017-06-14 12:40:58 -0700 | [diff] [blame] | 374 | return shards, nil |
| 375 | } |
| 376 | |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 377 | func main() { |
| 378 | flag.Parse() |
Kenny Root | b849459 | 2015-09-25 02:29:14 +0000 | [diff] [blame] | 379 | setWorkingDirectory() |
| 380 | |
David Benjamin | 4969cc9 | 2016-04-22 15:02:23 -0400 | [diff] [blame] | 381 | testCases, err := parseTestConfig("util/all_tests.json") |
Kenny Root | b849459 | 2015-09-25 02:29:14 +0000 | [diff] [blame] | 382 | if err != nil { |
| 383 | fmt.Printf("Failed to parse input: %s\n", err) |
| 384 | os.Exit(1) |
| 385 | } |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 386 | |
David Benjamin | 4969cc9 | 2016-04-22 15:02:23 -0400 | [diff] [blame] | 387 | var wg sync.WaitGroup |
| 388 | tests := make(chan test, *numWorkers) |
| 389 | results := make(chan result, *numWorkers) |
| 390 | |
| 391 | for i := 0; i < *numWorkers; i++ { |
| 392 | wg.Add(1) |
| 393 | go worker(tests, results, &wg) |
| 394 | } |
| 395 | |
| 396 | go func() { |
| 397 | for _, test := range testCases { |
Robert Sloan | 4d1ac50 | 2017-02-06 08:36:14 -0800 | [diff] [blame] | 398 | if *useSDE { |
Robert Sloan | 8ff0355 | 2017-06-14 12:40:58 -0700 | [diff] [blame] | 399 | // SDE generates plenty of tasks and gets slower |
| 400 | // with additional sharding. |
Robert Sloan | 4d1ac50 | 2017-02-06 08:36:14 -0800 | [diff] [blame] | 401 | for _, cpu := range sdeCPUs { |
| 402 | testForCPU := test |
| 403 | testForCPU.cpu = cpu |
| 404 | tests <- testForCPU |
| 405 | } |
Robert Sloan | c9abfe4 | 2018-11-26 12:19:07 -0800 | [diff] [blame] | 406 | } else if *simulateARMCPUs { |
| 407 | // This mode is run instead of the default path, |
| 408 | // so also include the native flow. |
| 409 | tests <- test |
| 410 | for _, cpu := range armCPUs { |
| 411 | testForCPU := test |
| 412 | testForCPU.cpu = cpu |
| 413 | tests <- testForCPU |
| 414 | } |
Robert Sloan | 4d1ac50 | 2017-02-06 08:36:14 -0800 | [diff] [blame] | 415 | } else { |
Robert Sloan | 8ff0355 | 2017-06-14 12:40:58 -0700 | [diff] [blame] | 416 | shards, err := test.getGTestShards() |
| 417 | if err != nil { |
| 418 | fmt.Printf("Error listing tests: %s\n", err) |
| 419 | os.Exit(1) |
| 420 | } |
| 421 | for _, shard := range shards { |
| 422 | tests <- shard |
| 423 | } |
Robert Sloan | 4d1ac50 | 2017-02-06 08:36:14 -0800 | [diff] [blame] | 424 | } |
David Benjamin | 4969cc9 | 2016-04-22 15:02:23 -0400 | [diff] [blame] | 425 | } |
| 426 | close(tests) |
| 427 | |
| 428 | wg.Wait() |
| 429 | close(results) |
| 430 | }() |
| 431 | |
Robert Sloan | c9abfe4 | 2018-11-26 12:19:07 -0800 | [diff] [blame] | 432 | testOutput := testresult.NewResults() |
| 433 | var failed, skipped []test |
David Benjamin | 4969cc9 | 2016-04-22 15:02:23 -0400 | [diff] [blame] | 434 | for testResult := range results { |
| 435 | test := testResult.Test |
Robert Sloan | 4d1ac50 | 2017-02-06 08:36:14 -0800 | [diff] [blame] | 436 | args := test.args |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 437 | |
Robert Sloan | c9abfe4 | 2018-11-26 12:19:07 -0800 | [diff] [blame] | 438 | if testResult.Error == errTestSkipped { |
| 439 | fmt.Printf("%s\n", test.longName()) |
| 440 | fmt.Printf("%s was skipped\n", args[0]) |
| 441 | skipped = append(skipped, test) |
| 442 | testOutput.AddSkip(test.longName()) |
| 443 | } else if testResult.Error != nil { |
Robert Sloan | 4562e9d | 2017-10-02 10:26:51 -0700 | [diff] [blame] | 444 | fmt.Printf("%s\n", test.longName()) |
Robert Sloan | 4d1ac50 | 2017-02-06 08:36:14 -0800 | [diff] [blame] | 445 | fmt.Printf("%s failed to complete: %s\n", args[0], testResult.Error) |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 446 | failed = append(failed, test) |
Robert Sloan | c9abfe4 | 2018-11-26 12:19:07 -0800 | [diff] [blame] | 447 | testOutput.AddResult(test.longName(), "CRASH") |
David Benjamin | 4969cc9 | 2016-04-22 15:02:23 -0400 | [diff] [blame] | 448 | } else if !testResult.Passed { |
Robert Sloan | 4562e9d | 2017-10-02 10:26:51 -0700 | [diff] [blame] | 449 | fmt.Printf("%s\n", test.longName()) |
Robert Sloan | 4d1ac50 | 2017-02-06 08:36:14 -0800 | [diff] [blame] | 450 | fmt.Printf("%s failed to print PASS on the last line.\n", args[0]) |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 451 | failed = append(failed, test) |
Robert Sloan | c9abfe4 | 2018-11-26 12:19:07 -0800 | [diff] [blame] | 452 | testOutput.AddResult(test.longName(), "FAIL") |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 453 | } else { |
Robert Sloan | 4562e9d | 2017-10-02 10:26:51 -0700 | [diff] [blame] | 454 | fmt.Printf("%s\n", test.shortName()) |
Robert Sloan | c9abfe4 | 2018-11-26 12:19:07 -0800 | [diff] [blame] | 455 | testOutput.AddResult(test.longName(), "PASS") |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 456 | } |
| 457 | } |
| 458 | |
| 459 | if *jsonOutput != "" { |
Robert Sloan | c9abfe4 | 2018-11-26 12:19:07 -0800 | [diff] [blame] | 460 | if err := testOutput.WriteToFile(*jsonOutput); err != nil { |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 461 | fmt.Fprintf(os.Stderr, "Error: %s\n", err) |
| 462 | } |
| 463 | } |
| 464 | |
Robert Sloan | c9abfe4 | 2018-11-26 12:19:07 -0800 | [diff] [blame] | 465 | if len(skipped) > 0 { |
| 466 | fmt.Printf("\n%d of %d tests were skipped:\n", len(skipped), len(testCases)) |
| 467 | for _, test := range skipped { |
| 468 | fmt.Printf("\t%s%s\n", strings.Join(test.args, " "), test.cpuMsg()) |
| 469 | } |
| 470 | } |
| 471 | |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 472 | if len(failed) > 0 { |
David Benjamin | 4969cc9 | 2016-04-22 15:02:23 -0400 | [diff] [blame] | 473 | fmt.Printf("\n%d of %d tests failed:\n", len(failed), len(testCases)) |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 474 | for _, test := range failed { |
Robert Sloan | 8ff0355 | 2017-06-14 12:40:58 -0700 | [diff] [blame] | 475 | fmt.Printf("\t%s%s\n", strings.Join(test.args, " "), test.cpuMsg()) |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 476 | } |
| 477 | os.Exit(1) |
| 478 | } |
| 479 | |
| 480 | fmt.Printf("\nAll tests passed!\n") |
| 481 | } |