Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 1 | /* Copyright (c) 2015, Google Inc. |
| 2 | * |
| 3 | * Permission to use, copy, modify, and/or distribute this software for any |
| 4 | * purpose with or without fee is hereby granted, provided that the above |
| 5 | * copyright notice and this permission notice appear in all copies. |
| 6 | * |
| 7 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES |
| 8 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF |
| 9 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY |
| 10 | * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES |
| 11 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION |
| 12 | * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN |
| 13 | * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ |
| 14 | |
| 15 | package main |
| 16 | |
| 17 | import ( |
| 18 | "bytes" |
| 19 | "encoding/json" |
| 20 | "flag" |
| 21 | "fmt" |
| 22 | "os" |
| 23 | "os/exec" |
| 24 | "path" |
Adam Langley | f4e4272 | 2015-06-04 17:45:09 -0700 | [diff] [blame] | 25 | "strconv" |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 26 | "strings" |
David Benjamin | 4969cc9 | 2016-04-22 15:02:23 -0400 | [diff] [blame] | 27 | "sync" |
Adam Langley | f4e4272 | 2015-06-04 17:45:09 -0700 | [diff] [blame] | 28 | "syscall" |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 29 | "time" |
| 30 | ) |
| 31 | |
| 32 | // TODO(davidben): Link tests with the malloc shim and port -malloc-test to this runner. |
| 33 | |
| 34 | var ( |
Adam Langley | f4e4272 | 2015-06-04 17:45:09 -0700 | [diff] [blame] | 35 | useValgrind = flag.Bool("valgrind", false, "If true, run code under valgrind") |
David Benjamin | 4969cc9 | 2016-04-22 15:02:23 -0400 | [diff] [blame] | 36 | useCallgrind = flag.Bool("callgrind", false, "If true, run code under valgrind to generate callgrind traces.") |
Adam Langley | f4e4272 | 2015-06-04 17:45:09 -0700 | [diff] [blame] | 37 | useGDB = flag.Bool("gdb", false, "If true, run BoringSSL code under gdb") |
| 38 | buildDir = flag.String("build-dir", "build", "The build directory to run the tests from.") |
David Benjamin | 4969cc9 | 2016-04-22 15:02:23 -0400 | [diff] [blame] | 39 | numWorkers = flag.Int("num-workers", 1, "Runs the given number of workers when testing.") |
Adam Langley | f4e4272 | 2015-06-04 17:45:09 -0700 | [diff] [blame] | 40 | jsonOutput = flag.String("json-output", "", "The file to output JSON results to.") |
| 41 | mallocTest = flag.Int64("malloc-test", -1, "If non-negative, run each test with each malloc in turn failing from the given number onwards.") |
| 42 | mallocTestDebug = flag.Bool("malloc-test-debug", false, "If true, ask each test to abort rather than fail a malloc. This can be used with a specific value for --malloc-test to identity the malloc failing that is causing problems.") |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 43 | ) |
| 44 | |
| 45 | type test []string |
| 46 | |
David Benjamin | 4969cc9 | 2016-04-22 15:02:23 -0400 | [diff] [blame] | 47 | type result struct { |
| 48 | Test test |
| 49 | Passed bool |
| 50 | Error error |
| 51 | } |
| 52 | |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 53 | // testOutput is a representation of Chromium's JSON test result format. See |
| 54 | // https://www.chromium.org/developers/the-json-test-results-format |
| 55 | type testOutput struct { |
| 56 | Version int `json:"version"` |
| 57 | Interrupted bool `json:"interrupted"` |
| 58 | PathDelimiter string `json:"path_delimiter"` |
| 59 | SecondsSinceEpoch float64 `json:"seconds_since_epoch"` |
| 60 | NumFailuresByType map[string]int `json:"num_failures_by_type"` |
| 61 | Tests map[string]testResult `json:"tests"` |
| 62 | } |
| 63 | |
| 64 | type testResult struct { |
| 65 | Actual string `json:"actual"` |
| 66 | Expected string `json:"expected"` |
| 67 | IsUnexpected bool `json:"is_unexpected"` |
| 68 | } |
| 69 | |
| 70 | func newTestOutput() *testOutput { |
| 71 | return &testOutput{ |
| 72 | Version: 3, |
| 73 | PathDelimiter: ".", |
| 74 | SecondsSinceEpoch: float64(time.Now().UnixNano()) / float64(time.Second/time.Nanosecond), |
| 75 | NumFailuresByType: make(map[string]int), |
| 76 | Tests: make(map[string]testResult), |
| 77 | } |
| 78 | } |
| 79 | |
| 80 | func (t *testOutput) addResult(name, result string) { |
| 81 | if _, found := t.Tests[name]; found { |
| 82 | panic(name) |
| 83 | } |
| 84 | t.Tests[name] = testResult{ |
| 85 | Actual: result, |
| 86 | Expected: "PASS", |
| 87 | IsUnexpected: result != "PASS", |
| 88 | } |
| 89 | t.NumFailuresByType[result]++ |
| 90 | } |
| 91 | |
| 92 | func (t *testOutput) writeTo(name string) error { |
| 93 | file, err := os.Create(name) |
| 94 | if err != nil { |
| 95 | return err |
| 96 | } |
| 97 | defer file.Close() |
| 98 | out, err := json.MarshalIndent(t, "", " ") |
| 99 | if err != nil { |
| 100 | return err |
| 101 | } |
| 102 | _, err = file.Write(out) |
| 103 | return err |
| 104 | } |
| 105 | |
| 106 | func valgrindOf(dbAttach bool, path string, args ...string) *exec.Cmd { |
David Benjamin | 7c0d06c | 2016-08-11 13:26:41 -0400 | [diff] [blame^] | 107 | valgrindArgs := []string{"--error-exitcode=99", "--track-origins=yes", "--leak-check=full", "--quiet"} |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 108 | if dbAttach { |
| 109 | valgrindArgs = append(valgrindArgs, "--db-attach=yes", "--db-command=xterm -e gdb -nw %f %p") |
| 110 | } |
| 111 | valgrindArgs = append(valgrindArgs, path) |
| 112 | valgrindArgs = append(valgrindArgs, args...) |
| 113 | |
| 114 | return exec.Command("valgrind", valgrindArgs...) |
| 115 | } |
| 116 | |
David Benjamin | 4969cc9 | 2016-04-22 15:02:23 -0400 | [diff] [blame] | 117 | func callgrindOf(path string, args ...string) *exec.Cmd { |
| 118 | valgrindArgs := []string{"-q", "--tool=callgrind", "--dump-instr=yes", "--collect-jumps=yes", "--callgrind-out-file=" + *buildDir + "/callgrind/callgrind.out.%p"} |
| 119 | valgrindArgs = append(valgrindArgs, path) |
| 120 | valgrindArgs = append(valgrindArgs, args...) |
| 121 | |
| 122 | return exec.Command("valgrind", valgrindArgs...) |
| 123 | } |
| 124 | |
Adam Langley | f4e4272 | 2015-06-04 17:45:09 -0700 | [diff] [blame] | 125 | func gdbOf(path string, args ...string) *exec.Cmd { |
| 126 | xtermArgs := []string{"-e", "gdb", "--args"} |
| 127 | xtermArgs = append(xtermArgs, path) |
| 128 | xtermArgs = append(xtermArgs, args...) |
| 129 | |
| 130 | return exec.Command("xterm", xtermArgs...) |
| 131 | } |
| 132 | |
| 133 | type moreMallocsError struct{} |
| 134 | |
| 135 | func (moreMallocsError) Error() string { |
| 136 | return "child process did not exhaust all allocation calls" |
| 137 | } |
| 138 | |
| 139 | var errMoreMallocs = moreMallocsError{} |
| 140 | |
| 141 | func runTestOnce(test test, mallocNumToFail int64) (passed bool, err error) { |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 142 | prog := path.Join(*buildDir, test[0]) |
| 143 | args := test[1:] |
| 144 | var cmd *exec.Cmd |
| 145 | if *useValgrind { |
| 146 | cmd = valgrindOf(false, prog, args...) |
David Benjamin | 4969cc9 | 2016-04-22 15:02:23 -0400 | [diff] [blame] | 147 | } else if *useCallgrind { |
| 148 | cmd = callgrindOf(prog, args...) |
Adam Langley | f4e4272 | 2015-06-04 17:45:09 -0700 | [diff] [blame] | 149 | } else if *useGDB { |
| 150 | cmd = gdbOf(prog, args...) |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 151 | } else { |
| 152 | cmd = exec.Command(prog, args...) |
| 153 | } |
| 154 | var stdoutBuf bytes.Buffer |
Adam Langley | f4e4272 | 2015-06-04 17:45:09 -0700 | [diff] [blame] | 155 | var stderrBuf bytes.Buffer |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 156 | cmd.Stdout = &stdoutBuf |
Adam Langley | f4e4272 | 2015-06-04 17:45:09 -0700 | [diff] [blame] | 157 | cmd.Stderr = &stderrBuf |
| 158 | if mallocNumToFail >= 0 { |
| 159 | cmd.Env = os.Environ() |
| 160 | cmd.Env = append(cmd.Env, "MALLOC_NUMBER_TO_FAIL="+strconv.FormatInt(mallocNumToFail, 10)) |
| 161 | if *mallocTestDebug { |
| 162 | cmd.Env = append(cmd.Env, "MALLOC_ABORT_ON_FAIL=1") |
| 163 | } |
| 164 | cmd.Env = append(cmd.Env, "_MALLOC_CHECK=1") |
| 165 | } |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 166 | |
| 167 | if err := cmd.Start(); err != nil { |
| 168 | return false, err |
| 169 | } |
| 170 | if err := cmd.Wait(); err != nil { |
Adam Langley | f4e4272 | 2015-06-04 17:45:09 -0700 | [diff] [blame] | 171 | if exitError, ok := err.(*exec.ExitError); ok { |
| 172 | if exitError.Sys().(syscall.WaitStatus).ExitStatus() == 88 { |
| 173 | return false, errMoreMallocs |
| 174 | } |
| 175 | } |
| 176 | fmt.Print(string(stderrBuf.Bytes())) |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 177 | return false, err |
| 178 | } |
Adam Langley | f4e4272 | 2015-06-04 17:45:09 -0700 | [diff] [blame] | 179 | fmt.Print(string(stderrBuf.Bytes())) |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 180 | |
| 181 | // Account for Windows line-endings. |
| 182 | stdout := bytes.Replace(stdoutBuf.Bytes(), []byte("\r\n"), []byte("\n"), -1) |
| 183 | |
| 184 | if bytes.HasSuffix(stdout, []byte("PASS\n")) && |
| 185 | (len(stdout) == 5 || stdout[len(stdout)-6] == '\n') { |
| 186 | return true, nil |
| 187 | } |
| 188 | return false, nil |
| 189 | } |
| 190 | |
Adam Langley | f4e4272 | 2015-06-04 17:45:09 -0700 | [diff] [blame] | 191 | func runTest(test test) (bool, error) { |
| 192 | if *mallocTest < 0 { |
| 193 | return runTestOnce(test, -1) |
| 194 | } |
| 195 | |
| 196 | for mallocNumToFail := int64(*mallocTest); ; mallocNumToFail++ { |
| 197 | if passed, err := runTestOnce(test, mallocNumToFail); err != errMoreMallocs { |
| 198 | if err != nil { |
| 199 | err = fmt.Errorf("at malloc %d: %s", mallocNumToFail, err) |
| 200 | } |
| 201 | return passed, err |
| 202 | } |
| 203 | } |
| 204 | } |
| 205 | |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 206 | // shortTestName returns the short name of a test. Except for evp_test, it |
| 207 | // assumes that any argument which ends in .txt is a path to a data file and not |
| 208 | // relevant to the test's uniqueness. |
| 209 | func shortTestName(test test) string { |
| 210 | var args []string |
| 211 | for _, arg := range test { |
| 212 | if test[0] == "crypto/evp/evp_test" || !strings.HasSuffix(arg, ".txt") { |
| 213 | args = append(args, arg) |
| 214 | } |
| 215 | } |
| 216 | return strings.Join(args, " ") |
| 217 | } |
| 218 | |
Kenny Root | b849459 | 2015-09-25 02:29:14 +0000 | [diff] [blame] | 219 | // setWorkingDirectory walks up directories as needed until the current working |
| 220 | // directory is the top of a BoringSSL checkout. |
| 221 | func setWorkingDirectory() { |
| 222 | for i := 0; i < 64; i++ { |
| 223 | if _, err := os.Stat("BUILDING.md"); err == nil { |
| 224 | return |
| 225 | } |
| 226 | os.Chdir("..") |
| 227 | } |
| 228 | |
| 229 | panic("Couldn't find BUILDING.md in a parent directory!") |
| 230 | } |
| 231 | |
| 232 | func parseTestConfig(filename string) ([]test, error) { |
| 233 | in, err := os.Open(filename) |
| 234 | if err != nil { |
| 235 | return nil, err |
| 236 | } |
| 237 | defer in.Close() |
| 238 | |
| 239 | decoder := json.NewDecoder(in) |
| 240 | var result []test |
| 241 | if err := decoder.Decode(&result); err != nil { |
| 242 | return nil, err |
| 243 | } |
| 244 | return result, nil |
| 245 | } |
| 246 | |
David Benjamin | 4969cc9 | 2016-04-22 15:02:23 -0400 | [diff] [blame] | 247 | func worker(tests <-chan test, results chan<- result, done *sync.WaitGroup) { |
| 248 | defer done.Done() |
| 249 | for test := range tests { |
| 250 | passed, err := runTest(test) |
| 251 | results <- result{test, passed, err} |
| 252 | } |
| 253 | } |
| 254 | |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 255 | func main() { |
| 256 | flag.Parse() |
Kenny Root | b849459 | 2015-09-25 02:29:14 +0000 | [diff] [blame] | 257 | setWorkingDirectory() |
| 258 | |
David Benjamin | 4969cc9 | 2016-04-22 15:02:23 -0400 | [diff] [blame] | 259 | testCases, err := parseTestConfig("util/all_tests.json") |
Kenny Root | b849459 | 2015-09-25 02:29:14 +0000 | [diff] [blame] | 260 | if err != nil { |
| 261 | fmt.Printf("Failed to parse input: %s\n", err) |
| 262 | os.Exit(1) |
| 263 | } |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 264 | |
David Benjamin | 4969cc9 | 2016-04-22 15:02:23 -0400 | [diff] [blame] | 265 | var wg sync.WaitGroup |
| 266 | tests := make(chan test, *numWorkers) |
| 267 | results := make(chan result, *numWorkers) |
| 268 | |
| 269 | for i := 0; i < *numWorkers; i++ { |
| 270 | wg.Add(1) |
| 271 | go worker(tests, results, &wg) |
| 272 | } |
| 273 | |
| 274 | go func() { |
| 275 | for _, test := range testCases { |
| 276 | tests <- test |
| 277 | } |
| 278 | close(tests) |
| 279 | |
| 280 | wg.Wait() |
| 281 | close(results) |
| 282 | }() |
| 283 | |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 284 | testOutput := newTestOutput() |
| 285 | var failed []test |
David Benjamin | 4969cc9 | 2016-04-22 15:02:23 -0400 | [diff] [blame] | 286 | for testResult := range results { |
| 287 | test := testResult.Test |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 288 | |
David Benjamin | 4969cc9 | 2016-04-22 15:02:23 -0400 | [diff] [blame] | 289 | fmt.Printf("%s\n", strings.Join([]string(test), " ")) |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 290 | name := shortTestName(test) |
David Benjamin | 4969cc9 | 2016-04-22 15:02:23 -0400 | [diff] [blame] | 291 | if testResult.Error != nil { |
| 292 | fmt.Printf("%s failed to complete: %s\n", test[0], testResult.Error) |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 293 | failed = append(failed, test) |
| 294 | testOutput.addResult(name, "CRASHED") |
David Benjamin | 4969cc9 | 2016-04-22 15:02:23 -0400 | [diff] [blame] | 295 | } else if !testResult.Passed { |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 296 | fmt.Printf("%s failed to print PASS on the last line.\n", test[0]) |
| 297 | failed = append(failed, test) |
| 298 | testOutput.addResult(name, "FAIL") |
| 299 | } else { |
| 300 | testOutput.addResult(name, "PASS") |
| 301 | } |
| 302 | } |
| 303 | |
| 304 | if *jsonOutput != "" { |
| 305 | if err := testOutput.writeTo(*jsonOutput); err != nil { |
| 306 | fmt.Fprintf(os.Stderr, "Error: %s\n", err) |
| 307 | } |
| 308 | } |
| 309 | |
| 310 | if len(failed) > 0 { |
David Benjamin | 4969cc9 | 2016-04-22 15:02:23 -0400 | [diff] [blame] | 311 | fmt.Printf("\n%d of %d tests failed:\n", len(failed), len(testCases)) |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 312 | for _, test := range failed { |
| 313 | fmt.Printf("\t%s\n", strings.Join([]string(test), " ")) |
| 314 | } |
| 315 | os.Exit(1) |
| 316 | } |
| 317 | |
| 318 | fmt.Printf("\nAll tests passed!\n") |
| 319 | } |