| // Copyright 2019 The SwiftShader Authors. All Rights Reserved. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| // Regres is a tool that detects test regressions with SwiftShader changes. |
| // |
| // Regres monitors changes that have been put up for review with Gerrit. |
| // Once a new patchset has been found, regres will checkout, build and test the |
| // change against the parent changelist. Any differences in results are reported |
| // as a review comment on the change. |
| // |
| // Once a day regres will also test another, larger set of tests, and post the |
| // full test results as a Gerrit changelist. The CI test lists can be based from |
| // this daily test list, so testing can be limited to tests that were known to |
| // pass. |
| package main |
| |
| import ( |
| "bytes" |
| "encoding/json" |
| "errors" |
| "flag" |
| "fmt" |
| "io/ioutil" |
| "log" |
| "os" |
| "os/exec" |
| "path/filepath" |
| "regexp" |
| "runtime" |
| "sort" |
| "strings" |
| "sync" |
| "time" |
| |
| "./cause" |
| "./git" |
| "./shell" |
| |
| gerrit "github.com/andygrunwald/go-gerrit" |
| ) |
| |
| const ( |
| gitURL = "https://swiftshader.googlesource.com/SwiftShader" |
| gerritURL = "https://swiftshader-review.googlesource.com/" |
| reportHeader = "Regres report:" |
| dataVersion = 1 |
| changeUpdateFrequency = time.Minute * 5 |
| changeQueryFrequency = time.Minute * 5 |
| testTimeout = time.Minute * 5 // timeout for a single test |
| buildTimeout = time.Minute * 10 // timeout for a build |
| dailyUpdateTestListHour = 5 // 5am |
| fullTestList = "tests/regres/full-tests.json" |
| ciTestList = "tests/regres/ci-tests.json" |
| ) |
| |
| var ( |
| numParallelTests = runtime.NumCPU() |
| |
| deqpPath = flag.String("deqp", "", "path to the deqp build directory") |
| cacheDir = flag.String("cache", "cache", "path to the output cache directory") |
| gerritEmail = flag.String("email", "$SS_REGRES_EMAIL", "gerrit email address for posting regres results") |
| gerritUser = flag.String("user", "$SS_REGRES_USER", "gerrit username for posting regres results") |
| gerritPass = flag.String("pass", "$SS_REGRES_PASS", "gerrit password for posting regres results") |
| keepCheckouts = flag.Bool("keep", false, "don't delete checkout directories after use") |
| dryRun = flag.Bool("dry", false, "don't post regres reports to gerrit") |
| maxProcMemory = flag.Uint64("max-proc-mem", shell.MaxProcMemory, "maximum virtual memory per child process") |
| ) |
| |
| func main() { |
| if runtime.GOOS != "linux" { |
| log.Fatal("regres only currently runs on linux") |
| } |
| |
| flag.ErrHelp = errors.New("regres is a tool to detect regressions between versions of SwiftShader") |
| flag.Parse() |
| |
| shell.MaxProcMemory = *maxProcMemory |
| |
| r := regres{ |
| deqpBuild: *deqpPath, |
| cacheRoot: *cacheDir, |
| gerritEmail: os.ExpandEnv(*gerritEmail), |
| gerritUser: os.ExpandEnv(*gerritUser), |
| gerritPass: os.ExpandEnv(*gerritPass), |
| keepCheckouts: *keepCheckouts, |
| dryRun: *dryRun, |
| } |
| |
| if err := r.run(); err != nil { |
| fmt.Fprintln(os.Stderr, err) |
| os.Exit(-1) |
| } |
| } |
| |
| type regres struct { |
| deqpBuild string // path to the build directory of deqp |
| cmake string // path to cmake |
| make string // path to make |
| cacheRoot string // path to the regres cache directory |
| gerritEmail string // gerrit email address used for posting results |
| gerritUser string // gerrit username used for posting results |
| gerritPass string // gerrit password used for posting results |
| keepCheckouts bool // don't delete source & build checkouts after testing |
| dryRun bool // don't post any reviews |
| maxProcMemory uint64 // max virtual memory for child processes |
| } |
| |
| // resolveDirs ensures that the necessary directories used can be found, and |
| // expands them to absolute paths. |
| func (r *regres) resolveDirs() error { |
| for _, path := range []*string{ |
| &r.deqpBuild, |
| &r.cacheRoot, |
| } { |
| abs, err := filepath.Abs(*path) |
| if err != nil { |
| return cause.Wrap(err, "Couldn't find path '%v'", *path) |
| } |
| if _, err := os.Stat(abs); err != nil { |
| return cause.Wrap(err, "Couldn't find path '%v'", abs) |
| } |
| *path = abs |
| } |
| return nil |
| } |
| |
| // resolveExes resolves all external executables used by regres. |
| func (r *regres) resolveExes() error { |
| type exe struct { |
| name string |
| path *string |
| } |
| for _, e := range []exe{ |
| {"cmake", &r.cmake}, |
| {"make", &r.make}, |
| } { |
| path, err := exec.LookPath(e.name) |
| if err != nil { |
| return cause.Wrap(err, "Couldn't find path to %s", e.name) |
| } |
| *e.path = path |
| } |
| return nil |
| } |
| |
| // run performs the main processing loop for the regress tool. It: |
| // * Scans for open and recently updated changes in gerrit using queryChanges() |
| // and changeInfo.update(). |
| // * Builds the most recent patchset and the commit's parent CL using |
| // r.newTest(<hash>).lazyRun(). |
| // * Compares the results of the tests using compare(). |
| // * Posts the results of the compare to gerrit as a review. |
| // * Repeats the above steps until the process is interrupted. |
| func (r *regres) run() error { |
| if err := r.resolveExes(); err != nil { |
| return cause.Wrap(err, "Couldn't resolve all exes") |
| } |
| |
| if err := r.resolveDirs(); err != nil { |
| return cause.Wrap(err, "Couldn't resolve all directories") |
| } |
| |
| client, err := gerrit.NewClient(gerritURL, nil) |
| if err != nil { |
| return cause.Wrap(err, "Couldn't create gerrit client") |
| } |
| if r.gerritUser != "" { |
| client.Authentication.SetBasicAuth(r.gerritUser, r.gerritPass) |
| } |
| |
| changes := map[string]*changeInfo{} // Change ID -> changeInfo |
| lastUpdatedTestLists := toDate(time.Now()) |
| lastQueriedChanges := time.Time{} |
| |
| for { |
| if now := time.Now(); toDate(now) != lastUpdatedTestLists && now.Hour() >= dailyUpdateTestListHour { |
| lastUpdatedTestLists = toDate(now) |
| if err := r.updateTestLists(client); err != nil { |
| log.Println(err.Error()) |
| } |
| } |
| |
| // Update list of tracked changes. |
| if time.Since(lastQueriedChanges) > changeQueryFrequency { |
| lastQueriedChanges = time.Now() |
| if err := queryChanges(client, changes); err != nil { |
| log.Println(err.Error()) |
| } |
| } |
| |
| // Update change info. |
| for _, change := range changes { |
| if time.Since(change.lastUpdated) > changeUpdateFrequency { |
| change.lastUpdated = time.Now() |
| err := change.update(client) |
| if err != nil { |
| log.Println(cause.Wrap(err, "Couldn't update info for change '%s'", change.id)) |
| } |
| } |
| } |
| |
| // Find the change with the highest priority. |
| var change *changeInfo |
| numPending := 0 |
| for _, c := range changes { |
| if c.pending { |
| numPending++ |
| if change == nil || c.priority > change.priority { |
| change = c |
| } |
| } |
| } |
| |
| if change == nil { |
| // Everything up to date. Take a break. |
| log.Println("Nothing to do. Sleeping") |
| time.Sleep(time.Minute) |
| continue |
| } |
| |
| log.Printf("%d changes queued for testing\n", numPending) |
| |
| log.Printf("Testing change '%s'\n", change.id) |
| |
| // Get the test results for the latest patchset in the change. |
| latest, err := r.newTest(ciTestList, change.latest).lazyRun() |
| if err != nil { |
| log.Println(cause.Wrap(err, "Failed to test changelist '%s'", change.latest)) |
| time.Sleep(time.Minute) |
| continue |
| } |
| |
| // Get the test results for the changes's parent changelist. |
| parent, err := r.newTest(ciTestList, change.parent).lazyRun() |
| if err != nil { |
| log.Println(cause.Wrap(err, "Failed to test changelist '%s'", change.parent)) |
| time.Sleep(time.Minute) |
| continue |
| } |
| |
| // Compare the latest patchset to the change's parent commit. |
| msg := compare(parent, latest) |
| |
| // Always include the reportHeader in the message. |
| // changeInfo.update() uses this header to detect whether a patchset has |
| // already got a test result. |
| msg = reportHeader + "\n\n" + msg |
| |
| if r.dryRun { |
| log.Printf("DRY RUN: add review to change '%v':\n%v\n", change.id, msg) |
| } else { |
| log.Printf("Posting review to '%s'\n", change.id) |
| _, _, err = client.Changes.SetReview(change.id, change.latest.String(), &gerrit.ReviewInput{ |
| Message: msg, |
| Tag: "autogenerated:regress", |
| }) |
| if err != nil { |
| return cause.Wrap(err, "Failed to post comments on change '%s'", change.id) |
| } |
| } |
| change.pending = false |
| } |
| } |
| |
| func (r *regres) updateTestLists(client *gerrit.Client) error { |
| log.Println("Updating test lists") |
| |
| headHash, err := git.FetchRefHash("HEAD", gitURL) |
| if err != nil { |
| return cause.Wrap(err, "Could not get hash of master HEAD") |
| } |
| |
| // Get the full test results for latest master. |
| t := r.newTest(fullTestList, headHash) |
| |
| // Keep the checked out directory after the test is run. We want this so |
| // we can build a new patchset containing the updated test lists. |
| t.keepCheckouts = true |
| if !r.keepCheckouts { |
| defer os.RemoveAll(t.srcDir) |
| } |
| |
| if _, err := t.run(); err != nil { |
| return cause.Wrap(err, "Failed to test changelist '%s'", headHash) |
| } |
| |
| // Stage all the updated test files. |
| for _, path := range t.writtenTestLists { |
| log.Println("Staging", path) |
| git.Add(t.srcDir, path) |
| } |
| |
| log.Println("Checking for existing test list") |
| results, _, err := client.Changes.QueryChanges(&gerrit.QueryChangeOptions{ |
| QueryOptions: gerrit.QueryOptions{ |
| Query: []string{fmt.Sprintf(`status:open+owner:"%v"`, r.gerritEmail)}, |
| Limit: 1, |
| }, |
| }) |
| if err != nil { |
| return cause.Wrap(err, "Failed to checking for existing test list") |
| } |
| |
| commitMsg := strings.Builder{} |
| commitMsg.WriteString("Regres: Update test lists @ " + headHash.String()[:8]) |
| if results != nil && len(*results) > 0 { |
| // Reuse gerrit change ID if there's already a change up for review. |
| id := (*results)[0].ChangeID |
| commitMsg.WriteString("\n\n") |
| commitMsg.WriteString("Change-Id: " + id) |
| } |
| |
| if err := git.Commit(t.srcDir, commitMsg.String(), git.CommitFlags{ |
| Name: "SwiftShader Regression Bot", |
| Email: r.gerritEmail, |
| }); err != nil { |
| return cause.Wrap(err, "Failed to commit test results") |
| } |
| |
| if r.dryRun { |
| log.Printf("DRY RUN: post results for review") |
| } else { |
| log.Println("Pushing test results for review") |
| if err := git.Push(t.srcDir, gitURL, "HEAD", "refs/for/master", git.PushFlags{ |
| Username: r.gerritUser, |
| Password: r.gerritPass, |
| }); err != nil { |
| return cause.Wrap(err, "Failed to push test results for review") |
| } |
| log.Println("Test results posted for review") |
| } |
| |
| return nil |
| } |
| |
| // changeInfo holds the important information about a single, open change in |
| // gerrit. |
| type changeInfo struct { |
| id string // Gerrit change ID. |
| pending bool // Is this change waiting a test for the latest patchset? |
| priority int // Calculated priority based on Gerrit labels. |
| latest git.Hash // Git hash of the latest patchset in the change. |
| parent git.Hash // Git hash of the changelist this change is based on. |
| lastUpdated time.Time // Time the change was last fetched. |
| } |
| |
| // queryChanges updates the changes map by querying gerrit for the latest open |
| // changes. |
| func queryChanges(client *gerrit.Client, changes map[string]*changeInfo) error { |
| log.Println("Checking for latest changes") |
| results, _, err := client.Changes.QueryChanges(&gerrit.QueryChangeOptions{ |
| QueryOptions: gerrit.QueryOptions{ |
| Query: []string{"status:open+-age:3d"}, |
| Limit: 100, |
| }, |
| }) |
| if err != nil { |
| return cause.Wrap(err, "Failed to get list of changes") |
| } |
| |
| ids := map[string]bool{} |
| for _, r := range *results { |
| ids[r.ChangeID] = true |
| } |
| |
| // Add new changes |
| for id := range ids { |
| if _, found := changes[id]; !found { |
| log.Printf("Tracking new change '%v'\n", id) |
| changes[id] = &changeInfo{id: id} |
| } |
| } |
| |
| // Remove old changes |
| for id := range changes { |
| if found := ids[id]; !found { |
| log.Printf("Untracking change '%v'\n", id) |
| delete(changes, id) |
| } |
| } |
| |
| return nil |
| } |
| |
| // update queries gerrit for information about the given change. |
| func (c *changeInfo) update(client *gerrit.Client) error { |
| change, _, err := client.Changes.GetChange(c.id, &gerrit.ChangeOptions{ |
| AdditionalFields: []string{"CURRENT_REVISION", "CURRENT_COMMIT", "MESSAGES", "LABELS"}, |
| }) |
| if err != nil { |
| return cause.Wrap(err, "Getting info for change '%s'", c.id) |
| } |
| |
| current, ok := change.Revisions[change.CurrentRevision] |
| if !ok { |
| return fmt.Errorf("Couldn't find current revision for change '%s'", c.id) |
| } |
| |
| if len(current.Commit.Parents) == 0 { |
| return fmt.Errorf("Couldn't find current commit for change '%s' has no parents(?)", c.id) |
| } |
| |
| kokoroPresubmit := change.Labels["Kokoro-Presubmit"].Approved.AccountID != 0 |
| codeReviewScore := change.Labels["Code-Review"].Value |
| presubmitReady := change.Labels["Presubmit-Ready"].Approved.AccountID != 0 |
| |
| c.priority = 0 |
| if presubmitReady { |
| c.priority += 10 |
| } |
| c.priority += codeReviewScore |
| if kokoroPresubmit { |
| c.priority++ |
| } |
| |
| // Is the change from a Googler? |
| canTest := strings.HasSuffix(current.Commit.Committer.Email, "@google.com") |
| |
| // Has the latest patchset already been tested? |
| if canTest { |
| for _, msg := range change.Messages { |
| if msg.RevisionNumber == current.Number && |
| strings.Contains(msg.Message, reportHeader) { |
| canTest = false |
| break |
| } |
| } |
| } |
| |
| c.pending = canTest |
| c.latest = git.ParseHash(change.CurrentRevision) |
| c.parent = git.ParseHash(current.Commit.Parents[0].Commit) |
| |
| return nil |
| } |
| |
| func (r *regres) newTest(testListPath string, commit git.Hash) *test { |
| srcDir := filepath.Join(r.cacheRoot, "src", commit.String()) |
| resDir := filepath.Join(r.cacheRoot, "res", commit.String()) |
| return &test{ |
| r: r, |
| commit: commit, |
| srcDir: srcDir, |
| resDir: resDir, |
| outDir: filepath.Join(srcDir, "out"), |
| buildDir: filepath.Join(srcDir, "build"), |
| testListPath: testListPath, |
| } |
| } |
| |
| type test struct { |
| r *regres |
| commit git.Hash // hash of the commit to test |
| srcDir string // directory for the SwiftShader checkout |
| resDir string // directory for the test results |
| outDir string // directory for SwiftShader output |
| buildDir string // directory for SwiftShader build |
| keepCheckouts bool // don't delete source & build checkouts after testing |
| testListPath string // relative path to the test list .json file |
| |
| writtenTestLists []string // paths to test updated lists that have been written |
| } |
| |
| // lazyRun lazily runs the test t. |
| // If the test results are not already cached, then test will setup the test |
| // environment, and call t.run(). |
| // The results of the test will be cached into r.cacheRoot. |
| func (t *test) lazyRun() (*CommitTestResults, error) { |
| load := func(data []byte) (interface{}, error) { |
| var res CommitTestResults |
| if err := json.NewDecoder(bytes.NewReader(data)).Decode(&res); err != nil { |
| return nil, err |
| } |
| if res.Version != dataVersion { |
| return nil, errors.New("Data is from an old version") |
| } |
| return &res, nil |
| } |
| |
| build := func() ([]byte, interface{}, error) { |
| res, err := t.run() |
| |
| if err != nil { |
| return nil, nil, err |
| } |
| |
| b := bytes.Buffer{} |
| enc := json.NewEncoder(&b) |
| enc.SetIndent("", " ") |
| if err := enc.Encode(res); err != nil { |
| return nil, nil, err |
| } |
| |
| return b.Bytes(), res, nil |
| } |
| |
| res, err := loadOrBuild(filepath.Join(t.resDir, "results.json"), load, build) |
| if err != nil { |
| return nil, err |
| } |
| |
| return res.(*CommitTestResults), nil |
| } |
| |
| // run executes the tests for the test environment t. |
| // If the source is not cached, run will fetch the commit to be tested, |
| // before building it, and then run the required tests. |
| func (t *test) run() (*CommitTestResults, error) { |
| if isDir(t.srcDir) && t.keepCheckouts { |
| log.Printf("Reusing source cache for commit '%s'\n", t.commit) |
| } else { |
| log.Printf("Checking out '%s'\n", t.commit) |
| os.RemoveAll(t.srcDir) |
| if err := git.Checkout(t.srcDir, gitURL, t.commit); err != nil { |
| return nil, cause.Wrap(err, "Checking out commit '%s'", t.commit) |
| } |
| log.Printf("Checked out commit '%s'\n", t.commit) |
| if !t.keepCheckouts { |
| defer os.RemoveAll(t.srcDir) |
| } |
| } |
| |
| if err := t.build(); err != nil { |
| log.Printf("Warning: Commit '%s' failed to build. %v", t.commit, err) |
| return &CommitTestResults{Version: dataVersion, Built: false}, nil |
| } |
| log.Printf("Built '%s'\n", t.commit) |
| |
| // Load the list of tests that need executing. |
| // Note: this list may vary by each commit. |
| testLists, err := t.loadTestList() |
| if err != nil { |
| return nil, cause.Wrap(err, "Loading test lists") |
| } |
| |
| results, err := t.runTests(testLists) |
| if err != nil { |
| return nil, cause.Wrap(err, "Running tests") |
| } |
| log.Printf("Ran tests for '%s'\n", t.commit) |
| |
| if t.keepCheckouts { |
| if err := t.writeTestListsByStatus(testLists, results); err != nil { |
| return nil, cause.Wrap(err, "Writing test lists by status") |
| } |
| } |
| |
| return results, nil |
| } |
| |
| // loadOrBuild is a helper for building a lazy resolved cache. |
| // loadOrBuild attempts to load the file at path. If the file exists and loaded |
| // successfully, then load() is called with the file data, and the the result |
| // object from load() is returned. |
| // If the file does not exist, the file cannot be loaded, or load() returns an |
| // error, then build() is called and the byte slice is saved to path, and the |
| // object is returned. |
| func loadOrBuild(path string, |
| load func([]byte) (interface{}, error), |
| build func() ([]byte, interface{}, error)) (interface{}, error) { |
| |
| if data, err := ioutil.ReadFile(path); err == nil { |
| out, err := load(data) |
| if err == nil { |
| return out, nil |
| } |
| log.Printf("Warning: Failed to load '%s': %v", path, err) |
| os.Remove(path) // Delete and rebuild. |
| } |
| |
| data, obj, err := build() |
| if err != nil { |
| return nil, err |
| } |
| |
| os.MkdirAll(filepath.Dir(path), 0777) |
| |
| if err := ioutil.WriteFile(path, data, 0777); err != nil { |
| log.Printf("Warning: Failed to write to '%s': %v", path, err) |
| } |
| return obj, nil |
| } |
| |
| // build builds the SwiftShader source into t.buildDir. |
| func (t *test) build() error { |
| log.Printf("Building '%s'\n", t.commit) |
| |
| if err := os.MkdirAll(t.buildDir, 0777); err != nil { |
| return cause.Wrap(err, "Failed to create build directory") |
| } |
| |
| if err := shell.Shell(buildTimeout, t.r.cmake, t.buildDir, |
| "-DCMAKE_BUILD_TYPE=Release", |
| "-DDCHECK_ALWAYS_ON=1", |
| ".."); err != nil { |
| return err |
| } |
| |
| if err := shell.Shell(buildTimeout, t.r.make, t.buildDir, fmt.Sprintf("-j%d", runtime.NumCPU())); err != nil { |
| return err |
| } |
| |
| return nil |
| } |
| |
| // runTests runs all the tests. |
| func (t *test) runTests(testLists []TestList) (*CommitTestResults, error) { |
| log.Printf("Running tests for '%s'\n", t.commit) |
| start := time.Now() |
| |
| // Wait group that completes once all the tests have finished. |
| wg := sync.WaitGroup{} |
| results := make(chan TestResult, 256) |
| |
| numTests := 0 |
| |
| // For each API that we are testing |
| for _, list := range testLists { |
| // Resolve the test runner |
| var exe string |
| switch list.API { |
| case egl: |
| exe = filepath.Join(t.r.deqpBuild, "modules", "egl", "deqp-egl") |
| case gles2: |
| exe = filepath.Join(t.r.deqpBuild, "modules", "gles2", "deqp-gles2") |
| case gles3: |
| exe = filepath.Join(t.r.deqpBuild, "modules", "gles3", "deqp-gles3") |
| case vulkan: |
| exe = filepath.Join(t.r.deqpBuild, "external", "vulkancts", "modules", "vulkan", "deqp-vk") |
| default: |
| return nil, fmt.Errorf("Unknown API '%v'", list.API) |
| } |
| if !isFile(exe) { |
| return nil, fmt.Errorf("Couldn't find dEQP executable at '%s'", exe) |
| } |
| |
| // Build a chan for the test names to be run. |
| tests := make(chan string, len(list.Tests)) |
| |
| // Start a number of go routines to run the tests. |
| wg.Add(numParallelTests) |
| for i := 0; i < numParallelTests; i++ { |
| go func() { |
| t.deqpTestRoutine(exe, tests, results) |
| wg.Done() |
| }() |
| } |
| |
| // Hand the tests to the deqpTestRoutines. |
| for _, t := range list.Tests { |
| tests <- t |
| } |
| |
| // Close the tests chan to indicate that there are no more tests to run. |
| // The deqpTestRoutine functions will return once all tests have been |
| // run. |
| close(tests) |
| |
| numTests += len(list.Tests) |
| } |
| |
| out := CommitTestResults{ |
| Version: dataVersion, |
| Built: true, |
| Tests: map[string]TestResult{}, |
| } |
| |
| // Collect the results. |
| finished := make(chan struct{}) |
| lastUpdate := time.Now() |
| go func() { |
| start, i := time.Now(), 0 |
| for r := range results { |
| i++ |
| out.Tests[r.Test] = r |
| if time.Since(lastUpdate) > time.Minute { |
| lastUpdate = time.Now() |
| remaining := numTests - i |
| log.Printf("Ran %d/%d tests (%v%%). Estimated completion in %v.\n", |
| i, numTests, percent(i, numTests), |
| (time.Since(start)/time.Duration(i))*time.Duration(remaining)) |
| } |
| } |
| close(finished) |
| }() |
| |
| wg.Wait() // Block until all the deqpTestRoutines have finished. |
| close(results) // Signal no more results. |
| <-finished // And wait for the result collecting go-routine to finish. |
| |
| out.Duration = time.Since(start) |
| |
| return &out, nil |
| } |
| |
| func (t *test) writeTestListsByStatus(testLists []TestList, results *CommitTestResults) error { |
| for _, list := range testLists { |
| files := map[Status]*os.File{} |
| ext := filepath.Ext(list.File) |
| name := list.File[:len(list.File)-len(ext)] |
| for _, status := range Statuses { |
| path := filepath.Join(t.srcDir, name+"-"+string(status)+ext) |
| dir := filepath.Dir(path) |
| os.MkdirAll(dir, 0777) |
| f, err := os.Create(path) |
| if err != nil { |
| return cause.Wrap(err, "Couldn't create file '%v'", path) |
| } |
| defer f.Close() |
| files[status] = f |
| |
| t.writtenTestLists = append(t.writtenTestLists, path) |
| } |
| |
| for _, testName := range list.Tests { |
| if r, found := results.Tests[testName]; found { |
| fmt.Fprintln(files[r.Status], testName) |
| } |
| } |
| } |
| |
| return nil |
| } |
| |
| // Status is an enumerator of test results. |
| type Status string |
| |
| const ( |
| // Pass is the status of a successful test. |
| Pass = Status("PASS") |
| // Fail is the status of a failed test. |
| Fail = Status("FAIL") |
| // Timeout is the status of a test that failed to complete in the alloted |
| // time. |
| Timeout = Status("TIMEOUT") |
| // Crash is the status of a test that crashed. |
| Crash = Status("CRASH") |
| // NotSupported is the status of a test feature not supported by the driver. |
| NotSupported = Status("NOT_SUPPORTED") |
| // CompatibilityWarning is the status passing test with a warning. |
| CompatibilityWarning = Status("COMPATIBILITY_WARNING") |
| // QualityWarning is the status passing test with a warning. |
| QualityWarning = Status("QUALITY_WARNING") |
| ) |
| |
| // Statuses is the full list of status types |
| var Statuses = []Status{Pass, Fail, Timeout, Crash, NotSupported, CompatibilityWarning, QualityWarning} |
| |
| // Failing returns true if the task status requires fixing. |
| func (s Status) Failing() bool { |
| switch s { |
| case Fail, Timeout, Crash: |
| return true |
| default: |
| return false |
| } |
| } |
| |
| // CommitTestResults holds the results the tests across all APIs for a given |
| // commit. The CommitTestResults structure may be serialized to cache the |
| // results. |
| type CommitTestResults struct { |
| Version int |
| Built bool |
| Tests map[string]TestResult |
| Duration time.Duration |
| } |
| |
| // compare returns a string describing all differences between two |
| // CommitTestResults. This string is used as the report message posted to the |
| // gerrit code review. |
| func compare(old, new *CommitTestResults) string { |
| switch { |
| case !old.Built && !new.Built: |
| return "Build continues to be broken." |
| case old.Built && !new.Built: |
| return "Build broken." |
| case !old.Built && !new.Built: |
| return "Build now fixed. Cannot compare against broken parent." |
| } |
| |
| oldStatusCounts, newStatusCounts := map[Status]int{}, map[Status]int{} |
| totalTests := 0 |
| |
| broken, fixed, failing, removed, changed := []string{}, []string{}, []string{}, []string{}, []string{} |
| |
| for test, new := range new.Tests { |
| old, found := old.Tests[test] |
| switch { |
| case (!found || old.Status == Pass) && new.Status.Failing(): |
| broken = append(broken, test) |
| case (found && old.Status.Failing()) && new.Status == Pass: |
| fixed = append(fixed, test) |
| case found && old.Status != new.Status: |
| changed = append(changed, test) |
| case found && old.Status.Failing() && new.Status.Failing(): |
| failing = append(failing, test) // Still broken |
| } |
| totalTests++ |
| if found { |
| oldStatusCounts[old.Status] = oldStatusCounts[old.Status] + 1 |
| } |
| newStatusCounts[new.Status] = newStatusCounts[new.Status] + 1 |
| } |
| |
| for test := range old.Tests { |
| if _, found := new.Tests[test]; !found { |
| removed = append(removed, test) |
| } |
| } |
| |
| sb := strings.Builder{} |
| |
| // list prints the list l to sb, truncating after a limit. |
| list := func(l []string) { |
| const max = 10 |
| for i, s := range l { |
| sb.WriteString(" ") |
| if i == max { |
| sb.WriteString(fmt.Sprintf("> %d more\n", len(l)-i)) |
| break |
| } |
| sb.WriteString(fmt.Sprintf("> %s", s)) |
| if n, ok := new.Tests[s]; ok { |
| if o, ok := old.Tests[s]; ok && n != o { |
| sb.WriteString(fmt.Sprintf(" - [%s -> %s]", o.Status, n.Status)) |
| } else { |
| sb.WriteString(fmt.Sprintf(" - [%s]", n.Status)) |
| } |
| } |
| sb.WriteString("\n") |
| } |
| } |
| |
| sb.WriteString(fmt.Sprintf(" Total tests: %d\n", totalTests)) |
| for _, s := range []struct { |
| label string |
| status Status |
| }{ |
| {" Pass", Pass}, |
| {" Fail", Fail}, |
| {" Timeout", Timeout}, |
| {" Crash", Crash}, |
| {" Not Supported", NotSupported}, |
| {"Compatibility Warning", CompatibilityWarning}, |
| {" Quality Warning", QualityWarning}, |
| } { |
| old, new := oldStatusCounts[s.status], newStatusCounts[s.status] |
| if old == 0 && new == 0 { |
| continue |
| } |
| change := percent64(int64(new-old), int64(old)) |
| switch { |
| case old == new: |
| sb.WriteString(fmt.Sprintf("%s: %v\n", s.label, new)) |
| case change == 0: |
| sb.WriteString(fmt.Sprintf("%s: %v -> %v\n", s.label, old, new)) |
| default: |
| sb.WriteString(fmt.Sprintf("%s: %v -> %v (%+d%%)\n", s.label, old, new, change)) |
| } |
| } |
| |
| if old, new := old.Duration, new.Duration; old != 0 && new != 0 { |
| label := " Time taken" |
| change := percent64(int64(new-old), int64(old)) |
| switch { |
| case old == new: |
| sb.WriteString(fmt.Sprintf("%s: %v\n", label, new)) |
| case change == 0: |
| sb.WriteString(fmt.Sprintf("%s: %v -> %v\n", label, old, new)) |
| default: |
| sb.WriteString(fmt.Sprintf("%s: %v -> %v (%+d%%)\n", label, old, new, change)) |
| } |
| } |
| |
| if n := len(broken); n > 0 { |
| sort.Strings(broken) |
| sb.WriteString(fmt.Sprintf("\n--- This change breaks %d tests: ---\n", n)) |
| list(broken) |
| } |
| if n := len(fixed); n > 0 { |
| sort.Strings(fixed) |
| sb.WriteString(fmt.Sprintf("\n--- This change fixes %d tests: ---\n", n)) |
| list(fixed) |
| } |
| if n := len(removed); n > 0 { |
| sort.Strings(removed) |
| sb.WriteString(fmt.Sprintf("\n--- This change removes %d tests: ---\n", n)) |
| list(removed) |
| } |
| if n := len(changed); n > 0 { |
| sort.Strings(changed) |
| sb.WriteString(fmt.Sprintf("\n--- This change alters %d tests: ---\n", n)) |
| list(changed) |
| } |
| |
| if len(broken) == 0 && len(fixed) == 0 && len(removed) == 0 && len(changed) == 0 { |
| sb.WriteString(fmt.Sprintf("\n--- No change in test results ---\n")) |
| } |
| |
| return sb.String() |
| } |
| |
| // TestResult holds the results of a single API test. |
| type TestResult struct { |
| Test string |
| Status Status |
| Err string `json:",omitempty"` |
| } |
| |
| func (r TestResult) String() string { |
| if r.Err != "" { |
| return fmt.Sprintf("%s: %s (%s)", r.Test, r.Status, r.Err) |
| } |
| return fmt.Sprintf("%s: %s", r.Test, r.Status) |
| } |
| |
| // Regular expression to parse the output of a dEQP test. |
| var parseRE = regexp.MustCompile(`(Fail|Pass|NotSupported|CompatibilityWarning|QualityWarning) \(([\s\S]*)\)`) |
| |
| // deqpTestRoutine repeatedly runs the dEQP test executable exe with the tests |
| // taken from tests. The output of the dEQP test is parsed, and the test result |
| // is written to results. |
| // deqpTestRoutine only returns once the tests chan has been closed. |
| // deqpTestRoutine does not close the results chan. |
| func (t *test) deqpTestRoutine(exe string, tests <-chan string, results chan<- TestResult) { |
| for name := range tests { |
| // log.Printf("Running test '%s'\n", name) |
| env := []string{ |
| "LD_LIBRARY_PATH=" + t.buildDir + ":" + os.Getenv("LD_LIBRARY_PATH"), |
| "VK_ICD_FILENAMES=" + filepath.Join(t.outDir, "Linux", "vk_swiftshader_icd.json"), |
| "DISPLAY=" + os.Getenv("DISPLAY"), |
| "LIBC_FATAL_STDERR_=1", // Put libc explosions into logs. |
| } |
| |
| out, err := shell.Exec(testTimeout, exe, filepath.Dir(exe), env, "--deqp-surface-type=pbuffer", "-n="+name) |
| switch err.(type) { |
| default: |
| results <- TestResult{ |
| Test: name, |
| Status: Crash, |
| Err: cause.Wrap(err, string(out)).Error(), |
| } |
| case shell.ErrTimeout: |
| results <- TestResult{ |
| Test: name, |
| Status: Timeout, |
| Err: cause.Wrap(err, string(out)).Error(), |
| } |
| case nil: |
| toks := parseRE.FindStringSubmatch(string(out)) |
| if len(toks) < 3 { |
| err := fmt.Sprintf("Couldn't parse test '%v' output:\n%s", name, string(out)) |
| log.Println("Warning: ", err) |
| results <- TestResult{Test: name, Status: Fail, Err: err} |
| continue |
| } |
| switch toks[1] { |
| case "Pass": |
| results <- TestResult{Test: name, Status: Pass} |
| case "NotSupported": |
| results <- TestResult{Test: name, Status: NotSupported} |
| case "CompatibilityWarning": |
| results <- TestResult{Test: name, Status: CompatibilityWarning} |
| case "QualityWarning": |
| results <- TestResult{Test: name, Status: CompatibilityWarning} |
| case "Fail": |
| var err string |
| if toks[2] != "Fail" { |
| err = toks[2] |
| } |
| results <- TestResult{Test: name, Status: Fail, Err: err} |
| default: |
| err := fmt.Sprintf("Couldn't parse test output:\n%s", string(out)) |
| log.Println("Warning: ", err) |
| results <- TestResult{Test: name, Status: Fail, Err: err} |
| } |
| } |
| } |
| } |
| |
| // API is an enumerator of graphics APIs. |
| type API string |
| |
| const ( |
| egl = API("egl") |
| gles2 = API("gles2") |
| gles3 = API("gles3") |
| vulkan = API("vulkan") |
| ) |
| |
| // TestList is a list of tests to be run for a given API. |
| type TestList struct { |
| Name string |
| File string |
| API API |
| Tests []string |
| } |
| |
| // loadTestList loads the test list json file. |
| // The file is first searched at {Commit}/{t.testListPath} |
| // If this cannot be found, then the file is searched at the fallback path |
| // {CWD}/{t.testListPath} |
| // This allows CLs to alter the list of tests to be run, as well as providing |
| // a default set. |
| func (t *test) loadTestList() ([]TestList, error) { |
| // find the test.json file in {SwiftShader}/tests/regres |
| root := t.srcDir |
| if isFile(filepath.Join(root, t.testListPath)) { |
| log.Println("Using test list from commit") |
| } else { |
| // Not found there. Search locally. |
| root, _ = os.Getwd() |
| if isFile(filepath.Join(root, t.testListPath)) { |
| log.Println("Using test list from regres") |
| } else { |
| return nil, fmt.Errorf("Could not find test list file '%v'", t.testListPath) |
| } |
| } |
| |
| jsonPath := filepath.Join(root, t.testListPath) |
| i, err := ioutil.ReadFile(jsonPath) |
| if err != nil { |
| return nil, cause.Wrap(err, "Couldn't read test list from '%s'", jsonPath) |
| } |
| |
| var groups []struct { |
| Name string |
| API string |
| TestFile string `json:"tests"` |
| } |
| if err := json.NewDecoder(bytes.NewReader(i)).Decode(&groups); err != nil { |
| return nil, cause.Wrap(err, "Couldn't parse '%s'", jsonPath) |
| } |
| |
| dir := filepath.Dir(jsonPath) |
| |
| out := make([]TestList, len(groups)) |
| for i, group := range groups { |
| path := filepath.Join(dir, group.TestFile) |
| tests, err := ioutil.ReadFile(path) |
| if err != nil { |
| return nil, cause.Wrap(err, "Couldn't read '%s'", tests) |
| } |
| relPath, err := filepath.Rel(root, path) |
| if err != nil { |
| return nil, cause.Wrap(err, "Couldn't get relative path for '%s'", path) |
| } |
| list := TestList{ |
| Name: group.Name, |
| File: relPath, |
| API: API(group.API), |
| } |
| for _, line := range strings.Split(string(tests), "\n") { |
| line = strings.TrimSpace(line) |
| if line != "" && !strings.HasPrefix(line, "#") { |
| list.Tests = append(list.Tests, line) |
| } |
| } |
| sort.Strings(list.Tests) |
| out[i] = list |
| } |
| |
| return out, nil |
| } |
| |
| // isDir returns true if path is a file. |
| func isFile(path string) bool { |
| s, err := os.Stat(path) |
| if err != nil { |
| return false |
| } |
| return !s.IsDir() |
| } |
| |
| // isDir returns true if path is a directory. |
| func isDir(path string) bool { |
| s, err := os.Stat(path) |
| if err != nil { |
| return false |
| } |
| return s.IsDir() |
| } |
| |
| // percent returns the percentage completion of i items out of n. |
| func percent(i, n int) int { |
| return int(percent64(int64(i), int64(n))) |
| } |
| |
| // percent64 returns the percentage completion of i items out of n. |
| func percent64(i, n int64) int64 { |
| if n == 0 { |
| return 0 |
| } |
| return (100 * i) / n |
| } |
| |
| type date struct { |
| year int |
| month time.Month |
| day int |
| } |
| |
| func toDate(t time.Time) date { |
| d := date{} |
| d.year, d.month, d.day = t.Date() |
| return d |
| } |