Add test/regres/export_to_sheets.go
Exports the latest test data to a Google Sheets document.
Change-Id: Ia1b38464daf7117da571d536e7ff029023b9de58
Reviewed-on: https://swiftshader-review.googlesource.com/c/SwiftShader/+/26748
Reviewed-by: Nicolas Capens <nicolascapens@google.com>
Tested-by: Ben Clayton <bclayton@google.com>
diff --git a/tests/regres/consts/consts.go b/tests/regres/consts/consts.go
new file mode 100644
index 0000000..ba76410
--- /dev/null
+++ b/tests/regres/consts/consts.go
@@ -0,0 +1,22 @@
+// Copyright 2019 The SwiftShader Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package consts holds constants shared between the regres tools.
+package consts
+
+const (
+ // TestListUpdateCommitSubjectPrefix is the commit message prefix on commits
+ // that update the full test lists results.
+ TestListUpdateCommitSubjectPrefix = "Regres: Update test lists @ "
+)
diff --git a/tests/regres/export_to_sheets.go b/tests/regres/export_to_sheets.go
new file mode 100644
index 0000000..70f9f4e
--- /dev/null
+++ b/tests/regres/export_to_sheets.go
@@ -0,0 +1,436 @@
+// Copyright 2019 The SwiftShader Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// export_to_sheets updates a Google sheets document with the latest test
+// results
+package main
+
+import (
+ "bufio"
+ "bytes"
+ "context"
+ "encoding/json"
+ "flag"
+ "fmt"
+ "io/ioutil"
+ "log"
+ "net/http"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "./cause"
+ "./consts"
+ "./git"
+ "./testlist"
+
+ "golang.org/x/oauth2"
+ "golang.org/x/oauth2/google"
+ "google.golang.org/api/sheets/v4"
+)
+
+var (
+ authdir = flag.String("authdir", "~/.regres-auth", "directory to hold credentials.json and generated token")
+ projectPath = flag.String("projpath", ".", "project path")
+ testListPath = flag.String("testlist", "tests/regres/full-tests.json", "project relative path to the test list .json file")
+ spreadsheetID = flag.String("spreadsheet", "1RCxbqtKNDG9rVMe_xHMapMBgzOCp24mumab73SbHtfw", "identifier of the spreadsheet to update")
+)
+
+const (
+ columnGitHash = "GIT_HASH"
+ columnGitDate = "GIT_DATE"
+)
+
+func main() {
+ flag.Parse()
+
+ if err := run(); err != nil {
+ log.Fatalln(err)
+ }
+}
+
+func run() error {
+ // Load the full test list. We use this to find the test file names.
+ lists, err := testlist.Load(".", *testListPath)
+ if err != nil {
+ return cause.Wrap(err, "Unable to load test list")
+ }
+
+ // Load the creditials used for editing the Google Sheets spreadsheet.
+ srv, err := createSheetsService(*authdir)
+ if err != nil {
+ return cause.Wrap(err, "Unable to authenticate")
+ }
+
+ // Ensure that there is a sheet for each of the test lists.
+ if err := createTestListSheets(srv, lists); err != nil {
+ return cause.Wrap(err, "Unable to create sheets")
+ }
+
+ spreadsheet, err := srv.Spreadsheets.Get(*spreadsheetID).Do()
+ if err != nil {
+ return cause.Wrap(err, "Unable to get spreadsheet")
+ }
+
+ req := sheets.BatchUpdateValuesRequest{
+ ValueInputOption: "RAW",
+ }
+
+ testListDir := filepath.Dir(filepath.Join(*projectPath, *testListPath))
+ changes, err := git.Log(testListDir, 100)
+ if err != nil {
+ return cause.Wrap(err, "Couldn't get git changes for '%v'", testListDir)
+ }
+
+ for _, group := range lists {
+ sheetName := group.Name
+ fmt.Println("Processing sheet", sheetName)
+ sheet := getSheet(spreadsheet, sheetName)
+ if sheet == nil {
+ return cause.Wrap(err, "Sheet '%v' not found", sheetName)
+ }
+
+ columnHeaders, err := fetchRow(srv, spreadsheet, sheet, 0)
+ if err != nil {
+ return cause.Wrap(err, "Couldn't get sheet '%v' column headers", sheetName)
+ }
+
+ columnIndices := listToMap(columnHeaders)
+
+ hashColumnIndex, found := columnIndices[columnGitHash]
+ if !found {
+ return cause.Wrap(err, "Couldn't find sheet '%v' column header '%v'", sheetName, columnGitHash)
+ }
+
+ hashValues, err := fetchColumn(srv, spreadsheet, sheet, hashColumnIndex)
+ if err != nil {
+ return cause.Wrap(err, "Couldn't get sheet '%v' column headers", sheetName)
+ }
+ hashValues = hashValues[1:] // Skip header
+
+ hashIndices := listToMap(hashValues)
+ rowValues := map[string]interface{}{}
+
+ rowInsertionPoint := 1 + len(hashValues)
+
+ for i := len(changes) - 1; i > 0; i-- {
+ change := changes[i]
+ if !strings.HasPrefix(change.Subject, consts.TestListUpdateCommitSubjectPrefix) {
+ continue
+ }
+
+ hash := change.Hash.String()
+ if _, found := hashIndices[hash]; found {
+ continue // Already in the sheet
+ }
+
+ rowValues[columnGitHash] = change.Hash.String()
+ rowValues[columnGitDate] = change.Date.Format("2006-01-02")
+
+ path := filepath.Join(*projectPath, group.File)
+ hasData := false
+ for _, status := range testlist.Statuses {
+ path := testlist.FilePathWithStatus(path, status)
+ data, err := git.Show(path, hash)
+ if err != nil {
+ continue
+ }
+ lines, err := countLines(data)
+ if err != nil {
+ return cause.Wrap(err, "Couldn't count lines in file '%s'", path)
+ }
+
+ rowValues[string(status)] = lines
+ hasData = true
+ }
+
+ if !hasData {
+ continue
+ }
+
+ data, err := mapToList(columnIndices, rowValues)
+ if err != nil {
+ return cause.Wrap(err, "Couldn't map row values to column for sheet %v. Column headers: [%+v]", sheetName, columnHeaders)
+ }
+
+ req.Data = append(req.Data, &sheets.ValueRange{
+ Range: rowRange(rowInsertionPoint, sheet),
+ Values: [][]interface{}{data},
+ })
+ rowInsertionPoint++
+
+ fmt.Printf("Adding test data at %v to %v\n", hash[:8], sheetName)
+ }
+ }
+
+ if _, err := srv.Spreadsheets.Values.BatchUpdate(*spreadsheetID, &req).Do(); err != nil {
+ return cause.Wrap(err, "Values BatchUpdate failed")
+ }
+
+ return nil
+}
+
+// listToMap returns the list l as a map where the key is the stringification
+// of the element, and the value is the element index.
+func listToMap(l []interface{}) map[string]int {
+ out := map[string]int{}
+ for i, v := range l {
+ out[fmt.Sprint(v)] = i
+ }
+ return out
+}
+
+// mapToList transforms the two maps into a single slice of values.
+// indices is a map of identifier to output slice element index.
+// values is a map of identifier to value.
+func mapToList(indices map[string]int, values map[string]interface{}) ([]interface{}, error) {
+ out := []interface{}{}
+ for name, value := range values {
+ index, ok := indices[name]
+ if !ok {
+ return nil, fmt.Errorf("No index for '%v'", name)
+ }
+ for len(out) <= index {
+ out = append(out, nil)
+ }
+ out[index] = value
+ }
+ return out, nil
+}
+
+// countLines returns the number of new lines in the byte slice data.
+func countLines(data []byte) (int, error) {
+ scanner := bufio.NewScanner(bytes.NewReader(data))
+ lines := 0
+ for scanner.Scan() {
+ lines++
+ }
+ return lines, nil
+}
+
+// getSheet returns the sheet with the given title name, or nil if the sheet
+// cannot be found.
+func getSheet(spreadsheet *sheets.Spreadsheet, name string) *sheets.Sheet {
+ for _, sheet := range spreadsheet.Sheets {
+ if sheet.Properties.Title == name {
+ return sheet
+ }
+ }
+ return nil
+}
+
+// rowRange returns a sheets range ("name!Ai:i") for the entire row with the
+// given index.
+func rowRange(index int, sheet *sheets.Sheet) string {
+ return fmt.Sprintf("%v!A%v:%v", sheet.Properties.Title, index+1, index+1)
+}
+
+// columnRange returns a sheets range ("name!i1:i") for the entire column with
+// the given index.
+func columnRange(index int, sheet *sheets.Sheet) string {
+ col := 'A' + index
+ if index > 25 {
+ panic("UNIMPLEMENTED")
+ }
+ return fmt.Sprintf("%v!%c1:%c", sheet.Properties.Title, col, col)
+}
+
+// fetchRow returns all the values in the given sheet's row.
+func fetchRow(srv *sheets.Service, spreadsheet *sheets.Spreadsheet, sheet *sheets.Sheet, row int) ([]interface{}, error) {
+ rng := rowRange(row, sheet)
+ data, err := srv.Spreadsheets.Values.Get(spreadsheet.SpreadsheetId, rng).Do()
+ if err != nil {
+ return nil, cause.Wrap(err, "Couldn't fetch %v", rng)
+ }
+ return data.Values[0], nil
+}
+
+// fetchColumn returns all the values in the given sheet's column.
+func fetchColumn(srv *sheets.Service, spreadsheet *sheets.Spreadsheet, sheet *sheets.Sheet, row int) ([]interface{}, error) {
+ rng := columnRange(row, sheet)
+ data, err := srv.Spreadsheets.Values.Get(spreadsheet.SpreadsheetId, rng).Do()
+ if err != nil {
+ return nil, cause.Wrap(err, "Couldn't fetch %v", rng)
+ }
+ out := make([]interface{}, len(data.Values))
+ for i, l := range data.Values {
+ if len(l) > 0 {
+ out[i] = l[0]
+ }
+ }
+ return out, nil
+}
+
+// insertRows inserts blank rows into the given sheet.
+func insertRows(srv *sheets.Service, spreadsheet *sheets.Spreadsheet, sheet *sheets.Sheet, aboveRow, count int) error {
+ req := sheets.BatchUpdateSpreadsheetRequest{
+ Requests: []*sheets.Request{{
+ InsertRange: &sheets.InsertRangeRequest{
+ Range: &sheets.GridRange{
+ SheetId: sheet.Properties.SheetId,
+ StartRowIndex: int64(aboveRow),
+ EndRowIndex: int64(aboveRow + count),
+ },
+ ShiftDimension: "ROWS",
+ }},
+ },
+ }
+ if _, err := srv.Spreadsheets.BatchUpdate(*spreadsheetID, &req).Do(); err != nil {
+ return cause.Wrap(err, "Values BatchUpdate failed")
+ }
+ return nil
+}
+
+// createTestListSheets adds a new sheet for each of the test lists, if they
+// do not already exist. These new sheets are populated with column headers.
+func createTestListSheets(srv *sheets.Service, testlists testlist.Lists) error {
+ spreadsheet, err := srv.Spreadsheets.Get(*spreadsheetID).Do()
+ if err != nil {
+ return cause.Wrap(err, "Unable to get spreadsheet")
+ }
+
+ spreadsheetReq := sheets.BatchUpdateSpreadsheetRequest{}
+ updateReq := sheets.BatchUpdateValuesRequest{ValueInputOption: "RAW"}
+ headers := []interface{}{columnGitHash, columnGitDate}
+ for _, s := range testlist.Statuses {
+ headers = append(headers, string(s))
+ }
+
+ for _, group := range testlists {
+ name := group.Name
+ if getSheet(spreadsheet, name) == nil {
+ spreadsheetReq.Requests = append(spreadsheetReq.Requests, &sheets.Request{
+ AddSheet: &sheets.AddSheetRequest{
+ Properties: &sheets.SheetProperties{
+ Title: name,
+ },
+ },
+ })
+ updateReq.Data = append(updateReq.Data,
+ &sheets.ValueRange{
+ Range: name + "!A1:Z",
+ Values: [][]interface{}{headers},
+ },
+ )
+ }
+ }
+
+ if len(spreadsheetReq.Requests) > 0 {
+ if _, err := srv.Spreadsheets.BatchUpdate(*spreadsheetID, &spreadsheetReq).Do(); err != nil {
+ return cause.Wrap(err, "Spreadsheets BatchUpdate failed")
+ }
+ }
+ if len(updateReq.Data) > 0 {
+ if _, err := srv.Spreadsheets.Values.BatchUpdate(*spreadsheetID, &updateReq).Do(); err != nil {
+ return cause.Wrap(err, "Values BatchUpdate failed")
+ }
+ }
+
+ return nil
+}
+
+// createSheetsService creates a new Google Sheets service using the credentials
+// in the credentials.json file.
+func createSheetsService(authdir string) (*sheets.Service, error) {
+ authdir = os.ExpandEnv(authdir)
+ if home, err := os.UserHomeDir(); err == nil {
+ authdir = strings.ReplaceAll(authdir, "~", home)
+ }
+
+ os.MkdirAll(authdir, 0777)
+
+ credentialsPath := filepath.Join(authdir, "credentials.json")
+ b, err := ioutil.ReadFile(credentialsPath)
+ if err != nil {
+ return nil, cause.Wrap(err, "Unable to read client secret file '%v'\n"+
+ "Obtain this file from: https://console.developers.google.com/apis/credentials", credentialsPath)
+ }
+
+ config, err := google.ConfigFromJSON(b, "https://www.googleapis.com/auth/spreadsheets")
+ if err != nil {
+ return nil, cause.Wrap(err, "Unable to parse client secret file to config")
+ }
+
+ client, err := getClient(authdir, config)
+ if err != nil {
+ return nil, cause.Wrap(err, "Unable obtain client")
+ }
+
+ srv, err := sheets.New(client)
+ if err != nil {
+ return nil, cause.Wrap(err, "Unable to retrieve Sheets client")
+ }
+ return srv, nil
+}
+
+// Retrieve a token, saves the token, then returns the generated client.
+func getClient(authdir string, config *oauth2.Config) (*http.Client, error) {
+ // The file token.json stores the user's access and refresh tokens, and is
+ // created automatically when the authorization flow completes for the first
+ // time.
+ tokFile := filepath.Join(authdir, "token.json")
+ tok, err := tokenFromFile(tokFile)
+ if err != nil {
+ tok, err = getTokenFromWeb(config)
+ if err != nil {
+ return nil, cause.Wrap(err, "Unable to get token from web")
+ }
+ if err := saveToken(tokFile, tok); err != nil {
+ log.Println("Warning: failed to write token: %v", err)
+ }
+ }
+ return config.Client(context.Background(), tok), nil
+}
+
+// Request a token from the web, then returns the retrieved token.
+func getTokenFromWeb(config *oauth2.Config) (*oauth2.Token, error) {
+ authURL := config.AuthCodeURL("state-token", oauth2.AccessTypeOffline)
+ fmt.Printf("Go to the following link in your browser then type the "+
+ "authorization code: \n%v\n", authURL)
+
+ var authCode string
+ if _, err := fmt.Scan(&authCode); err != nil {
+ return nil, cause.Wrap(err, "Unable to read authorization code")
+ }
+
+ tok, err := config.Exchange(context.TODO(), authCode)
+ if err != nil {
+ return nil, cause.Wrap(err, "Unable to retrieve token from web")
+ }
+ return tok, nil
+}
+
+// Retrieves a token from a local file.
+func tokenFromFile(path string) (*oauth2.Token, error) {
+ f, err := os.Open(path)
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+ tok := &oauth2.Token{}
+ err = json.NewDecoder(f).Decode(tok)
+ return tok, err
+}
+
+// Saves a token to a file path.
+func saveToken(path string, token *oauth2.Token) error {
+ fmt.Printf("Saving credential file to: %s\n", path)
+ f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
+ if err != nil {
+ return cause.Wrap(err, "Unable to cache oauth token")
+ }
+ defer f.Close()
+ json.NewEncoder(f).Encode(token)
+ return nil
+}
diff --git a/tests/regres/git/git.go b/tests/regres/git/git.go
index 9842b5c..787fa7c 100644
--- a/tests/regres/git/git.go
+++ b/tests/regres/git/git.go
@@ -22,6 +22,7 @@
"net/url"
"os"
"os/exec"
+ "strings"
"time"
"../cause"
@@ -139,3 +140,83 @@
}
return ParseHash(string(out)), nil
}
+
+type ChangeList struct {
+ Hash Hash
+ Date time.Time
+ Author string
+ Subject string
+ Description string
+}
+
+// Log returns the top count ChangeLists at HEAD.
+func Log(path string, count int) ([]ChangeList, error) {
+ return LogFrom(path, "HEAD", count)
+}
+
+// LogFrom returns the top count ChangeList starting from at.
+func LogFrom(path, at string, count int) ([]ChangeList, error) {
+ if at == "" {
+ at = "HEAD"
+ }
+ out, err := shell.Exec(gitTimeout, exe, "", nil, "log", at, "--pretty=format:"+prettyFormat, fmt.Sprintf("-%d", count), path)
+ if err != nil {
+ return nil, err
+ }
+ return parseLog(string(out)), nil
+}
+
+// Parent returns the parent ChangeList for cl.
+func Parent(cl ChangeList) (ChangeList, error) {
+ out, err := shell.Exec(gitTimeout, exe, "", nil, "log", "--pretty=format:"+prettyFormat, fmt.Sprintf("%v^", cl.Hash))
+ if err != nil {
+ return ChangeList{}, err
+ }
+ cls := parseLog(string(out))
+ if len(cls) == 0 {
+ return ChangeList{}, fmt.Errorf("Unexpected output")
+ }
+ return cls[0], nil
+}
+
+// HeadCL returns the HEAD ChangeList at the given commit/tag/branch.
+func HeadCL(path string) (ChangeList, error) {
+ cls, err := LogFrom(path, "HEAD", 1)
+ if err != nil {
+ return ChangeList{}, err
+ }
+ if len(cls) == 0 {
+ return ChangeList{}, fmt.Errorf("No commits found")
+ }
+ return cls[0], nil
+}
+
+// Show content of the file at path for the given commit/tag/branch.
+func Show(path, at string) ([]byte, error) {
+ return shell.Exec(gitTimeout, exe, "", nil, "show", at+":"+path)
+}
+
+const prettyFormat = "ǁ%Hǀ%cIǀ%an <%ae>ǀ%sǀ%b"
+
+func parseLog(str string) []ChangeList {
+ msgs := strings.Split(str, "ǁ")
+ cls := make([]ChangeList, 0, len(msgs))
+ for _, s := range msgs {
+ if parts := strings.Split(s, "ǀ"); len(parts) == 5 {
+ cl := ChangeList{
+ Hash: ParseHash(parts[0]),
+ Author: strings.TrimSpace(parts[2]),
+ Subject: strings.TrimSpace(parts[3]),
+ Description: strings.TrimSpace(parts[4]),
+ }
+ date, err := time.Parse(time.RFC3339, parts[1])
+ if err != nil {
+ panic(err)
+ }
+ cl.Date = date
+
+ cls = append(cls, cl)
+ }
+ }
+ return cls
+}
diff --git a/tests/regres/main.go b/tests/regres/main.go
index d89a606..9766f09 100644
--- a/tests/regres/main.go
+++ b/tests/regres/main.go
@@ -42,6 +42,7 @@
"time"
"./cause"
+ "./consts"
"./git"
"./shell"
"./testlist"
@@ -444,7 +445,7 @@
}
commitMsg := strings.Builder{}
- commitMsg.WriteString("Regres: Update test lists @ " + headHash.String()[:8])
+ commitMsg.WriteString(consts.TestListUpdateCommitSubjectPrefix + headHash.String()[:8])
if results != nil && len(*changes) > 0 {
// Reuse gerrit change ID if there's already a change up for review.
id := (*changes)[0].ChangeID
@@ -739,11 +740,9 @@
out := []string{}
for _, list := range testLists {
- files := map[Status]*os.File{}
- ext := filepath.Ext(list.File)
- name := list.File[:len(list.File)-len(ext)]
- for _, status := range Statuses {
- path := filepath.Join(t.srcDir, name+"-"+string(status)+ext)
+ files := map[testlist.Status]*os.File{}
+ for _, status := range testlist.Statuses {
+ path := testlist.FilePathWithStatus(filepath.Join(t.srcDir, list.File), status)
dir := filepath.Dir(path)
os.MkdirAll(dir, 0777)
f, err := os.Create(path)
@@ -772,50 +771,6 @@
return filepath.Join(t.resDir, testLists.Hash())
}
-// Status is an enumerator of test results.
-type Status string
-
-const (
- // Pass is the status of a successful test.
- Pass = Status("PASS")
- // Fail is the status of a failed test.
- Fail = Status("FAIL")
- // Timeout is the status of a test that failed to complete in the alloted
- // time.
- Timeout = Status("TIMEOUT")
- // Crash is the status of a test that crashed.
- Crash = Status("CRASH")
- // NotSupported is the status of a test feature not supported by the driver.
- NotSupported = Status("NOT_SUPPORTED")
- // CompatibilityWarning is the status passing test with a warning.
- CompatibilityWarning = Status("COMPATIBILITY_WARNING")
- // QualityWarning is the status passing test with a warning.
- QualityWarning = Status("QUALITY_WARNING")
-)
-
-// Statuses is the full list of status types
-var Statuses = []Status{Pass, Fail, Timeout, Crash, NotSupported, CompatibilityWarning, QualityWarning}
-
-// Failing returns true if the task status requires fixing.
-func (s Status) Failing() bool {
- switch s {
- case Fail, Timeout, Crash:
- return true
- default:
- return false
- }
-}
-
-// Passing returns true if the task status is considered a pass.
-func (s Status) Passing() bool {
- switch s {
- case Pass, CompatibilityWarning, QualityWarning:
- return true
- default:
- return false
- }
-}
-
// CommitTestResults holds the results the tests across all APIs for a given
// commit. The CommitTestResults structure may be serialized to cache the
// results.
@@ -874,7 +829,7 @@
return "Build now fixed. Cannot compare against broken parent."
}
- oldStatusCounts, newStatusCounts := map[Status]int{}, map[Status]int{}
+ oldStatusCounts, newStatusCounts := map[testlist.Status]int{}, map[testlist.Status]int{}
totalTests := 0
broken, fixed, failing, removed, changed := []string{}, []string{}, []string{}, []string{}, []string{}
@@ -934,15 +889,15 @@
sb.WriteString(fmt.Sprintf(" Total tests: %d\n", totalTests))
for _, s := range []struct {
label string
- status Status
+ status testlist.Status
}{
- {" Pass", Pass},
- {" Fail", Fail},
- {" Timeout", Timeout},
- {" Crash", Crash},
- {" Not Supported", NotSupported},
- {"Compatibility Warning", CompatibilityWarning},
- {" Quality Warning", QualityWarning},
+ {" Pass", testlist.Pass},
+ {" Fail", testlist.Fail},
+ {" Timeout", testlist.Timeout},
+ {" Crash", testlist.Crash},
+ {" Not Supported", testlist.NotSupported},
+ {"Compatibility Warning", testlist.CompatibilityWarning},
+ {" Quality Warning", testlist.QualityWarning},
} {
old, new := oldStatusCounts[s.status], newStatusCounts[s.status]
if old == 0 && new == 0 {
@@ -1003,7 +958,7 @@
// TestResult holds the results of a single API test.
type TestResult struct {
Test string
- Status Status
+ Status testlist.Status
Err string `json:",omitempty"`
}
@@ -1037,13 +992,13 @@
default:
results <- TestResult{
Test: name,
- Status: Crash,
+ Status: testlist.Crash,
Err: cause.Wrap(err, string(out)).Error(),
}
case shell.ErrTimeout:
results <- TestResult{
Test: name,
- Status: Timeout,
+ Status: testlist.Timeout,
Err: cause.Wrap(err, string(out)).Error(),
}
case nil:
@@ -1051,28 +1006,28 @@
if len(toks) < 3 {
err := fmt.Sprintf("Couldn't parse test '%v' output:\n%s", name, string(out))
log.Println("Warning: ", err)
- results <- TestResult{Test: name, Status: Fail, Err: err}
+ results <- TestResult{Test: name, Status: testlist.Fail, Err: err}
continue
}
switch toks[1] {
case "Pass":
- results <- TestResult{Test: name, Status: Pass}
+ results <- TestResult{Test: name, Status: testlist.Pass}
case "NotSupported":
- results <- TestResult{Test: name, Status: NotSupported}
+ results <- TestResult{Test: name, Status: testlist.NotSupported}
case "CompatibilityWarning":
- results <- TestResult{Test: name, Status: CompatibilityWarning}
+ results <- TestResult{Test: name, Status: testlist.CompatibilityWarning}
case "QualityWarning":
- results <- TestResult{Test: name, Status: QualityWarning}
+ results <- TestResult{Test: name, Status: testlist.QualityWarning}
case "Fail":
var err string
if toks[2] != "Fail" {
err = toks[2]
}
- results <- TestResult{Test: name, Status: Fail, Err: err}
+ results <- TestResult{Test: name, Status: testlist.Fail, Err: err}
default:
err := fmt.Sprintf("Couldn't parse test output:\n%s", string(out))
log.Println("Warning: ", err)
- results <- TestResult{Test: name, Status: Fail, Err: err}
+ results <- TestResult{Test: name, Status: testlist.Fail, Err: err}
}
}
}
diff --git a/tests/regres/testlist/testlist.go b/tests/regres/testlist/testlist.go
index d2da533..ea24dcc 100644
--- a/tests/regres/testlist/testlist.go
+++ b/tests/regres/testlist/testlist.go
@@ -145,3 +145,55 @@
return out, nil
}
+
+// Status is an enumerator of test results.
+type Status string
+
+const (
+ // Pass is the status of a successful test.
+ Pass = Status("PASS")
+ // Fail is the status of a failed test.
+ Fail = Status("FAIL")
+ // Timeout is the status of a test that failed to complete in the alloted
+ // time.
+ Timeout = Status("TIMEOUT")
+ // Crash is the status of a test that crashed.
+ Crash = Status("CRASH")
+ // NotSupported is the status of a test feature not supported by the driver.
+ NotSupported = Status("NOT_SUPPORTED")
+ // CompatibilityWarning is the status passing test with a warning.
+ CompatibilityWarning = Status("COMPATIBILITY_WARNING")
+ // QualityWarning is the status passing test with a warning.
+ QualityWarning = Status("QUALITY_WARNING")
+)
+
+// Statuses is the full list of status types
+var Statuses = []Status{Pass, Fail, Timeout, Crash, NotSupported, CompatibilityWarning, QualityWarning}
+
+// Failing returns true if the task status requires fixing.
+func (s Status) Failing() bool {
+ switch s {
+ case Fail, Timeout, Crash:
+ return true
+ default:
+ return false
+ }
+}
+
+// Passing returns true if the task status is considered a pass.
+func (s Status) Passing() bool {
+ switch s {
+ case Pass, CompatibilityWarning, QualityWarning:
+ return true
+ default:
+ return false
+ }
+}
+
+// FilePathWithStatus returns the path to the test list file with the status
+// appended before the file extension.
+func FilePathWithStatus(listPath string, status Status) string {
+ ext := filepath.Ext(listPath)
+ name := listPath[:len(listPath)-len(ext)]
+ return name + "-" + string(status) + ext
+}