Stephan Altmueller | c35959f | 2018-01-08 15:53:37 -0500 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2018 Google Inc. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license that can be |
| 5 | * found in the LICENSE file. |
| 6 | */ |
| 7 | |
| 8 | package main |
| 9 | |
| 10 | import ( |
| 11 | "bytes" |
Stephan Altmueller | 88df8d2 | 2018-03-07 14:44:44 -0500 | [diff] [blame] | 12 | "context" |
Stephan Altmueller | c35959f | 2018-01-08 15:53:37 -0500 | [diff] [blame] | 13 | "encoding/json" |
| 14 | "flag" |
| 15 | "fmt" |
Stephan Altmueller | 88df8d2 | 2018-03-07 14:44:44 -0500 | [diff] [blame] | 16 | "io" |
| 17 | "io/ioutil" |
Stephan Altmueller | c35959f | 2018-01-08 15:53:37 -0500 | [diff] [blame] | 18 | "net/http" |
| 19 | "os" |
| 20 | "os/exec" |
| 21 | "sort" |
Stephan Altmueller | dd3eca1 | 2018-02-02 15:05:42 -0500 | [diff] [blame] | 22 | "strconv" |
Stephan Altmueller | c35959f | 2018-01-08 15:53:37 -0500 | [diff] [blame] | 23 | "strings" |
| 24 | "syscall" |
| 25 | "time" |
| 26 | |
Stephan Altmueller | 88df8d2 | 2018-03-07 14:44:44 -0500 | [diff] [blame] | 27 | "go.skia.org/infra/go/gcs" |
| 28 | |
| 29 | "cloud.google.com/go/storage" |
| 30 | "google.golang.org/api/option" |
Stephan Altmueller | c35959f | 2018-01-08 15:53:37 -0500 | [diff] [blame] | 31 | gstorage "google.golang.org/api/storage/v1" |
| 32 | |
| 33 | "go.skia.org/infra/go/auth" |
| 34 | "go.skia.org/infra/go/common" |
| 35 | "go.skia.org/infra/go/sklog" |
| 36 | "go.skia.org/infra/go/util" |
Stephan Altmueller | c35959f | 2018-01-08 15:53:37 -0500 | [diff] [blame] | 37 | ) |
| 38 | |
| 39 | const ( |
| 40 | META_DATA_FILENAME = "meta.json" |
| 41 | ) |
| 42 | |
| 43 | // Command line flags. |
| 44 | var ( |
Stephan Altmueller | 88df8d2 | 2018-03-07 14:44:44 -0500 | [diff] [blame] | 45 | devicesFile = flag.String("devices", "", "JSON file that maps device ids to versions to run on. Same format as produced by the dump_devices flag.") |
Stephan Altmueller | c35959f | 2018-01-08 15:53:37 -0500 | [diff] [blame] | 46 | dryRun = flag.Bool("dryrun", false, "Print out the command and quit without triggering tests.") |
Stephan Altmueller | 88df8d2 | 2018-03-07 14:44:44 -0500 | [diff] [blame] | 47 | dumpDevFile = flag.String("dump_devices", "", "Creates a JSON file with all physical devices that are not deprecated.") |
| 48 | minAPIVersion = flag.Int("min_api", 0, "Minimum API version required by device.") |
| 49 | maxAPIVersion = flag.Int("max_api", 99, "Maximum API version required by device.") |
| 50 | properties = flag.String("properties", "", "Custom meta data to be added to the uploaded APK. Comma separated list of key=value pairs, i.e. 'k1=v1,k2=v2,k3=v3.") |
| 51 | serviceAccountFile = flag.String("service_account_file", "", "Credentials file for service account.") |
| 52 | uploadGCSPath = flag.String("upload_path", "", "GCS path (bucket/path) to where the APK should be uploaded to. It's assume to a full path (not a directory).") |
Stephan Altmueller | c35959f | 2018-01-08 15:53:37 -0500 | [diff] [blame] | 53 | ) |
| 54 | |
| 55 | const ( |
| 56 | RUN_TESTS_TEMPLATE = `gcloud beta firebase test android run |
| 57 | --type=game-loop |
| 58 | --app=%s |
| 59 | --results-bucket=%s |
| 60 | --results-dir=%s |
Stephan Altmueller | 8ee7fb0 | 2018-01-16 15:55:48 -0500 | [diff] [blame] | 61 | --directories-to-pull=/sdcard/Android/data/org.skia.skqp |
| 62 | --timeout 30m |
Stephan Altmueller | c35959f | 2018-01-08 15:53:37 -0500 | [diff] [blame] | 63 | %s |
| 64 | ` |
Stephan Altmueller | 88df8d2 | 2018-03-07 14:44:44 -0500 | [diff] [blame] | 65 | MODEL_VERSION_TMPL = "--device model=%s,version=%s,orientation=portrait" |
| 66 | RESULT_BUCKET = "skia-firebase-test-lab" |
| 67 | RESULT_DIR_TMPL = "testruns/%s/%s" |
| 68 | RUN_ID_TMPL = "testrun-%d" |
| 69 | CMD_AVAILABLE_DEVICES = "gcloud firebase test android models list --format json" |
Stephan Altmueller | c35959f | 2018-01-08 15:53:37 -0500 | [diff] [blame] | 70 | ) |
| 71 | |
| 72 | func main() { |
| 73 | common.Init() |
| 74 | |
Stephan Altmueller | 88df8d2 | 2018-03-07 14:44:44 -0500 | [diff] [blame] | 75 | // Get the path to the APK. It can be empty if we are dumping the device list. |
| 76 | apkPath := flag.Arg(0) |
| 77 | if *dumpDevFile == "" && apkPath == "" { |
| 78 | sklog.Errorf("Missing APK. The APK file needs to be passed as the positional argument.") |
| 79 | os.Exit(1) |
| 80 | } |
Stephan Altmueller | c35959f | 2018-01-08 15:53:37 -0500 | [diff] [blame] | 81 | |
Stephan Altmueller | 88df8d2 | 2018-03-07 14:44:44 -0500 | [diff] [blame] | 82 | // Get the available devices. |
| 83 | fbDevices, deviceList, err := getAvailableDevices() |
| 84 | if err != nil { |
| 85 | sklog.Fatalf("Error retrieving available devices: %s", err) |
| 86 | } |
| 87 | |
| 88 | // Dump the device list and exit. |
| 89 | if *dumpDevFile != "" { |
| 90 | if err := writeDeviceList(*dumpDevFile, deviceList); err != nil { |
| 91 | sklog.Fatalf("Unable to write devices: %s", err) |
| 92 | } |
| 93 | return |
| 94 | } |
| 95 | |
| 96 | // If no devices are explicitly listed. Use all of them. |
| 97 | whiteList := deviceList |
| 98 | if *devicesFile != "" { |
| 99 | whiteList, err = readDeviceList(*devicesFile) |
| 100 | if err != nil { |
| 101 | sklog.Fatalf("Error reading device file: %s", err) |
| 102 | } |
| 103 | } |
| 104 | |
| 105 | // Make sure we can authenticate locally and in the cloud. |
Stephan Altmueller | c35959f | 2018-01-08 15:53:37 -0500 | [diff] [blame] | 106 | client, err := auth.NewJWTServiceAccountClient("", *serviceAccountFile, nil, gstorage.CloudPlatformScope, "https://www.googleapis.com/auth/userinfo.email") |
| 107 | if err != nil { |
| 108 | sklog.Fatalf("Failed to authenticate service account: %s. Run 'get_service_account' to obtain a service account file.", err) |
| 109 | } |
| 110 | |
Stephan Altmueller | 88df8d2 | 2018-03-07 14:44:44 -0500 | [diff] [blame] | 111 | // Filter the devices according the white list and other parameters. |
| 112 | devices, ignoredDevices := filterDevices(fbDevices, whiteList, *minAPIVersion, *maxAPIVersion) |
| 113 | sklog.Infof("---\nSelected devices:") |
Stephan Altmueller | c35959f | 2018-01-08 15:53:37 -0500 | [diff] [blame] | 114 | logDevices(devices) |
| 115 | |
Stephan Altmueller | 88df8d2 | 2018-03-07 14:44:44 -0500 | [diff] [blame] | 116 | if len(devices) == 0 { |
| 117 | sklog.Errorf("No devices selected. Not running tests.") |
| 118 | os.Exit(1) |
| 119 | } |
| 120 | |
| 121 | if err := runTests(apkPath, devices, ignoredDevices, client, *dryRun); err != nil { |
| 122 | sklog.Fatalf("Error running tests on Firebase: %s", err) |
| 123 | } |
| 124 | |
| 125 | if !*dryRun && (*uploadGCSPath != "") && (*properties != "") { |
| 126 | if err := uploadAPK(apkPath, *uploadGCSPath, *properties, client); err != nil { |
| 127 | sklog.Fatalf("Error uploading APK to '%s': %s", *uploadGCSPath, err) |
| 128 | } |
Stephan Altmueller | c35959f | 2018-01-08 15:53:37 -0500 | [diff] [blame] | 129 | } |
| 130 | } |
| 131 | |
Stephan Altmueller | 88df8d2 | 2018-03-07 14:44:44 -0500 | [diff] [blame] | 132 | // getAvailableDevices queries Firebase Testlab for all physical devices that |
| 133 | // are not deprecated. It returns two lists with the same information. |
| 134 | // The first contains all device information as returned by Firebase while |
| 135 | // the second contains the information necessary to use in a whitelist. |
| 136 | func getAvailableDevices() ([]*DeviceVersions, DeviceList, error) { |
Stephan Altmueller | c35959f | 2018-01-08 15:53:37 -0500 | [diff] [blame] | 137 | // Get the list of all devices in JSON format from Firebase testlab. |
| 138 | var buf bytes.Buffer |
Stephan Altmueller | 88df8d2 | 2018-03-07 14:44:44 -0500 | [diff] [blame] | 139 | var errBuf bytes.Buffer |
| 140 | cmd := parseCommand(CMD_AVAILABLE_DEVICES) |
Stephan Altmueller | c35959f | 2018-01-08 15:53:37 -0500 | [diff] [blame] | 141 | cmd.Stdout = &buf |
Stephan Altmueller | 88df8d2 | 2018-03-07 14:44:44 -0500 | [diff] [blame] | 142 | cmd.Stderr = io.MultiWriter(os.Stdout, &errBuf) |
Stephan Altmueller | c35959f | 2018-01-08 15:53:37 -0500 | [diff] [blame] | 143 | if err := cmd.Run(); err != nil { |
Stephan Altmueller | 88df8d2 | 2018-03-07 14:44:44 -0500 | [diff] [blame] | 144 | return nil, nil, sklog.FmtErrorf("Error running: %s\nError:%s\nStdErr:%s", CMD_AVAILABLE_DEVICES, err, errBuf) |
Stephan Altmueller | c35959f | 2018-01-08 15:53:37 -0500 | [diff] [blame] | 145 | } |
| 146 | |
| 147 | // Unmarshal the result. |
Stephan Altmueller | 88df8d2 | 2018-03-07 14:44:44 -0500 | [diff] [blame] | 148 | foundDevices := []*DeviceVersions{} |
Stephan Altmueller | c35959f | 2018-01-08 15:53:37 -0500 | [diff] [blame] | 149 | bufBytes := buf.Bytes() |
| 150 | if err := json.Unmarshal(bufBytes, &foundDevices); err != nil { |
| 151 | return nil, nil, sklog.FmtErrorf("Unmarshal of device information failed: %s \nJSON Input: %s\n", err, string(bufBytes)) |
| 152 | } |
| 153 | |
Stephan Altmueller | 88df8d2 | 2018-03-07 14:44:44 -0500 | [diff] [blame] | 154 | // Filter the devices and copy them to device list. |
| 155 | devList := DeviceList{} |
| 156 | ret := make([]*DeviceVersions, 0, len(foundDevices)) |
| 157 | for _, foundDev := range foundDevices { |
| 158 | // Only consider physical devices and devices that are not deprecated. |
| 159 | if (foundDev.Form == "PHYSICAL") && !util.In("deprecated", foundDev.Tags) { |
| 160 | ret = append(ret, foundDev) |
| 161 | devList = append(devList, &DevInfo{ |
| 162 | ID: foundDev.ID, |
| 163 | Name: foundDev.Name, |
| 164 | RunVersions: foundDev.VersionIDs, |
| 165 | }) |
Stephan Altmueller | c35959f | 2018-01-08 15:53:37 -0500 | [diff] [blame] | 166 | } |
| 167 | } |
Stephan Altmueller | 88df8d2 | 2018-03-07 14:44:44 -0500 | [diff] [blame] | 168 | return foundDevices, devList, nil |
| 169 | } |
| 170 | |
| 171 | // filterDevices filters the given devices by ensuring that they are in the white list |
| 172 | // and within the given api version range. |
| 173 | // It returns two lists: (accepted_devices, ignored_devices) |
| 174 | func filterDevices(foundDevices []*DeviceVersions, whiteList DeviceList, minAPIVersion, maxAPIVersion int) ([]*DeviceVersions, []*DeviceVersions) { |
| 175 | // iterate over the available devices and partition them. |
| 176 | allDevices := make([]*DeviceVersions, 0, len(foundDevices)) |
| 177 | ret := make([]*DeviceVersions, 0, len(foundDevices)) |
| 178 | ignored := make([]*DeviceVersions, 0, len(foundDevices)) |
| 179 | for _, dev := range foundDevices { |
| 180 | // Only include devices that are on the whitelist and have versions defined. |
| 181 | if targetDev := whiteList.find(dev.ID); targetDev != nil && (len(targetDev.RunVersions) > 0) { |
| 182 | versionSet := util.NewStringSet(dev.VersionIDs) |
| 183 | reqVersions := util.NewStringSet(filterVersions(targetDev.RunVersions, minAPIVersion, maxAPIVersion)) |
| 184 | whiteListVersions := versionSet.Intersect(reqVersions).Keys() |
| 185 | ignoredVersions := versionSet.Complement(reqVersions).Keys() |
| 186 | sort.Strings(whiteListVersions) |
| 187 | sort.Strings(ignoredVersions) |
| 188 | if len(whiteListVersions) > 0 { |
| 189 | ret = append(ret, &DeviceVersions{FirebaseDevice: dev.FirebaseDevice, RunVersions: whiteListVersions}) |
| 190 | } |
| 191 | if len(ignoredVersions) > 0 { |
| 192 | ignored = append(ignored, &DeviceVersions{FirebaseDevice: dev.FirebaseDevice, RunVersions: ignoredVersions}) |
| 193 | } |
| 194 | } else { |
| 195 | ignored = append(ignored, &DeviceVersions{FirebaseDevice: dev.FirebaseDevice, RunVersions: dev.VersionIDs}) |
| 196 | } |
| 197 | allDevices = append(allDevices, &DeviceVersions{FirebaseDevice: dev.FirebaseDevice, RunVersions: dev.VersionIDs}) |
| 198 | } |
Stephan Altmueller | c35959f | 2018-01-08 15:53:37 -0500 | [diff] [blame] | 199 | |
| 200 | sklog.Infof("All devices:") |
| 201 | logDevices(allDevices) |
| 202 | |
Stephan Altmueller | 88df8d2 | 2018-03-07 14:44:44 -0500 | [diff] [blame] | 203 | return ret, ignored |
Stephan Altmueller | c35959f | 2018-01-08 15:53:37 -0500 | [diff] [blame] | 204 | } |
| 205 | |
Stephan Altmueller | dd3eca1 | 2018-02-02 15:05:42 -0500 | [diff] [blame] | 206 | // filterVersions returns the elements in versionIDs where minVersion <= element <= maxVersion. |
| 207 | func filterVersions(versionIDs []string, minVersion, maxVersion int) []string { |
| 208 | ret := make([]string, 0, len(versionIDs)) |
| 209 | for _, versionID := range versionIDs { |
| 210 | id, err := strconv.Atoi(versionID) |
| 211 | if err != nil { |
| 212 | sklog.Fatalf("Error parsing version id '%s': %s", versionID, err) |
| 213 | } |
| 214 | if (id >= minVersion) && (id <= maxVersion) { |
| 215 | ret = append(ret, versionID) |
| 216 | } |
| 217 | } |
| 218 | return ret |
| 219 | } |
| 220 | |
Stephan Altmueller | c35959f | 2018-01-08 15:53:37 -0500 | [diff] [blame] | 221 | // runTests runs the given apk on the given list of devices. |
Stephan Altmueller | 88df8d2 | 2018-03-07 14:44:44 -0500 | [diff] [blame] | 222 | func runTests(apk_path string, devices, ignoredDevices []*DeviceVersions, client *http.Client, dryRun bool) error { |
Stephan Altmueller | c35959f | 2018-01-08 15:53:37 -0500 | [diff] [blame] | 223 | // Get the model-version we want to test. Assume on average each model has 5 supported versions. |
| 224 | modelSelectors := make([]string, 0, len(devices)*5) |
| 225 | for _, devRec := range devices { |
Stephan Altmueller | 88df8d2 | 2018-03-07 14:44:44 -0500 | [diff] [blame] | 226 | for _, version := range devRec.RunVersions { |
| 227 | modelSelectors = append(modelSelectors, fmt.Sprintf(MODEL_VERSION_TMPL, devRec.FirebaseDevice.ID, version)) |
Stephan Altmueller | c35959f | 2018-01-08 15:53:37 -0500 | [diff] [blame] | 228 | } |
| 229 | } |
| 230 | |
| 231 | now := time.Now() |
| 232 | nowMs := now.UnixNano() / int64(time.Millisecond) |
| 233 | runID := fmt.Sprintf(RUN_ID_TMPL, nowMs) |
| 234 | resultsDir := fmt.Sprintf(RESULT_DIR_TMPL, now.Format("2006/01/02/15"), runID) |
| 235 | cmdStr := fmt.Sprintf(RUN_TESTS_TEMPLATE, apk_path, RESULT_BUCKET, resultsDir, strings.Join(modelSelectors, "\n")) |
| 236 | cmdStr = strings.TrimSpace(strings.Replace(cmdStr, "\n", " ", -1)) |
| 237 | |
| 238 | // Run the command. |
Stephan Altmueller | 88df8d2 | 2018-03-07 14:44:44 -0500 | [diff] [blame] | 239 | var errBuf bytes.Buffer |
Stephan Altmueller | c35959f | 2018-01-08 15:53:37 -0500 | [diff] [blame] | 240 | cmd := parseCommand(cmdStr) |
| 241 | cmd.Stdout = os.Stdout |
Stephan Altmueller | 88df8d2 | 2018-03-07 14:44:44 -0500 | [diff] [blame] | 242 | cmd.Stderr = io.MultiWriter(os.Stdout, &errBuf) |
Stephan Altmueller | c35959f | 2018-01-08 15:53:37 -0500 | [diff] [blame] | 243 | exitCode := 0 |
| 244 | |
| 245 | if dryRun { |
| 246 | fmt.Printf("[dry run]: Would have run this command: %s\n", cmdStr) |
| 247 | return nil |
| 248 | } |
| 249 | |
| 250 | if err := cmd.Run(); err != nil { |
| 251 | // Get the exit code. |
| 252 | if exitError, ok := err.(*exec.ExitError); ok { |
| 253 | ws := exitError.Sys().(syscall.WaitStatus) |
| 254 | exitCode = ws.ExitStatus() |
| 255 | } |
Stephan Altmueller | 88df8d2 | 2018-03-07 14:44:44 -0500 | [diff] [blame] | 256 | |
Stephan Altmueller | c35959f | 2018-01-08 15:53:37 -0500 | [diff] [blame] | 257 | sklog.Errorf("Error running tests: %s", err) |
| 258 | sklog.Errorf("Exit code: %d", exitCode) |
| 259 | |
| 260 | // Exit code 10 means triggering on Testlab succeeded, but but some of the |
| 261 | // runs on devices failed. We consider it a success for this script. |
| 262 | if exitCode != 10 { |
Stephan Altmueller | 88df8d2 | 2018-03-07 14:44:44 -0500 | [diff] [blame] | 263 | return sklog.FmtErrorf("Error running: %s\nError:%s\nStdErr:%s", cmdStr, err, errBuf) |
Stephan Altmueller | c35959f | 2018-01-08 15:53:37 -0500 | [diff] [blame] | 264 | } |
| 265 | } |
| 266 | |
| 267 | // Store the result in a meta json file. |
Stephan Altmueller | 88df8d2 | 2018-03-07 14:44:44 -0500 | [diff] [blame] | 268 | meta := &TestRunMeta{ |
Stephan Altmueller | c35959f | 2018-01-08 15:53:37 -0500 | [diff] [blame] | 269 | ID: runID, |
| 270 | TS: nowMs, |
| 271 | Devices: devices, |
| 272 | IgnoredDevices: ignoredDevices, |
| 273 | ExitCode: exitCode, |
| 274 | } |
| 275 | |
Stephan Altmueller | 88df8d2 | 2018-03-07 14:44:44 -0500 | [diff] [blame] | 276 | targetPath := fmt.Sprintf("%s/%s/%s", RESULT_BUCKET, resultsDir, META_DATA_FILENAME) |
| 277 | if err := meta.writeToGCS(targetPath, client); err != nil { |
| 278 | return err |
| 279 | } |
| 280 | sklog.Infof("Meta data written to gs://%s", targetPath) |
Stephan Altmueller | c35959f | 2018-01-08 15:53:37 -0500 | [diff] [blame] | 281 | return nil |
| 282 | } |
| 283 | |
Stephan Altmueller | 88df8d2 | 2018-03-07 14:44:44 -0500 | [diff] [blame] | 284 | // uploadAPK uploads the APK at the given path to the bucket/path in gcsPath. |
| 285 | // The key-value pairs in propStr are set as custom meta data of the APK. |
| 286 | func uploadAPK(apkPath, gcsPath, propStr string, client *http.Client) error { |
| 287 | properties, err := splitProperties(propStr) |
| 288 | if err != nil { |
| 289 | return err |
| 290 | } |
| 291 | apkFile, err := os.Open(apkPath) |
| 292 | if err != nil { |
| 293 | return err |
| 294 | } |
| 295 | defer util.Close(apkFile) |
| 296 | |
| 297 | if err := copyReaderToGCS(gcsPath, apkFile, client, "application/vnd.android.package-archive", properties, true, false); err != nil { |
| 298 | return err |
| 299 | } |
| 300 | |
| 301 | sklog.Infof("APK uploaded to gs://%s", gcsPath) |
| 302 | return nil |
| 303 | } |
| 304 | |
| 305 | // splitProperties receives a comma separated list of 'key=value' pairs and |
| 306 | // returnes them as a map. |
| 307 | func splitProperties(propStr string) (map[string]string, error) { |
| 308 | splitProps := strings.Split(propStr, ",") |
| 309 | properties := make(map[string]string, len(splitProps)) |
| 310 | for _, oneProp := range splitProps { |
| 311 | kv := strings.Split(oneProp, "=") |
| 312 | if len(kv) != 2 { |
| 313 | return nil, sklog.FmtErrorf("Inavlid porperties format. Unable to parse '%s'", propStr) |
| 314 | } |
| 315 | properties[strings.TrimSpace(kv[0])] = strings.TrimSpace(kv[1]) |
| 316 | } |
| 317 | return properties, nil |
| 318 | } |
| 319 | |
Stephan Altmueller | c35959f | 2018-01-08 15:53:37 -0500 | [diff] [blame] | 320 | // logDevices logs the given list of devices. |
Stephan Altmueller | 88df8d2 | 2018-03-07 14:44:44 -0500 | [diff] [blame] | 321 | func logDevices(devices []*DeviceVersions) { |
Stephan Altmueller | c35959f | 2018-01-08 15:53:37 -0500 | [diff] [blame] | 322 | sklog.Infof("Found %d devices.", len(devices)) |
| 323 | for _, dev := range devices { |
Stephan Altmueller | 88df8d2 | 2018-03-07 14:44:44 -0500 | [diff] [blame] | 324 | fbDev := dev.FirebaseDevice |
| 325 | sklog.Infof("%-15s %-30s %v / %v", fbDev.ID, fbDev.Name, fbDev.VersionIDs, dev.RunVersions) |
Stephan Altmueller | c35959f | 2018-01-08 15:53:37 -0500 | [diff] [blame] | 326 | } |
| 327 | } |
| 328 | |
| 329 | // parseCommad parses a command line and wraps it in an exec.Command instance. |
| 330 | func parseCommand(cmdStr string) *exec.Cmd { |
| 331 | cmdArgs := strings.Split(strings.TrimSpace(cmdStr), " ") |
| 332 | for idx := range cmdArgs { |
| 333 | cmdArgs[idx] = strings.TrimSpace(cmdArgs[idx]) |
| 334 | } |
| 335 | return exec.Command(cmdArgs[0], cmdArgs[1:]...) |
| 336 | } |
Stephan Altmueller | 88df8d2 | 2018-03-07 14:44:44 -0500 | [diff] [blame] | 337 | |
| 338 | // DeviceList is a simple list of devices, primarily used to define the |
| 339 | // whitelist of devices we want to run on. |
| 340 | type DeviceList []*DevInfo |
| 341 | |
| 342 | type DevInfo struct { |
| 343 | ID string `json:"id"` |
| 344 | Name string `json:"name"` |
| 345 | RunVersions []string `json:"runVersions"` |
| 346 | } |
| 347 | |
| 348 | func (d DeviceList) find(id string) *DevInfo { |
| 349 | for _, devInfo := range d { |
| 350 | if devInfo.ID == id { |
| 351 | return devInfo |
| 352 | } |
| 353 | } |
| 354 | return nil |
| 355 | } |
| 356 | |
| 357 | func writeDeviceList(fileName string, devList DeviceList) error { |
| 358 | jsonBytes, err := json.MarshalIndent(devList, "", " ") |
| 359 | if err != nil { |
| 360 | return sklog.FmtErrorf("Unable to encode JSON: %s", err) |
| 361 | } |
| 362 | |
| 363 | if err := ioutil.WriteFile(fileName, jsonBytes, 0644); err != nil { |
| 364 | sklog.FmtErrorf("Unable to write file '%s': %s", fileName, err) |
| 365 | } |
| 366 | return nil |
| 367 | } |
| 368 | |
| 369 | func readDeviceList(fileName string) (DeviceList, error) { |
| 370 | inFile, err := os.Open(fileName) |
| 371 | if err != nil { |
| 372 | return nil, sklog.FmtErrorf("Unable to open file '%s': %s", fileName, err) |
| 373 | } |
| 374 | defer util.Close(inFile) |
| 375 | |
| 376 | var devList DeviceList |
| 377 | if err := json.NewDecoder(inFile).Decode(&devList); err != nil { |
| 378 | return nil, sklog.FmtErrorf("Unable to decode JSON from '%s': %s", fileName, err) |
| 379 | } |
| 380 | return devList, nil |
| 381 | } |
| 382 | |
| 383 | // FirebaseDevice contains the information and JSON tags for device information |
| 384 | // returned by firebase. |
| 385 | type FirebaseDevice struct { |
| 386 | Brand string `json:"brand"` |
| 387 | Form string `json:"form"` |
| 388 | ID string `json:"id"` |
| 389 | Manufacturer string `json:"manufacturer"` |
| 390 | Name string `json:"name"` |
| 391 | VersionIDs []string `json:"supportedVersionIds"` |
| 392 | Tags []string `json:"tags"` |
| 393 | } |
| 394 | |
| 395 | // DeviceVersions combines device information from Firebase Testlab with |
| 396 | // a selected list of versions. This is used to define a subset of versions |
| 397 | // used by a devices. |
| 398 | type DeviceVersions struct { |
| 399 | *FirebaseDevice |
| 400 | |
| 401 | // RunVersions contains the version ids of interest contained in Device. |
| 402 | RunVersions []string |
| 403 | } |
| 404 | |
| 405 | // TestRunMeta contains the meta data of a complete testrun on firebase. |
| 406 | type TestRunMeta struct { |
| 407 | ID string `json:"id"` |
| 408 | TS int64 `json:"timeStamp"` |
| 409 | Devices []*DeviceVersions `json:"devices"` |
| 410 | IgnoredDevices []*DeviceVersions `json:"ignoredDevices"` |
| 411 | ExitCode int `json:"exitCode"` |
| 412 | } |
| 413 | |
| 414 | // writeToGCS writes the meta data as JSON to the given bucket and path in |
| 415 | // GCS. It assumes that the provided client has permissions to write to the |
| 416 | // specified location in GCS. |
| 417 | func (t *TestRunMeta) writeToGCS(gcsPath string, client *http.Client) error { |
| 418 | jsonBytes, err := json.Marshal(t) |
| 419 | if err != nil { |
| 420 | return err |
| 421 | } |
| 422 | return copyReaderToGCS(gcsPath, bytes.NewReader(jsonBytes), client, "", nil, false, true) |
| 423 | } |
| 424 | |
| 425 | // TODO(stephana): Merge copyReaderToGCS into the go/gcs in |
| 426 | // the infra repository. |
| 427 | |
| 428 | // copyReaderToGCS reads all available content from the given reader and writes |
| 429 | // it to the given path in GCS. |
| 430 | func copyReaderToGCS(gcsPath string, reader io.Reader, client *http.Client, contentType string, metaData map[string]string, public bool, gzip bool) error { |
| 431 | storageClient, err := storage.NewClient(context.Background(), option.WithHTTPClient(client)) |
| 432 | if err != nil { |
| 433 | return err |
| 434 | } |
| 435 | bucket, path := gcs.SplitGSPath(gcsPath) |
| 436 | w := storageClient.Bucket(bucket).Object(path).NewWriter(context.Background()) |
| 437 | |
| 438 | // Set the content if requested. |
| 439 | if contentType != "" { |
| 440 | w.ObjectAttrs.ContentType = contentType |
| 441 | } |
| 442 | |
| 443 | // Set the meta data if requested |
| 444 | if metaData != nil { |
| 445 | w.Metadata = metaData |
| 446 | } |
| 447 | |
| 448 | // Make the object public if requested. |
| 449 | if public { |
| 450 | w.ACL = []storage.ACLRule{{Entity: storage.AllUsers, Role: storage.RoleReader}} |
| 451 | } |
| 452 | |
| 453 | // Write the everything the reader can provide to the GCS object. Either |
| 454 | // gzip'ed or plain. |
| 455 | if gzip { |
| 456 | w.ObjectAttrs.ContentEncoding = "gzip" |
| 457 | err = util.WithGzipWriter(w, func(w io.Writer) error { |
| 458 | _, err := io.Copy(w, reader) |
| 459 | return err |
| 460 | }) |
| 461 | } else { |
| 462 | _, err = io.Copy(w, reader) |
| 463 | } |
| 464 | |
| 465 | // Make sure we return an error when we close the remote object. |
| 466 | if err != nil { |
| 467 | _ = w.CloseWithError(err) |
| 468 | return err |
| 469 | } |
| 470 | return w.Close() |
| 471 | } |