gen_tasks.go: Implement TODOs, bring up to speed with swarm_trigger
BUG=skia:
GOLD_TRYBOT_URL= https://gold.skia.org/search?issue=4547
Change-Id: Ia22a4836732dd33f30f4f6e4e49f5b39294ec53f
Reviewed-on: https://skia-review.googlesource.com/4547
Reviewed-by: Ben Wagner <benjaminwagner@google.com>
Commit-Queue: Eric Boren <borenet@google.com>
diff --git a/infra/bots/gen_tasks.go b/infra/bots/gen_tasks.go
index 846b4c4..28e6b24 100644
--- a/infra/bots/gen_tasks.go
+++ b/infra/bots/gen_tasks.go
@@ -13,9 +13,9 @@
"fmt"
"os"
"path"
- "path/filepath"
"sort"
"strings"
+ "time"
"github.com/skia-dev/glog"
"go.skia.org/infra/go/util"
@@ -46,8 +46,9 @@
"Housekeeper-PerCommit-InfraTests",
}
- // UPLOAD_DIMENSIONS are the Swarming dimensions for upload tasks.
- UPLOAD_DIMENSIONS = []string{
+ // LINUX_GCE_DIMENSIONS are the Swarming dimensions for Linux GCE
+ // instances.
+ LINUX_GCE_DIMENSIONS = []string{
"cpu:x86-64-avx2",
"gpu:none",
"os:Ubuntu",
@@ -98,11 +99,6 @@
// swarmDimensions generates swarming bot dimensions for the given task.
func swarmDimensions(parts map[string]string) []string {
- if parts["extra_config"] == "SkiaCT" {
- return []string{
- "pool:SkiaCT",
- }
- }
d := map[string]string{
"pool": POOL_SKIA,
}
@@ -118,28 +114,31 @@
if strings.Contains(parts["os"], "Android") {
// For Android, the device type is a better dimension
// than CPU or GPU.
- d["device_type"] = map[string]string{
- "AndroidOne": "sprout",
- "GalaxyS3": "m0", // "smdk4x12", Detected incorrectly by swarming?
- "GalaxyS4": "", // TODO(borenet,kjlubick)
- "GalaxyS7": "heroqlteatt",
- "NVIDIA_Shield": "foster",
- "Nexus10": "manta",
- "Nexus5": "hammerhead",
- "Nexus6": "shamu",
- "Nexus6p": "angler",
- "Nexus7": "grouper",
- "Nexus7v2": "flo",
- "Nexus9": "flounder",
- "NexusPlayer": "fugu",
+ deviceInfo := map[string][]string{
+ "AndroidOne": {"sprout", "MOB30Q"},
+ "GalaxyS7": {"heroqlteatt", "MMB29M"},
+ "NVIDIA_Shield": {"foster", "MRA58K"},
+ "Nexus10": {"manta", "LMY49J"},
+ "Nexus5": {"hammerhead", "MOB31E"},
+ "Nexus6": {"shamu", "M"},
+ "Nexus6p": {"angler", "NMF26C"},
+ "Nexus7": {"grouper", "LMY47V"},
+ "Nexus7v2": {"flo", "M"},
+ "Nexus9": {"flounder", "NRD91D"},
+ "NexusPlayer": {"fugu", "NRD90R"},
+ "Pixel": {"sailfish", "NMF25"},
+ "PixelC": {"dragon", "NMF26C"},
+ "PixelXL": {"marlin", "NMF25"},
}[parts["model"]]
+ d["device_type"] = deviceInfo[0]
+ d["device_os"] = deviceInfo[1]
} else if strings.Contains(parts["os"], "iOS") {
d["device"] = map[string]string{
- "iPad4": "iPad4,1",
+ "iPad4": "iPad5,1",
}[parts["model"]]
// TODO(borenet): Replace this hack with something
// better.
- d["os"] = "iOS-9.2"
+ d["os"] = "iOS-9.3.1"
} else if parts["cpu_or_gpu"] == "CPU" {
d["gpu"] = "none"
d["cpu"] = map[string]string{
@@ -156,15 +155,16 @@
}
} else {
d["gpu"] = map[string]string{
- "GeForce320M": "10de:08a4",
- "GT610": "10de:104a",
- "GTX550Ti": "10de:1244",
- "GTX660": "10de:11c0",
- "GTX960": "10de:1401",
- "HD4000": "8086:0a2e",
- "HD4600": "8086:0412",
- "HD7770": "1002:683d",
- "iHD530": "8086:1912",
+ "GeForce320M": "10de:08a4",
+ "GT610": "10de:104a",
+ "GTX550Ti": "10de:1244",
+ "GTX660": "10de:11c0",
+ "GTX960": "10de:1401",
+ "HD4000": "8086:0a2e",
+ "HD4600": "8086:0412",
+ "HD7770": "1002:683d",
+ "iHD530": "8086:1912",
+ "IntelIris6100": "8086:162b",
}[parts["cpu_or_gpu_value"]]
}
} else {
@@ -238,21 +238,81 @@
// task in the generated chain of tasks, which the Job should add as a
// dependency.
func recreateSKPs(b *specs.TasksCfgBuilder, name string) string {
- // TODO
+ b.MustAddTask(name, &specs.TaskSpec{
+ CipdPackages: []*specs.CipdPackage{},
+ Dimensions: LINUX_GCE_DIMENSIONS,
+ ExtraArgs: []string{
+ "--workdir", "../../..", "swarm_RecreateSKPs",
+ fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO),
+ fmt.Sprintf("buildername=%s", name),
+ "mastername=fake-master",
+ "buildnumber=2",
+ "slavename=fake-buildslave",
+ "nobuildbot=True",
+ fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR),
+ fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION),
+ fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE),
+ fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE),
+ fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET),
+ },
+ Isolate: "compile_skia.isolate",
+ Priority: 0.8,
+ })
return name
}
// ctSKPs generates a CT SKPs task. Returns the name of the last task in the
// generated chain of tasks, which the Job should add as a dependency.
func ctSKPs(b *specs.TasksCfgBuilder, name string) string {
- // TODO
+ b.MustAddTask(name, &specs.TaskSpec{
+ CipdPackages: []*specs.CipdPackage{},
+ Dimensions: []string{"pool:SkiaCT"},
+ ExecutionTimeout: 24 * time.Hour,
+ ExtraArgs: []string{
+ "--workdir", "../../..", "swarm_ct_skps",
+ fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO),
+ fmt.Sprintf("buildername=%s", name),
+ "mastername=fake-master",
+ "buildnumber=2",
+ "slavename=fake-buildslave",
+ "nobuildbot=True",
+ fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR),
+ fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION),
+ fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE),
+ fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE),
+ fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET),
+ },
+ IoTimeout: time.Hour,
+ Isolate: "ct_skps_skia.isolate",
+ Priority: 0.8,
+ })
return name
}
// housekeeper generates a Housekeeper task. Returns the name of the last task
// in the generated chain of tasks, which the Job should add as a dependency.
func housekeeper(b *specs.TasksCfgBuilder, name, compileTaskName string) string {
- // TODO
+ b.MustAddTask(name, &specs.TaskSpec{
+ CipdPackages: []*specs.CipdPackage{},
+ Dependencies: []string{compileTaskName},
+ Dimensions: LINUX_GCE_DIMENSIONS,
+ ExtraArgs: []string{
+ "--workdir", "../../..", "swarm_housekeeper",
+ fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO),
+ fmt.Sprintf("buildername=%s", name),
+ "mastername=fake-master",
+ "buildnumber=2",
+ "slavename=fake-buildslave",
+ "nobuildbot=True",
+ fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR),
+ fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION),
+ fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE),
+ fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE),
+ fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET),
+ },
+ Isolate: "housekeeper_skia.isolate",
+ Priority: 0.8,
+ })
return name
}
@@ -261,7 +321,7 @@
func infra(b *specs.TasksCfgBuilder, name string) string {
b.MustAddTask(name, &specs.TaskSpec{
CipdPackages: []*specs.CipdPackage{},
- Dimensions: UPLOAD_DIMENSIONS,
+ Dimensions: LINUX_GCE_DIMENSIONS,
ExtraArgs: []string{
"--workdir", "../../..", "swarm_infra",
fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO),
@@ -303,7 +363,7 @@
// test generates a Test task. Returns the name of the last task in the
// generated chain of tasks, which the Job should add as a dependency.
func test(b *specs.TasksCfgBuilder, name string, parts map[string]string, compileTaskName string, pkgs []*specs.CipdPackage) string {
- b.MustAddTask(name, &specs.TaskSpec{
+ s := &specs.TaskSpec{
CipdPackages: pkgs,
Dependencies: []string{compileTaskName},
Dimensions: swarmDimensions(parts),
@@ -323,13 +383,22 @@
},
Isolate: "test_skia.isolate",
Priority: 0.8,
- })
+ }
+ if strings.Contains(parts["extra_config"], "Valgrind") {
+ s.ExecutionTimeout = 9 * time.Hour
+ s.Expiration = 48 * time.Hour
+ s.IoTimeout = time.Hour
+ } else if strings.Contains(parts["extra_config"], "MSAN") {
+ s.ExecutionTimeout = 9 * time.Hour
+ }
+ b.MustAddTask(name, s)
+
// Upload results if necessary.
if doUpload(name) {
uploadName := fmt.Sprintf("%s%s%s", PREFIX_UPLOAD, jobNameSchema.Sep, name)
b.MustAddTask(uploadName, &specs.TaskSpec{
Dependencies: []string{name},
- Dimensions: UPLOAD_DIMENSIONS,
+ Dimensions: LINUX_GCE_DIMENSIONS,
ExtraArgs: []string{
"--workdir", "../../..", "upload_dm_results",
fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO),
@@ -355,7 +424,7 @@
// perf generates a Perf task. Returns the name of the last task in the
// generated chain of tasks, which the Job should add as a dependency.
func perf(b *specs.TasksCfgBuilder, name string, parts map[string]string, compileTaskName string, pkgs []*specs.CipdPackage) string {
- b.MustAddTask(name, &specs.TaskSpec{
+ s := &specs.TaskSpec{
CipdPackages: pkgs,
Dependencies: []string{compileTaskName},
Dimensions: swarmDimensions(parts),
@@ -375,13 +444,22 @@
},
Isolate: "perf_skia.isolate",
Priority: 0.8,
- })
+ }
+ if strings.Contains(parts["extra_config"], "Valgrind") {
+ s.ExecutionTimeout = 9 * time.Hour
+ s.Expiration = 48 * time.Hour
+ s.IoTimeout = time.Hour
+ } else if strings.Contains(parts["extra_config"], "MSAN") {
+ s.ExecutionTimeout = 9 * time.Hour
+ }
+ b.MustAddTask(name, s)
+
// Upload results if necessary.
if strings.Contains(name, "Release") && doUpload(name) {
uploadName := fmt.Sprintf("%s%s%s", PREFIX_UPLOAD, jobNameSchema.Sep, name)
b.MustAddTask(uploadName, &specs.TaskSpec{
Dependencies: []string{name},
- Dimensions: UPLOAD_DIMENSIONS,
+ Dimensions: LINUX_GCE_DIMENSIONS,
ExtraArgs: []string{
"--workdir", "../../..", "upload_nano_results",
fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO),
@@ -439,9 +517,8 @@
if err != nil {
glog.Fatal(err)
}
- // Temporarily disable the Housekeeper's compile Task, since we aren't
- // yet running that Job.
- if parts["role"] != "Housekeeper" {
+ // The InfraTests bot doesn't need a compile task.
+ if parts["role"] != "Build" && name != "Housekeeper-PerCommit-InfraTests" {
compile(b, compileTaskName, compileTaskParts)
}
@@ -456,6 +533,9 @@
b.MustGetCipdPackageFromAsset("skp"),
b.MustGetCipdPackageFromAsset("svg"),
}
+ if strings.Contains(name, "Ubuntu") && strings.Contains(name, "SAN") {
+ pkgs = append(pkgs, b.MustGetCipdPackageFromAsset("clang_linux"))
+ }
// Test bots.
if parts["role"] == "Test" {
@@ -474,27 +554,6 @@
})
}
-// getCheckoutRoot returns the path of the root of the Skia checkout, or an
-// error if it cannot be found.
-func getCheckoutRoot() string {
- cwd, err := os.Getwd()
- if err != nil {
- glog.Fatal(err)
- }
- for {
- if _, err := os.Stat(cwd); err != nil {
- glog.Fatal(err)
- }
- s, err := os.Stat(path.Join(cwd, ".git"))
- if err == nil && s.IsDir() {
- // TODO(borenet): Should we verify that this is a Skia
- // checkout and not something else?
- return cwd
- }
- cwd = filepath.Clean(path.Join(cwd, ".."))
- }
-}
-
// Regenerate the tasks.json file.
func main() {
b := specs.MustNewTasksCfgBuilder()
diff --git a/infra/bots/tasks.json b/infra/bots/tasks.json
index d2af816..fe088c3 100644
--- a/infra/bots/tasks.json
+++ b/infra/bots/tasks.json
@@ -262,6 +262,7 @@
"Build-Ubuntu-Clang-arm-Release-GN_Android"
],
"dimensions": [
+ "device_os:MOB30Q",
"device_type:sprout",
"os:Android",
"pool:Skia"
@@ -419,4 +420,4 @@
"priority": 0.8
}
}
-}
\ No newline at end of file
+}