diff options
Diffstat (limited to 'infra/bots/gen_tasks.go')
-rw-r--r-- | infra/bots/gen_tasks.go | 193 |
1 files changed, 126 insertions, 67 deletions
diff --git a/infra/bots/gen_tasks.go b/infra/bots/gen_tasks.go index 846b4c4d1c..28e6b2488f 100644 --- a/infra/bots/gen_tasks.go +++ b/infra/bots/gen_tasks.go @@ -13,9 +13,9 @@ import ( "fmt" "os" "path" - "path/filepath" "sort" "strings" + "time" "github.com/skia-dev/glog" "go.skia.org/infra/go/util" @@ -46,8 +46,9 @@ var ( "Housekeeper-PerCommit-InfraTests", } - // UPLOAD_DIMENSIONS are the Swarming dimensions for upload tasks. - UPLOAD_DIMENSIONS = []string{ + // LINUX_GCE_DIMENSIONS are the Swarming dimensions for Linux GCE + // instances. + LINUX_GCE_DIMENSIONS = []string{ "cpu:x86-64-avx2", "gpu:none", "os:Ubuntu", @@ -98,11 +99,6 @@ func deriveCompileTaskName(jobName string, parts map[string]string) string { // swarmDimensions generates swarming bot dimensions for the given task. func swarmDimensions(parts map[string]string) []string { - if parts["extra_config"] == "SkiaCT" { - return []string{ - "pool:SkiaCT", - } - } d := map[string]string{ "pool": POOL_SKIA, } @@ -118,28 +114,31 @@ func swarmDimensions(parts map[string]string) []string { if strings.Contains(parts["os"], "Android") { // For Android, the device type is a better dimension // than CPU or GPU. - d["device_type"] = map[string]string{ - "AndroidOne": "sprout", - "GalaxyS3": "m0", // "smdk4x12", Detected incorrectly by swarming? - "GalaxyS4": "", // TODO(borenet,kjlubick) - "GalaxyS7": "heroqlteatt", - "NVIDIA_Shield": "foster", - "Nexus10": "manta", - "Nexus5": "hammerhead", - "Nexus6": "shamu", - "Nexus6p": "angler", - "Nexus7": "grouper", - "Nexus7v2": "flo", - "Nexus9": "flounder", - "NexusPlayer": "fugu", + deviceInfo := map[string][]string{ + "AndroidOne": {"sprout", "MOB30Q"}, + "GalaxyS7": {"heroqlteatt", "MMB29M"}, + "NVIDIA_Shield": {"foster", "MRA58K"}, + "Nexus10": {"manta", "LMY49J"}, + "Nexus5": {"hammerhead", "MOB31E"}, + "Nexus6": {"shamu", "M"}, + "Nexus6p": {"angler", "NMF26C"}, + "Nexus7": {"grouper", "LMY47V"}, + "Nexus7v2": {"flo", "M"}, + "Nexus9": {"flounder", "NRD91D"}, + "NexusPlayer": {"fugu", "NRD90R"}, + "Pixel": {"sailfish", "NMF25"}, + "PixelC": {"dragon", "NMF26C"}, + "PixelXL": {"marlin", "NMF25"}, }[parts["model"]] + d["device_type"] = deviceInfo[0] + d["device_os"] = deviceInfo[1] } else if strings.Contains(parts["os"], "iOS") { d["device"] = map[string]string{ - "iPad4": "iPad4,1", + "iPad4": "iPad5,1", }[parts["model"]] // TODO(borenet): Replace this hack with something // better. - d["os"] = "iOS-9.2" + d["os"] = "iOS-9.3.1" } else if parts["cpu_or_gpu"] == "CPU" { d["gpu"] = "none" d["cpu"] = map[string]string{ @@ -156,15 +155,16 @@ func swarmDimensions(parts map[string]string) []string { } } else { d["gpu"] = map[string]string{ - "GeForce320M": "10de:08a4", - "GT610": "10de:104a", - "GTX550Ti": "10de:1244", - "GTX660": "10de:11c0", - "GTX960": "10de:1401", - "HD4000": "8086:0a2e", - "HD4600": "8086:0412", - "HD7770": "1002:683d", - "iHD530": "8086:1912", + "GeForce320M": "10de:08a4", + "GT610": "10de:104a", + "GTX550Ti": "10de:1244", + "GTX660": "10de:11c0", + "GTX960": "10de:1401", + "HD4000": "8086:0a2e", + "HD4600": "8086:0412", + "HD7770": "1002:683d", + "iHD530": "8086:1912", + "IntelIris6100": "8086:162b", }[parts["cpu_or_gpu_value"]] } } else { @@ -238,21 +238,81 @@ func compile(b *specs.TasksCfgBuilder, name string, parts map[string]string) str // task in the generated chain of tasks, which the Job should add as a // dependency. func recreateSKPs(b *specs.TasksCfgBuilder, name string) string { - // TODO + b.MustAddTask(name, &specs.TaskSpec{ + CipdPackages: []*specs.CipdPackage{}, + Dimensions: LINUX_GCE_DIMENSIONS, + ExtraArgs: []string{ + "--workdir", "../../..", "swarm_RecreateSKPs", + fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO), + fmt.Sprintf("buildername=%s", name), + "mastername=fake-master", + "buildnumber=2", + "slavename=fake-buildslave", + "nobuildbot=True", + fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR), + fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION), + fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE), + fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE), + fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET), + }, + Isolate: "compile_skia.isolate", + Priority: 0.8, + }) return name } // ctSKPs generates a CT SKPs task. Returns the name of the last task in the // generated chain of tasks, which the Job should add as a dependency. func ctSKPs(b *specs.TasksCfgBuilder, name string) string { - // TODO + b.MustAddTask(name, &specs.TaskSpec{ + CipdPackages: []*specs.CipdPackage{}, + Dimensions: []string{"pool:SkiaCT"}, + ExecutionTimeout: 24 * time.Hour, + ExtraArgs: []string{ + "--workdir", "../../..", "swarm_ct_skps", + fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO), + fmt.Sprintf("buildername=%s", name), + "mastername=fake-master", + "buildnumber=2", + "slavename=fake-buildslave", + "nobuildbot=True", + fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR), + fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION), + fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE), + fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE), + fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET), + }, + IoTimeout: time.Hour, + Isolate: "ct_skps_skia.isolate", + Priority: 0.8, + }) return name } // housekeeper generates a Housekeeper task. Returns the name of the last task // in the generated chain of tasks, which the Job should add as a dependency. func housekeeper(b *specs.TasksCfgBuilder, name, compileTaskName string) string { - // TODO + b.MustAddTask(name, &specs.TaskSpec{ + CipdPackages: []*specs.CipdPackage{}, + Dependencies: []string{compileTaskName}, + Dimensions: LINUX_GCE_DIMENSIONS, + ExtraArgs: []string{ + "--workdir", "../../..", "swarm_housekeeper", + fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO), + fmt.Sprintf("buildername=%s", name), + "mastername=fake-master", + "buildnumber=2", + "slavename=fake-buildslave", + "nobuildbot=True", + fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR), + fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION), + fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE), + fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE), + fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET), + }, + Isolate: "housekeeper_skia.isolate", + Priority: 0.8, + }) return name } @@ -261,7 +321,7 @@ func housekeeper(b *specs.TasksCfgBuilder, name, compileTaskName string) string func infra(b *specs.TasksCfgBuilder, name string) string { b.MustAddTask(name, &specs.TaskSpec{ CipdPackages: []*specs.CipdPackage{}, - Dimensions: UPLOAD_DIMENSIONS, + Dimensions: LINUX_GCE_DIMENSIONS, ExtraArgs: []string{ "--workdir", "../../..", "swarm_infra", fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO), @@ -303,7 +363,7 @@ func doUpload(name string) bool { // test generates a Test task. Returns the name of the last task in the // generated chain of tasks, which the Job should add as a dependency. func test(b *specs.TasksCfgBuilder, name string, parts map[string]string, compileTaskName string, pkgs []*specs.CipdPackage) string { - b.MustAddTask(name, &specs.TaskSpec{ + s := &specs.TaskSpec{ CipdPackages: pkgs, Dependencies: []string{compileTaskName}, Dimensions: swarmDimensions(parts), @@ -323,13 +383,22 @@ func test(b *specs.TasksCfgBuilder, name string, parts map[string]string, compil }, Isolate: "test_skia.isolate", Priority: 0.8, - }) + } + if strings.Contains(parts["extra_config"], "Valgrind") { + s.ExecutionTimeout = 9 * time.Hour + s.Expiration = 48 * time.Hour + s.IoTimeout = time.Hour + } else if strings.Contains(parts["extra_config"], "MSAN") { + s.ExecutionTimeout = 9 * time.Hour + } + b.MustAddTask(name, s) + // Upload results if necessary. if doUpload(name) { uploadName := fmt.Sprintf("%s%s%s", PREFIX_UPLOAD, jobNameSchema.Sep, name) b.MustAddTask(uploadName, &specs.TaskSpec{ Dependencies: []string{name}, - Dimensions: UPLOAD_DIMENSIONS, + Dimensions: LINUX_GCE_DIMENSIONS, ExtraArgs: []string{ "--workdir", "../../..", "upload_dm_results", fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO), @@ -355,7 +424,7 @@ func test(b *specs.TasksCfgBuilder, name string, parts map[string]string, compil // perf generates a Perf task. Returns the name of the last task in the // generated chain of tasks, which the Job should add as a dependency. func perf(b *specs.TasksCfgBuilder, name string, parts map[string]string, compileTaskName string, pkgs []*specs.CipdPackage) string { - b.MustAddTask(name, &specs.TaskSpec{ + s := &specs.TaskSpec{ CipdPackages: pkgs, Dependencies: []string{compileTaskName}, Dimensions: swarmDimensions(parts), @@ -375,13 +444,22 @@ func perf(b *specs.TasksCfgBuilder, name string, parts map[string]string, compil }, Isolate: "perf_skia.isolate", Priority: 0.8, - }) + } + if strings.Contains(parts["extra_config"], "Valgrind") { + s.ExecutionTimeout = 9 * time.Hour + s.Expiration = 48 * time.Hour + s.IoTimeout = time.Hour + } else if strings.Contains(parts["extra_config"], "MSAN") { + s.ExecutionTimeout = 9 * time.Hour + } + b.MustAddTask(name, s) + // Upload results if necessary. if strings.Contains(name, "Release") && doUpload(name) { uploadName := fmt.Sprintf("%s%s%s", PREFIX_UPLOAD, jobNameSchema.Sep, name) b.MustAddTask(uploadName, &specs.TaskSpec{ Dependencies: []string{name}, - Dimensions: UPLOAD_DIMENSIONS, + Dimensions: LINUX_GCE_DIMENSIONS, ExtraArgs: []string{ "--workdir", "../../..", "upload_nano_results", fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO), @@ -439,9 +517,8 @@ func process(b *specs.TasksCfgBuilder, name string) { if err != nil { glog.Fatal(err) } - // Temporarily disable the Housekeeper's compile Task, since we aren't - // yet running that Job. - if parts["role"] != "Housekeeper" { + // The InfraTests bot doesn't need a compile task. + if parts["role"] != "Build" && name != "Housekeeper-PerCommit-InfraTests" { compile(b, compileTaskName, compileTaskParts) } @@ -456,6 +533,9 @@ func process(b *specs.TasksCfgBuilder, name string) { b.MustGetCipdPackageFromAsset("skp"), b.MustGetCipdPackageFromAsset("svg"), } + if strings.Contains(name, "Ubuntu") && strings.Contains(name, "SAN") { + pkgs = append(pkgs, b.MustGetCipdPackageFromAsset("clang_linux")) + } // Test bots. if parts["role"] == "Test" { @@ -474,27 +554,6 @@ func process(b *specs.TasksCfgBuilder, name string) { }) } -// getCheckoutRoot returns the path of the root of the Skia checkout, or an -// error if it cannot be found. -func getCheckoutRoot() string { - cwd, err := os.Getwd() - if err != nil { - glog.Fatal(err) - } - for { - if _, err := os.Stat(cwd); err != nil { - glog.Fatal(err) - } - s, err := os.Stat(path.Join(cwd, ".git")) - if err == nil && s.IsDir() { - // TODO(borenet): Should we verify that this is a Skia - // checkout and not something else? - return cwd - } - cwd = filepath.Clean(path.Join(cwd, "..")) - } -} - // Regenerate the tasks.json file. func main() { b := specs.MustNewTasksCfgBuilder() |