gen_tasks.go: Implement TODOs, bring up to speed with swarm_trigger

BUG=skia:

GOLD_TRYBOT_URL= https://gold.skia.org/search?issue=4547

Change-Id: Ia22a4836732dd33f30f4f6e4e49f5b39294ec53f
Reviewed-on: https://skia-review.googlesource.com/4547
Reviewed-by: Ben Wagner <benjaminwagner@google.com>
Commit-Queue: Eric Boren <borenet@google.com>

2 files changed, 128 insertions, 68 deletions

diff --git a/infra/bots/gen_tasks.go b/infra/bots/gen_tasks.go
index 846b4c4d1c..28e6b2488f 100644
--- a/infra/bots/gen_tasks.go
+++ b/infra/bots/gen_tasks.go
@@ -13,9 +13,9 @@ import (
 	"fmt"
 	"os"
 	"path"
-	"path/filepath"
 	"sort"
 	"strings"
+	"time"
 
 	"github.com/skia-dev/glog"
 	"go.skia.org/infra/go/util"
@@ -46,8 +46,9 @@ var (
 		"Housekeeper-PerCommit-InfraTests",
 	}
 
-	// UPLOAD_DIMENSIONS are the Swarming dimensions for upload tasks.
-	UPLOAD_DIMENSIONS = []string{
+	// LINUX_GCE_DIMENSIONS are the Swarming dimensions for Linux GCE
+	// instances.
+	LINUX_GCE_DIMENSIONS = []string{
 		"cpu:x86-64-avx2",
 		"gpu:none",
 		"os:Ubuntu",
@@ -98,11 +99,6 @@ func deriveCompileTaskName(jobName string, parts map[string]string) string {
 
 // swarmDimensions generates swarming bot dimensions for the given task.
 func swarmDimensions(parts map[string]string) []string {
-	if parts["extra_config"] == "SkiaCT" {
-		return []string{
-			"pool:SkiaCT",
-		}
-	}
 	d := map[string]string{
 		"pool": POOL_SKIA,
 	}
@@ -118,28 +114,31 @@ func swarmDimensions(parts map[string]string) []string {
 		if strings.Contains(parts["os"], "Android") {
 			// For Android, the device type is a better dimension
 			// than CPU or GPU.
-			d["device_type"] = map[string]string{
-				"AndroidOne":    "sprout",
-				"GalaxyS3":      "m0", // "smdk4x12", Detected incorrectly by swarming?
-				"GalaxyS4":      "",   // TODO(borenet,kjlubick)
-				"GalaxyS7":      "heroqlteatt",
-				"NVIDIA_Shield": "foster",
-				"Nexus10":       "manta",
-				"Nexus5":        "hammerhead",
-				"Nexus6":        "shamu",
-				"Nexus6p":       "angler",
-				"Nexus7":        "grouper",
-				"Nexus7v2":      "flo",
-				"Nexus9":        "flounder",
-				"NexusPlayer":   "fugu",
+			deviceInfo := map[string][]string{
+				"AndroidOne":    {"sprout", "MOB30Q"},
+				"GalaxyS7":      {"heroqlteatt", "MMB29M"},
+				"NVIDIA_Shield": {"foster", "MRA58K"},
+				"Nexus10":       {"manta", "LMY49J"},
+				"Nexus5":        {"hammerhead", "MOB31E"},
+				"Nexus6":        {"shamu", "M"},
+				"Nexus6p":       {"angler", "NMF26C"},
+				"Nexus7":        {"grouper", "LMY47V"},
+				"Nexus7v2":      {"flo", "M"},
+				"Nexus9":        {"flounder", "NRD91D"},
+				"NexusPlayer":   {"fugu", "NRD90R"},
+				"Pixel":         {"sailfish", "NMF25"},
+				"PixelC":        {"dragon", "NMF26C"},
+				"PixelXL":       {"marlin", "NMF25"},
 			}[parts["model"]]
+			d["device_type"] = deviceInfo[0]
+			d["device_os"] = deviceInfo[1]
 		} else if strings.Contains(parts["os"], "iOS") {
 			d["device"] = map[string]string{
-				"iPad4": "iPad4,1",
+				"iPad4": "iPad5,1",
 			}[parts["model"]]
 			// TODO(borenet): Replace this hack with something
 			// better.
-			d["os"] = "iOS-9.2"
+			d["os"] = "iOS-9.3.1"
 		} else if parts["cpu_or_gpu"] == "CPU" {
 			d["gpu"] = "none"
 			d["cpu"] = map[string]string{
@@ -156,15 +155,16 @@ func swarmDimensions(parts map[string]string) []string {
 			}
 		} else {
 			d["gpu"] = map[string]string{
-				"GeForce320M": "10de:08a4",
-				"GT610":       "10de:104a",
-				"GTX550Ti":    "10de:1244",
-				"GTX660":      "10de:11c0",
-				"GTX960":      "10de:1401",
-				"HD4000":      "8086:0a2e",
-				"HD4600":      "8086:0412",
-				"HD7770":      "1002:683d",
-				"iHD530":      "8086:1912",
+				"GeForce320M":   "10de:08a4",
+				"GT610":         "10de:104a",
+				"GTX550Ti":      "10de:1244",
+				"GTX660":        "10de:11c0",
+				"GTX960":        "10de:1401",
+				"HD4000":        "8086:0a2e",
+				"HD4600":        "8086:0412",
+				"HD7770":        "1002:683d",
+				"iHD530":        "8086:1912",
+				"IntelIris6100": "8086:162b",
 			}[parts["cpu_or_gpu_value"]]
 		}
 	} else {
@@ -238,21 +238,81 @@ func compile(b *specs.TasksCfgBuilder, name string, parts map[string]string) str
 // task in the generated chain of tasks, which the Job should add as a
 // dependency.
 func recreateSKPs(b *specs.TasksCfgBuilder, name string) string {
-	// TODO
+	b.MustAddTask(name, &specs.TaskSpec{
+		CipdPackages: []*specs.CipdPackage{},
+		Dimensions:   LINUX_GCE_DIMENSIONS,
+		ExtraArgs: []string{
+			"--workdir", "../../..", "swarm_RecreateSKPs",
+			fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO),
+			fmt.Sprintf("buildername=%s", name),
+			"mastername=fake-master",
+			"buildnumber=2",
+			"slavename=fake-buildslave",
+			"nobuildbot=True",
+			fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR),
+			fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION),
+			fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE),
+			fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE),
+			fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET),
+		},
+		Isolate:  "compile_skia.isolate",
+		Priority: 0.8,
+	})
 	return name
 }
 
 // ctSKPs generates a CT SKPs task. Returns the name of the last task in the
 // generated chain of tasks, which the Job should add as a dependency.
 func ctSKPs(b *specs.TasksCfgBuilder, name string) string {
-	// TODO
+	b.MustAddTask(name, &specs.TaskSpec{
+		CipdPackages:     []*specs.CipdPackage{},
+		Dimensions:       []string{"pool:SkiaCT"},
+		ExecutionTimeout: 24 * time.Hour,
+		ExtraArgs: []string{
+			"--workdir", "../../..", "swarm_ct_skps",
+			fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO),
+			fmt.Sprintf("buildername=%s", name),
+			"mastername=fake-master",
+			"buildnumber=2",
+			"slavename=fake-buildslave",
+			"nobuildbot=True",
+			fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR),
+			fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION),
+			fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE),
+			fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE),
+			fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET),
+		},
+		IoTimeout: time.Hour,
+		Isolate:   "ct_skps_skia.isolate",
+		Priority:  0.8,
+	})
 	return name
 }
 
 // housekeeper generates a Housekeeper task. Returns the name of the last task
 // in the generated chain of tasks, which the Job should add as a dependency.
 func housekeeper(b *specs.TasksCfgBuilder, name, compileTaskName string) string {
-	// TODO
+	b.MustAddTask(name, &specs.TaskSpec{
+		CipdPackages: []*specs.CipdPackage{},
+		Dependencies: []string{compileTaskName},
+		Dimensions:   LINUX_GCE_DIMENSIONS,
+		ExtraArgs: []string{
+			"--workdir", "../../..", "swarm_housekeeper",
+			fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO),
+			fmt.Sprintf("buildername=%s", name),
+			"mastername=fake-master",
+			"buildnumber=2",
+			"slavename=fake-buildslave",
+			"nobuildbot=True",
+			fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR),
+			fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION),
+			fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE),
+			fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE),
+			fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET),
+		},
+		Isolate:  "housekeeper_skia.isolate",
+		Priority: 0.8,
+	})
 	return name
 }
 
@@ -261,7 +321,7 @@ func housekeeper(b *specs.TasksCfgBuilder, name, compileTaskName string) string
 func infra(b *specs.TasksCfgBuilder, name string) string {
 	b.MustAddTask(name, &specs.TaskSpec{
 		CipdPackages: []*specs.CipdPackage{},
-		Dimensions:   UPLOAD_DIMENSIONS,
+		Dimensions:   LINUX_GCE_DIMENSIONS,
 		ExtraArgs: []string{
 			"--workdir", "../../..", "swarm_infra",
 			fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO),
@@ -303,7 +363,7 @@ func doUpload(name string) bool {
 // test generates a Test task. Returns the name of the last task in the
 // generated chain of tasks, which the Job should add as a dependency.
 func test(b *specs.TasksCfgBuilder, name string, parts map[string]string, compileTaskName string, pkgs []*specs.CipdPackage) string {
-	b.MustAddTask(name, &specs.TaskSpec{
+	s := &specs.TaskSpec{
 		CipdPackages: pkgs,
 		Dependencies: []string{compileTaskName},
 		Dimensions:   swarmDimensions(parts),
@@ -323,13 +383,22 @@ func test(b *specs.TasksCfgBuilder, name string, parts map[string]string, compil
 		},
 		Isolate:  "test_skia.isolate",
 		Priority: 0.8,
-	})
+	}
+	if strings.Contains(parts["extra_config"], "Valgrind") {
+		s.ExecutionTimeout = 9 * time.Hour
+		s.Expiration = 48 * time.Hour
+		s.IoTimeout = time.Hour
+	} else if strings.Contains(parts["extra_config"], "MSAN") {
+		s.ExecutionTimeout = 9 * time.Hour
+	}
+	b.MustAddTask(name, s)
+
 	// Upload results if necessary.
 	if doUpload(name) {
 		uploadName := fmt.Sprintf("%s%s%s", PREFIX_UPLOAD, jobNameSchema.Sep, name)
 		b.MustAddTask(uploadName, &specs.TaskSpec{
 			Dependencies: []string{name},
-			Dimensions:   UPLOAD_DIMENSIONS,
+			Dimensions:   LINUX_GCE_DIMENSIONS,
 			ExtraArgs: []string{
 				"--workdir", "../../..", "upload_dm_results",
 				fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO),
@@ -355,7 +424,7 @@ func test(b *specs.TasksCfgBuilder, name string, parts map[string]string, compil
 // perf generates a Perf task. Returns the name of the last task in the
 // generated chain of tasks, which the Job should add as a dependency.
 func perf(b *specs.TasksCfgBuilder, name string, parts map[string]string, compileTaskName string, pkgs []*specs.CipdPackage) string {
-	b.MustAddTask(name, &specs.TaskSpec{
+	s := &specs.TaskSpec{
 		CipdPackages: pkgs,
 		Dependencies: []string{compileTaskName},
 		Dimensions:   swarmDimensions(parts),
@@ -375,13 +444,22 @@ func perf(b *specs.TasksCfgBuilder, name string, parts map[string]string, compil
 		},
 		Isolate:  "perf_skia.isolate",
 		Priority: 0.8,
-	})
+	}
+	if strings.Contains(parts["extra_config"], "Valgrind") {
+		s.ExecutionTimeout = 9 * time.Hour
+		s.Expiration = 48 * time.Hour
+		s.IoTimeout = time.Hour
+	} else if strings.Contains(parts["extra_config"], "MSAN") {
+		s.ExecutionTimeout = 9 * time.Hour
+	}
+	b.MustAddTask(name, s)
+
 	// Upload results if necessary.
 	if strings.Contains(name, "Release") && doUpload(name) {
 		uploadName := fmt.Sprintf("%s%s%s", PREFIX_UPLOAD, jobNameSchema.Sep, name)
 		b.MustAddTask(uploadName, &specs.TaskSpec{
 			Dependencies: []string{name},
-			Dimensions:   UPLOAD_DIMENSIONS,
+			Dimensions:   LINUX_GCE_DIMENSIONS,
 			ExtraArgs: []string{
 				"--workdir", "../../..", "upload_nano_results",
 				fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO),
@@ -439,9 +517,8 @@ func process(b *specs.TasksCfgBuilder, name string) {
 	if err != nil {
 		glog.Fatal(err)
 	}
-	// Temporarily disable the Housekeeper's compile Task, since we aren't
-	// yet running that Job.
-	if parts["role"] != "Housekeeper" {
+	// The InfraTests bot doesn't need a compile task.
+	if parts["role"] != "Build" && name != "Housekeeper-PerCommit-InfraTests" {
 		compile(b, compileTaskName, compileTaskParts)
 	}
 
@@ -456,6 +533,9 @@ func process(b *specs.TasksCfgBuilder, name string) {
 		b.MustGetCipdPackageFromAsset("skp"),
 		b.MustGetCipdPackageFromAsset("svg"),
 	}
+	if strings.Contains(name, "Ubuntu") && strings.Contains(name, "SAN") {
+		pkgs = append(pkgs, b.MustGetCipdPackageFromAsset("clang_linux"))
+	}
 
 	// Test bots.
 	if parts["role"] == "Test" {
@@ -474,27 +554,6 @@ func process(b *specs.TasksCfgBuilder, name string) {
 	})
 }
 
-// getCheckoutRoot returns the path of the root of the Skia checkout, or an
-// error if it cannot be found.
-func getCheckoutRoot() string {
-	cwd, err := os.Getwd()
-	if err != nil {
-		glog.Fatal(err)
-	}
-	for {
-		if _, err := os.Stat(cwd); err != nil {
-			glog.Fatal(err)
-		}
-		s, err := os.Stat(path.Join(cwd, ".git"))
-		if err == nil && s.IsDir() {
-			// TODO(borenet): Should we verify that this is a Skia
-			// checkout and not something else?
-			return cwd
-		}
-		cwd = filepath.Clean(path.Join(cwd, ".."))
-	}
-}
-
 // Regenerate the tasks.json file.
 func main() {
 	b := specs.MustNewTasksCfgBuilder()
diff --git a/infra/bots/tasks.json b/infra/bots/tasks.json
index d2af81691e..fe088c3c42 100644
--- a/infra/bots/tasks.json
+++ b/infra/bots/tasks.json
@@ -262,6 +262,7 @@
         "Build-Ubuntu-Clang-arm-Release-GN_Android"
       ],
       "dimensions": [
+        "device_os:MOB30Q",
         "device_type:sprout",
         "os:Android",
         "pool:Skia"
@@ -419,4 +420,4 @@
       "priority": 0.8
     }
   }
-}
\ No newline at end of file
+}