Reland "[recipes] Use named caches for git and workdirs"

This reverts commit 01d3eb64e7b11b22cf206758f01b06a7ae0bbb20.

Bug: skia:
Change-Id: Ic05b89009eb59a231488bef382dd232a50d2b00f
Reviewed-on: https://skia-review.googlesource.com/124260
Reviewed-by: Ben Wagner <benjaminwagner@google.com>
Commit-Queue: Eric Boren <borenet@google.com>

1 files changed, 67 insertions, 34 deletions

diff --git a/infra/bots/gen_tasks.go b/infra/bots/gen_tasks.go
index 2f3e97e38e..920dc3a84e 100644
--- a/infra/bots/gen_tasks.go
+++ b/infra/bots/gen_tasks.go
@@ -108,6 +108,24 @@ var (
 	// Defines the structure of job names.
 	jobNameSchema *JobNameSchema
 
+	// Named caches used by tasks.
+	CACHES_GIT = []*specs.Cache{
+		&specs.Cache{
+			Name: "git",
+			Path: "cache/git",
+		},
+		&specs.Cache{
+			Name: "git_cache",
+			Path: "cache/git_cache",
+		},
+	}
+	CACHES_WORKDIR = []*specs.Cache{
+		&specs.Cache{
+			Name: "work",
+			Path: "cache/work",
+		},
+	}
+
 	// TODO(borenet): Roll these versions automatically!
 	CIPD_PKGS_PYTHON = []*specs.CipdPackage{
 		&specs.CipdPackage{
@@ -221,7 +239,13 @@ func kitchenTask(name, recipe, isolate, serviceAccount string, dimensions []stri
 	if outputDir != OUTPUT_NONE {
 		outputs = []string{outputDir}
 	}
-	return &specs.TaskSpec{
+	task := &specs.TaskSpec{
+		Caches: []*specs.Cache{
+			&specs.Cache{
+				Name: "vpython",
+				Path: "cache/vpython",
+			},
+		},
 		CipdPackages: cipd,
 		Command: []string{
 			"./kitchen${EXECUTABLE_SUFFIX}", "cook",
@@ -260,6 +284,8 @@ func kitchenTask(name, recipe, isolate, serviceAccount string, dimensions []stri
 		Priority:       0.8,
 		ServiceAccount: serviceAccount,
 	}
+	timeout(task, time.Hour)
+	return task
 }
 
 // internalHardwareLabel returns the internal ID for the bot, if any.
@@ -667,11 +693,26 @@ func getIsolatedCIPDDeps(parts map[string]string) []string {
 	return deps
 }
 
+// usesGit adds attributes to tasks which use git.
+func usesGit(t *specs.TaskSpec, name string) {
+	t.Caches = append(t.Caches, CACHES_GIT...)
+	if !strings.Contains(name, "NoDEPS") {
+		t.Caches = append(t.Caches, CACHES_WORKDIR...)
+	}
+	t.CipdPackages = append(t.CipdPackages, CIPD_PKGS_GIT...)
+}
+
+// timeout sets the timeout(s) for this task.
+func timeout(task *specs.TaskSpec, timeout time.Duration) {
+	task.ExecutionTimeout = timeout
+	task.IoTimeout = timeout // With kitchen, step logs don't count toward IoTimeout.
+}
+
 // compile generates a compile task. Returns the name of the last task in the
 // generated chain of tasks, which the Job should add as a dependency.
 func compile(b *specs.TasksCfgBuilder, name string, parts map[string]string) string {
 	task := kitchenTask(name, "compile", "swarm_recipe.isolate", SERVICE_ACCOUNT_COMPILE, swarmDimensions(parts), nil, OUTPUT_BUILD)
-	task.CipdPackages = append(task.CipdPackages, CIPD_PKGS_GIT...)
+	usesGit(task, name)
 
 	// Android bots require a toolchain.
 	if strings.Contains(name, "Android") {
@@ -701,6 +742,8 @@ func compile(b *specs.TasksCfgBuilder, name string, parts map[string]string) str
 			task.CipdPackages = append(task.CipdPackages, b.MustGetCipdPackageFromAsset("armhf_sysroot"))
 			task.CipdPackages = append(task.CipdPackages, b.MustGetCipdPackageFromAsset("chromebook_arm_gles"))
 		}
+	} else if strings.Contains(name, "CommandBuffer") {
+		timeout(task, 2*time.Hour)
 	} else if strings.Contains(name, "Debian") {
 		if strings.Contains(name, "Clang") {
 			task.CipdPackages = append(task.CipdPackages, b.MustGetCipdPackageFromAsset("clang_linux"))
@@ -767,8 +810,7 @@ func recreateSKPs(b *specs.TasksCfgBuilder, name string) string {
 	task := kitchenTask(name, "recreate_skps", "swarm_recipe.isolate", SERVICE_ACCOUNT_RECREATE_SKPS, linuxGceDimensions(), nil, OUTPUT_NONE)
 	task.CipdPackages = append(task.CipdPackages, CIPD_PKGS_GIT...)
 	task.CipdPackages = append(task.CipdPackages, b.MustGetCipdPackageFromAsset("go"))
-	task.ExecutionTimeout = 4 * time.Hour
-	task.IoTimeout = 4 * time.Hour // With kitchen, step logs don't count toward IoTimeout.
+	timeout(task, 4*time.Hour)
 	b.MustAddTask(name, task)
 	return name
 }
@@ -781,10 +823,9 @@ func ctSKPs(b *specs.TasksCfgBuilder, name string) string {
 		fmt.Sprintf("os:%s", DEFAULT_OS_LINUX_GCE),
 	}
 	task := kitchenTask(name, "ct_skps", "skia_repo.isolate", SERVICE_ACCOUNT_CT_SKPS, dims, nil, OUTPUT_NONE)
-	task.CipdPackages = append(task.CipdPackages, CIPD_PKGS_GIT...)
+	usesGit(task, name)
 	task.CipdPackages = append(task.CipdPackages, b.MustGetCipdPackageFromAsset("clang_linux"))
-	task.ExecutionTimeout = 24 * time.Hour
-	task.IoTimeout = 24 * time.Hour
+	timeout(task, 24*time.Hour)
 	task.MaxAttempts = 1
 	b.MustAddTask(name, task)
 	return name
@@ -794,6 +835,7 @@ func ctSKPs(b *specs.TasksCfgBuilder, name string) string {
 // by hand.
 func checkGeneratedFiles(b *specs.TasksCfgBuilder, name string) string {
 	task := kitchenTask(name, "check_generated_files", "swarm_recipe.isolate", SERVICE_ACCOUNT_COMPILE, linuxGceDimensions(), nil, OUTPUT_NONE)
+	task.Caches = append(task.Caches, CACHES_WORKDIR...)
 	b.MustAddTask(name, task)
 	return name
 }
@@ -802,7 +844,7 @@ func checkGeneratedFiles(b *specs.TasksCfgBuilder, name string) string {
 // in the generated chain of tasks, which the Job should add as a dependency.
 func housekeeper(b *specs.TasksCfgBuilder, name, compileTaskName string) string {
 	task := kitchenTask(name, "housekeeper", "swarm_recipe.isolate", SERVICE_ACCOUNT_HOUSEKEEPER, linuxGceDimensions(), nil, OUTPUT_NONE)
-	task.CipdPackages = append(task.CipdPackages, CIPD_PKGS_GIT...)
+	usesGit(task, name)
 	task.CipdPackages = append(task.CipdPackages, b.MustGetCipdPackageFromAsset("go"))
 	task.Dependencies = append(task.Dependencies, compileTaskName)
 	b.MustAddTask(name, task)
@@ -813,11 +855,11 @@ func housekeeper(b *specs.TasksCfgBuilder, name, compileTaskName string) string
 // in the generated chain of tasks, which the Job should add as a dependency.
 func bookmaker(b *specs.TasksCfgBuilder, name, compileTaskName string) string {
 	task := kitchenTask(name, "bookmaker", "swarm_recipe.isolate", SERVICE_ACCOUNT_BOOKMAKER, linuxGceDimensions(), nil, OUTPUT_NONE)
+	task.Caches = append(task.Caches, CACHES_WORKDIR...)
 	task.CipdPackages = append(task.CipdPackages, CIPD_PKGS_GIT...)
 	task.CipdPackages = append(task.CipdPackages, b.MustGetCipdPackageFromAsset("go"))
 	task.Dependencies = append(task.Dependencies, compileTaskName)
-	task.ExecutionTimeout = 2 * time.Hour
-	task.IoTimeout = 2 * time.Hour
+	timeout(task, 2*time.Hour)
 	b.MustAddTask(name, task)
 	return name
 }
@@ -827,9 +869,8 @@ func bookmaker(b *specs.TasksCfgBuilder, name, compileTaskName string) string {
 // should add as a dependency.
 func androidFrameworkCompile(b *specs.TasksCfgBuilder, name string) string {
 	task := kitchenTask(name, "android_compile", "swarm_recipe.isolate", SERVICE_ACCOUNT_COMPILE, linuxGceDimensions(), nil, OUTPUT_NONE)
-	task.ExecutionTimeout = 1 * time.Hour
-	task.IoTimeout = 1 * time.Hour
 	task.MaxAttempts = 1
+	timeout(task, time.Hour)
 	b.MustAddTask(name, task)
 	return name
 }
@@ -838,7 +879,7 @@ func androidFrameworkCompile(b *specs.TasksCfgBuilder, name string) string {
 // generated chain of tasks, which the Job should add as a dependency.
 func infra(b *specs.TasksCfgBuilder, name string) string {
 	task := kitchenTask(name, "infra", "swarm_recipe.isolate", SERVICE_ACCOUNT_COMPILE, linuxGceDimensions(), nil, OUTPUT_NONE)
-	task.CipdPackages = append(task.CipdPackages, CIPD_PKGS_GIT...)
+	usesGit(task, name)
 	task.CipdPackages = append(task.CipdPackages, b.MustGetCipdPackageFromAsset("go"))
 	b.MustAddTask(name, task)
 	return name
@@ -856,7 +897,7 @@ func getParentRevisionName(compileTaskName string, parts map[string]string) stri
 // generated chain of tasks, which the Job should add as a dependency.
 func calmbench(b *specs.TasksCfgBuilder, name string, parts map[string]string, compileTaskName, compileParentName string) string {
 	task := kitchenTask(name, "calmbench", "calmbench.isolate", "", swarmDimensions(parts), nil, OUTPUT_PERF)
-	task.CipdPackages = append(task.CipdPackages, CIPD_PKGS_GIT...)
+	usesGit(task, name)
 	task.CipdPackages = append(task.CipdPackages, b.MustGetCipdPackageFromAsset("go"))
 	task.Dependencies = append(task.Dependencies, compileTaskName, compileParentName, ISOLATE_SKP_NAME, ISOLATE_SVG_NAME)
 	task.MaxAttempts = 1
@@ -916,23 +957,19 @@ func test(b *specs.TasksCfgBuilder, name string, parts map[string]string, compil
 	if deps := getIsolatedCIPDDeps(parts); len(deps) > 0 {
 		task.Dependencies = append(task.Dependencies, deps...)
 	}
-	task.ExecutionTimeout = 4 * time.Hour
 	task.Expiration = 20 * time.Hour
-	task.IoTimeout = 4 * time.Hour
 	task.MaxAttempts = 1
+	timeout(task, 4*time.Hour)
 	if strings.Contains(parts["extra_config"], "Valgrind") {
-		task.ExecutionTimeout = 9 * time.Hour
+		timeout(task, 9*time.Hour)
 		task.Expiration = 48 * time.Hour
-		task.IoTimeout = 9 * time.Hour
 		task.CipdPackages = append(task.CipdPackages, b.MustGetCipdPackageFromAsset("valgrind"))
 		task.Dimensions = append(task.Dimensions, "valgrind:1")
 	} else if strings.Contains(parts["extra_config"], "MSAN") {
-		task.ExecutionTimeout = 9 * time.Hour
-		task.IoTimeout = 9 * time.Hour
+		timeout(task, 9*time.Hour)
 	} else if parts["arch"] == "x86" && parts["configuration"] == "Debug" {
 		// skia:6737
-		task.ExecutionTimeout = 6 * time.Hour
-		task.IoTimeout = 6 * time.Hour
+		timeout(task, 6*time.Hour)
 	}
 	b.MustAddTask(name, task)
 
@@ -972,10 +1009,10 @@ func coverage(b *specs.TasksCfgBuilder, name string, parts map[string]string, co
 		task := kitchenTask(n, "test", "test_skia_bundled.isolate", "", swarmDimensions(parts), nil, OUTPUT_COVERAGE)
 		task.CipdPackages = append(task.CipdPackages, pkgs...)
 		task.Dependencies = append(task.Dependencies, compileTaskName)
-		task.ExecutionTimeout = 4 * time.Hour
+
 		task.Expiration = 20 * time.Hour
-		task.IoTimeout = 4 * time.Hour
 		task.MaxAttempts = 1
+		timeout(task, 4*time.Hour)
 		if deps := getIsolatedCIPDDeps(parts); len(deps) > 0 {
 			task.Dependencies = append(task.Dependencies, deps...)
 		}
@@ -988,7 +1025,7 @@ func coverage(b *specs.TasksCfgBuilder, name string, parts map[string]string, co
 		"gs_bucket": CONFIG.GsBucketCoverage,
 	}
 	uploadTask := kitchenTask(uploadName, "upload_coverage_results", "swarm_recipe.isolate", SERVICE_ACCOUNT_UPLOAD_COVERAGE, linuxGceDimensions(), extraProps, OUTPUT_NONE)
-	uploadTask.CipdPackages = append(uploadTask.CipdPackages, CIPD_PKGS_GIT...)
+	usesGit(uploadTask, uploadName)
 	uploadTask.CipdPackages = append(uploadTask.CipdPackages, CIPD_PKGS_GSUTIL...)
 	// We need clang_linux to get access to the llvm-profdata and llvm-cov binaries
 	// which are used to deal with the raw coverage data output by the Test step.
@@ -1016,27 +1053,23 @@ func perf(b *specs.TasksCfgBuilder, name string, parts map[string]string, compil
 	task := kitchenTask(name, recipe, isolate, "", swarmDimensions(parts), nil, OUTPUT_PERF)
 	task.CipdPackages = append(task.CipdPackages, pkgs...)
 	task.Dependencies = append(task.Dependencies, compileTaskName)
-	task.ExecutionTimeout = 4 * time.Hour
 	task.Expiration = 20 * time.Hour
-	task.IoTimeout = 4 * time.Hour
 	task.MaxAttempts = 1
+	timeout(task, 4*time.Hour)
 	if deps := getIsolatedCIPDDeps(parts); len(deps) > 0 {
 		task.Dependencies = append(task.Dependencies, deps...)
 	}
 
 	if strings.Contains(parts["extra_config"], "Valgrind") {
-		task.ExecutionTimeout = 9 * time.Hour
+		timeout(task, 9*time.Hour)
 		task.Expiration = 48 * time.Hour
-		task.IoTimeout = 9 * time.Hour
 		task.CipdPackages = append(task.CipdPackages, b.MustGetCipdPackageFromAsset("valgrind"))
 		task.Dimensions = append(task.Dimensions, "valgrind:1")
 	} else if strings.Contains(parts["extra_config"], "MSAN") {
-		task.ExecutionTimeout = 9 * time.Hour
-		task.IoTimeout = 9 * time.Hour
+		timeout(task, 9*time.Hour)
 	} else if parts["arch"] == "x86" && parts["configuration"] == "Debug" {
 		// skia:6737
-		task.ExecutionTimeout = 6 * time.Hour
-		task.IoTimeout = 6 * time.Hour
+		timeout(task, 6*time.Hour)
 	}
 	iid := internalHardwareLabel(parts)
 	if iid != nil {
@@ -1085,7 +1118,7 @@ func presubmit(b *specs.TasksCfgBuilder, name string) string {
 	}
 	replaceArg("-repository", "https://chromium.googlesource.com/chromium/tools/build")
 	replaceArg("-revision", "HEAD")
-	task.CipdPackages = append(task.CipdPackages, CIPD_PKGS_GIT...)
+	usesGit(task, name)
 	task.Dependencies = []string{} // No bundled recipes for this one.
 	b.MustAddTask(name, task)
 	return name