From e8615f969e92009551b70c85d1913202390e0b0d Mon Sep 17 00:00:00 2001 From: Eric Boren Date: Thu, 26 Apr 2018 14:23:08 -0400 Subject: [recipes] Use named caches for git and workdirs Bug: skia: Change-Id: Iaf3d190252c9483c8b9e58a3ab0305db4173cc1c Reviewed-on: https://skia-review.googlesource.com/123929 Commit-Queue: Eric Boren Reviewed-by: Ben Wagner --- infra/bots/gen_tasks.go | 48 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 41 insertions(+), 7 deletions(-) (limited to 'infra/bots/gen_tasks.go') diff --git a/infra/bots/gen_tasks.go b/infra/bots/gen_tasks.go index 950ebda530..383c455c9f 100644 --- a/infra/bots/gen_tasks.go +++ b/infra/bots/gen_tasks.go @@ -108,6 +108,24 @@ var ( // Defines the structure of job names. jobNameSchema *JobNameSchema + // Named caches used by tasks. + CACHES_GIT = []*specs.Cache{ + &specs.Cache{ + Name: "git", + Path: "cache/git", + }, + &specs.Cache{ + Name: "git_cache", + Path: "cache/git_cache", + }, + } + CACHES_WORKDIR = []*specs.Cache{ + &specs.Cache{ + Name: "work", + Path: "cache/work", + }, + } + // TODO(borenet): Roll these versions automatically! CIPD_PKGS_PYTHON = []*specs.CipdPackage{ &specs.CipdPackage{ @@ -222,6 +240,12 @@ func kitchenTask(name, recipe, isolate, serviceAccount string, dimensions []stri outputs = []string{outputDir} } return &specs.TaskSpec{ + Caches: []*specs.Cache{ + &specs.Cache{ + Name: "vpython", + Path: "cache/vpython", + }, + }, CipdPackages: cipd, Command: []string{ "./kitchen${EXECUTABLE_SUFFIX}", "cook", @@ -663,11 +687,20 @@ func getIsolatedCIPDDeps(parts map[string]string) []string { return deps } +// usesGit adds attributes to tasks which use git. +func usesGit(t *specs.TaskSpec, name string) { + t.Caches = append(t.Caches, CACHES_GIT...) + if !strings.Contains(name, "NoDEPS") { + t.Caches = append(t.Caches, CACHES_WORKDIR...) + } + t.CipdPackages = append(t.CipdPackages, CIPD_PKGS_GIT...) +} + // compile generates a compile task. Returns the name of the last task in the // generated chain of tasks, which the Job should add as a dependency. func compile(b *specs.TasksCfgBuilder, name string, parts map[string]string) string { task := kitchenTask(name, "compile", "swarm_recipe.isolate", SERVICE_ACCOUNT_COMPILE, swarmDimensions(parts), nil, OUTPUT_BUILD) - task.CipdPackages = append(task.CipdPackages, CIPD_PKGS_GIT...) + usesGit(task, name) // Android bots require a toolchain. if strings.Contains(name, "Android") { @@ -777,7 +810,7 @@ func ctSKPs(b *specs.TasksCfgBuilder, name string) string { fmt.Sprintf("os:%s", DEFAULT_OS_LINUX_GCE), } task := kitchenTask(name, "ct_skps", "skia_repo.isolate", SERVICE_ACCOUNT_CT_SKPS, dims, nil, OUTPUT_NONE) - task.CipdPackages = append(task.CipdPackages, CIPD_PKGS_GIT...) + usesGit(task, name) task.CipdPackages = append(task.CipdPackages, b.MustGetCipdPackageFromAsset("clang_linux")) task.ExecutionTimeout = 24 * time.Hour task.IoTimeout = 24 * time.Hour @@ -798,7 +831,8 @@ func checkGeneratedFiles(b *specs.TasksCfgBuilder, name string) string { // in the generated chain of tasks, which the Job should add as a dependency. func housekeeper(b *specs.TasksCfgBuilder, name, compileTaskName string) string { task := kitchenTask(name, "housekeeper", "swarm_recipe.isolate", SERVICE_ACCOUNT_HOUSEKEEPER, linuxGceDimensions(), nil, OUTPUT_NONE) - task.CipdPackages = append(task.CipdPackages, CIPD_PKGS_GIT...) + usesGit(task, name) + task.Caches = append(task.Caches, CACHES_WORKDIR...) task.CipdPackages = append(task.CipdPackages, b.MustGetCipdPackageFromAsset("go")) task.Dependencies = append(task.Dependencies, compileTaskName) b.MustAddTask(name, task) @@ -834,7 +868,7 @@ func androidFrameworkCompile(b *specs.TasksCfgBuilder, name string) string { // generated chain of tasks, which the Job should add as a dependency. func infra(b *specs.TasksCfgBuilder, name string) string { task := kitchenTask(name, "infra", "swarm_recipe.isolate", SERVICE_ACCOUNT_COMPILE, linuxGceDimensions(), nil, OUTPUT_NONE) - task.CipdPackages = append(task.CipdPackages, CIPD_PKGS_GIT...) + usesGit(task, name) task.CipdPackages = append(task.CipdPackages, b.MustGetCipdPackageFromAsset("go")) b.MustAddTask(name, task) return name @@ -852,7 +886,7 @@ func getParentRevisionName(compileTaskName string, parts map[string]string) stri // generated chain of tasks, which the Job should add as a dependency. func calmbench(b *specs.TasksCfgBuilder, name string, parts map[string]string, compileTaskName, compileParentName string) string { task := kitchenTask(name, "calmbench", "calmbench.isolate", "", swarmDimensions(parts), nil, OUTPUT_PERF) - task.CipdPackages = append(task.CipdPackages, CIPD_PKGS_GIT...) + usesGit(task, name) task.CipdPackages = append(task.CipdPackages, b.MustGetCipdPackageFromAsset("go")) task.Dependencies = append(task.Dependencies, compileTaskName, compileParentName, ISOLATE_SKP_NAME, ISOLATE_SVG_NAME) task.MaxAttempts = 1 @@ -984,7 +1018,7 @@ func coverage(b *specs.TasksCfgBuilder, name string, parts map[string]string, co "gs_bucket": CONFIG.GsBucketCoverage, } uploadTask := kitchenTask(uploadName, "upload_coverage_results", "swarm_recipe.isolate", SERVICE_ACCOUNT_UPLOAD_COVERAGE, linuxGceDimensions(), extraProps, OUTPUT_NONE) - uploadTask.CipdPackages = append(uploadTask.CipdPackages, CIPD_PKGS_GIT...) + usesGit(uploadTask, uploadName) uploadTask.CipdPackages = append(uploadTask.CipdPackages, CIPD_PKGS_GSUTIL...) // We need clang_linux to get access to the llvm-profdata and llvm-cov binaries // which are used to deal with the raw coverage data output by the Test step. @@ -1081,7 +1115,7 @@ func presubmit(b *specs.TasksCfgBuilder, name string) string { } replaceArg("-repository", "https://chromium.googlesource.com/chromium/tools/build") replaceArg("-revision", "HEAD") - task.CipdPackages = append(task.CipdPackages, CIPD_PKGS_GIT...) + usesGit(task, name) task.Dependencies = []string{} // No bundled recipes for this one. b.MustAddTask(name, task) return name -- cgit v1.2.3