Add Linux CPU Coverage Bot

This simply uploads the results of an LLVM coverage to GCS for later ingestion/display. Bug: skia:7080 Change-Id: I7dcfa2307a239734a614990aca899ea37129126b Reviewed-on: https://skia-review.googlesource.com/53880 Commit-Queue: Kevin Lubick <kjlubick@google.com> Reviewed-by: Eric Boren <borenet@google.com>
author: Kevin Lubick <kjlubick@google.com> 2017-10-09 15:26:19 -0400
committer: Skia Commit-Bot <skia-commit-bot@chromium.org> 2017-10-09 19:58:34 +0000
commit: c795a4c8862dbab914561fadf7a3567c55362ae4 (patch)
tree: 725e0fc4209029005230fdd4db48035d3aebd52a /infra
parent: df007e1a7ae808ad41eb2bd01f6a658c5b438285 (diff)
31 files changed, 2148 insertions, 54 deletions
diff --git a/infra/bots/README.recipes.md b/infra/bots/README.recipes.md
index 5335690334..58753dd0f7 100644
--- a/infra/bots/README.recipes.md
+++ b/infra/bots/README.recipes.md
@@ -9,6 +9,7 @@
   * [env](#recipe_modules-env)
   * [flavor](#recipe_modules-flavor)
   * [git](#recipe_modules-git)
+  * [gsutil](#recipe_modules-gsutil)
   * [infra](#recipe_modules-infra)
   * [isolate](#recipe_modules-isolate)
   * [run](#recipe_modules-run)
@@ -28,6 +29,7 @@
   * [env:examples/full](#recipes-env_examples_full)
   * [flavor:examples/full](#recipes-flavor_examples_full)
   * [git:examples/full](#recipes-git_examples_full)
+  * [gsutil:examples/full](#recipes-gsutil_examples_full)
   * [housekeeper](#recipes-housekeeper)
   * [infra](#recipes-infra)
   * [infra:examples/full](#recipes-infra_examples_full)
@@ -45,6 +47,7 @@
   * [swarming_client:examples/full](#recipes-swarming_client_examples_full)
   * [test](#recipes-test)
   * [update_meta_config](#recipes-update_meta_config) &mdash; Recipe for the Bot that updates meta config.
+  * [upload_coverage_results](#recipes-upload_coverage_results)
   * [upload_dm_results](#recipes-upload_dm_results)
   * [upload_nano_results](#recipes-upload_nano_results)
   * [vars:examples/full](#recipes-vars_examples_full)
@@ -153,27 +156,45 @@ Add Git to PATH
 
 Requires the infra/git and infra/tools/git CIPD packages to be installed
 in the 'git' relative path.
+### *recipe_modules* / [gsutil](/infra/bots/recipe_modules/gsutil)
+
+[DEPS](/infra/bots/recipe_modules/gsutil/__init__.py#5): [recipe\_engine/context][recipe_engine/recipe_modules/context], [recipe\_engine/step][recipe_engine/recipe_modules/step], [run](#recipe_modules-run), [vars](#recipe_modules-vars)
+
+#### **class [GSUtilApi](/infra/bots/recipe_modules/gsutil/api.py#10)([RecipeApi][recipe_engine/wkt/RecipeApi]):**
+
+&mdash; **def [cp](/infra/bots/recipe_modules/gsutil/api.py#11)(self, name, src, dst, extra_args=None):**
+
+Attempt to upload or download files to/from Google Cloud Storage (GCS).
+
+Args:
+  name: string. Will be used to fill out the step name.
+  src: string. Absolute path for a local file or gcs file (e.g. gs://...)
+  dst: string. Same as src.
+  extra_args: optional list of args to be passed to gsutil. e.g. [-Z] asks
+    all files be compressed with gzip after upload and before download.
+
+If the operation fails, it will be retried multiple times.
 ### *recipe_modules* / [infra](/infra/bots/recipe_modules/infra)
 
 [DEPS](/infra/bots/recipe_modules/infra/__init__.py#5): [recipe\_engine/context][recipe_engine/recipe_modules/context], [recipe\_engine/step][recipe_engine/recipe_modules/step], [run](#recipe_modules-run), [vars](#recipe_modules-vars)
 
-#### **class [InfraApi](/infra/bots/recipe_modules/infra/api.py#13)([RecipeApi][recipe_engine/wkt/RecipeApi]):**
+#### **class [InfraApi](/infra/bots/recipe_modules/infra/api.py#14)([RecipeApi][recipe_engine/wkt/RecipeApi]):**
 
-&emsp; **@property**<br>&mdash; **def [go\_bin](/infra/bots/recipe_modules/infra/api.py#18)(self):**
+&emsp; **@property**<br>&mdash; **def [go\_bin](/infra/bots/recipe_modules/infra/api.py#19)(self):**
 
-&emsp; **@property**<br>&mdash; **def [go\_env](/infra/bots/recipe_modules/infra/api.py#26)(self):**
+&emsp; **@property**<br>&mdash; **def [go\_env](/infra/bots/recipe_modules/infra/api.py#27)(self):**
 
-&emsp; **@property**<br>&mdash; **def [go\_exe](/infra/bots/recipe_modules/infra/api.py#22)(self):**
+&emsp; **@property**<br>&mdash; **def [go\_exe](/infra/bots/recipe_modules/infra/api.py#23)(self):**
 
-&mdash; **def [go\_version](/infra/bots/recipe_modules/infra/api.py#38)(self):**
+&mdash; **def [go\_version](/infra/bots/recipe_modules/infra/api.py#39)(self):**
 
 Print the Go version.
 
-&emsp; **@property**<br>&mdash; **def [gopath](/infra/bots/recipe_modules/infra/api.py#34)(self):**
+&emsp; **@property**<br>&mdash; **def [gopath](/infra/bots/recipe_modules/infra/api.py#35)(self):**
 
-&emsp; **@property**<br>&mdash; **def [goroot](/infra/bots/recipe_modules/infra/api.py#14)(self):**
+&emsp; **@property**<br>&mdash; **def [goroot](/infra/bots/recipe_modules/infra/api.py#15)(self):**
 
-&mdash; **def [update\_go\_deps](/infra/bots/recipe_modules/infra/api.py#52)(self):**
+&mdash; **def [update\_go\_deps](/infra/bots/recipe_modules/infra/api.py#53)(self):**
 
 Attempt to update go dependencies.
 
@@ -634,7 +655,7 @@ Does nothing if script's version is already known.
 
 #### **class [SkiaVarsApi](/infra/bots/recipe_modules/vars/api.py#16)([RecipeApi][recipe_engine/wkt/RecipeApi]):**
 
-&emsp; **@property**<br>&mdash; **def [is\_linux](/infra/bots/recipe_modules/vars/api.py#151)(self):**
+&emsp; **@property**<br>&mdash; **def [is\_linux](/infra/bots/recipe_modules/vars/api.py#158)(self):**
 
 &mdash; **def [make\_path](/infra/bots/recipe_modules/vars/api.py#18)(self, \*path):**
 
@@ -644,13 +665,13 @@ Return a Path object for the given path.
 
 Prepare the variables.
 
-&emsp; **@property**<br>&mdash; **def [swarming\_bot\_id](/infra/bots/recipe_modules/vars/api.py#193)(self):**
+&emsp; **@property**<br>&mdash; **def [swarming\_bot\_id](/infra/bots/recipe_modules/vars/api.py#200)(self):**
 
-&emsp; **@property**<br>&mdash; **def [swarming\_task\_id](/infra/bots/recipe_modules/vars/api.py#204)(self):**
+&emsp; **@property**<br>&mdash; **def [swarming\_task\_id](/infra/bots/recipe_modules/vars/api.py#211)(self):**
 
-&emsp; **@property**<br>&mdash; **def [upload\_dm\_results](/infra/bots/recipe_modules/vars/api.py#155)(self):**
+&emsp; **@property**<br>&mdash; **def [upload\_dm\_results](/infra/bots/recipe_modules/vars/api.py#162)(self):**
 
-&emsp; **@property**<br>&mdash; **def [upload\_perf\_results](/infra/bots/recipe_modules/vars/api.py#173)(self):**
+&emsp; **@property**<br>&mdash; **def [upload\_perf\_results](/infra/bots/recipe_modules/vars/api.py#180)(self):**
 ## Recipes
 
 ### *recipes* / [builder\_name\_schema:examples/full](/infra/bots/recipe_modules/builder_name_schema/examples/full.py)
@@ -711,6 +732,11 @@ Return a list of targets to build, depending on the builder type.
 [DEPS](/infra/bots/recipe_modules/git/examples/full.py#6): [recipe\_engine/step][recipe_engine/recipe_modules/step], [git](#recipe_modules-git)
 
 &mdash; **def [RunSteps](/infra/bots/recipe_modules/git/examples/full.py#12)(api):**
+### *recipes* / [gsutil:examples/full](/infra/bots/recipe_modules/gsutil/examples/full.py)
+
+[DEPS](/infra/bots/recipe_modules/gsutil/examples/full.py#9): [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/python][recipe_engine/recipe_modules/python], [recipe\_engine/step][recipe_engine/recipe_modules/step], [gsutil](#recipe_modules-gsutil), [run](#recipe_modules-run), [vars](#recipe_modules-vars)
+
+&mdash; **def [RunSteps](/infra/bots/recipe_modules/gsutil/examples/full.py#20)(api):**
 ### *recipes* / [housekeeper](/infra/bots/recipes/housekeeper.py)
 
 [DEPS](/infra/bots/recipes/housekeeper.py#8): [depot\_tools/bot\_update][depot_tools/recipe_modules/bot_update], [recipe\_engine/context][recipe_engine/recipe_modules/context], [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/python][recipe_engine/recipe_modules/python], [recipe\_engine/step][recipe_engine/recipe_modules/step], [core](#recipe_modules-core), [run](#recipe_modules-run), [vars](#recipe_modules-vars)
@@ -824,13 +850,16 @@ Run the DM test.
 Recipe for the Bot that updates meta config.
 
 &mdash; **def [RunSteps](/infra/bots/recipes/update_meta_config.py#38)(api):**
-### *recipes* / [upload\_dm\_results](/infra/bots/recipes/upload_dm_results.py)
+### *recipes* / [upload\_coverage\_results](/infra/bots/recipes/upload_coverage_results.py)
+
+[DEPS](/infra/bots/recipes/upload_coverage_results.py#12): [recipe\_engine/json][recipe_engine/recipe_modules/json], [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/step][recipe_engine/recipe_modules/step], [recipe\_engine/time][recipe_engine/recipe_modules/time], [gsutil](#recipe_modules-gsutil)
 
-[DEPS](/infra/bots/recipes/upload_dm_results.py#12): [recipe\_engine/file][recipe_engine/recipe_modules/file], [recipe\_engine/json][recipe_engine/recipe_modules/json], [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/step][recipe_engine/recipe_modules/step], [recipe\_engine/time][recipe_engine/recipe_modules/time]
+&mdash; **def [RunSteps](/infra/bots/recipes/upload_coverage_results.py#33)(api):**
+### *recipes* / [upload\_dm\_results](/infra/bots/recipes/upload_dm_results.py)
 
-&mdash; **def [RunSteps](/infra/bots/recipes/upload_dm_results.py#47)(api):**
+[DEPS](/infra/bots/recipes/upload_dm_results.py#12): [recipe\_engine/file][recipe_engine/recipe_modules/file], [recipe\_engine/json][recipe_engine/recipe_modules/json], [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/step][recipe_engine/recipe_modules/step], [recipe\_engine/time][recipe_engine/recipe_modules/time], [gsutil](#recipe_modules-gsutil)
 
-&mdash; **def [cp](/infra/bots/recipes/upload_dm_results.py#28)(api, name, src, dst, extra_args=None):**
+&mdash; **def [RunSteps](/infra/bots/recipes/upload_dm_results.py#28)(api):**
 ### *recipes* / [upload\_nano\_results](/infra/bots/recipes/upload_nano_results.py)
 
 [DEPS](/infra/bots/recipes/upload_nano_results.py#9): [recipe\_engine/context][recipe_engine/recipe_modules/context], [recipe\_engine/file][recipe_engine/recipe_modules/file], [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/step][recipe_engine/recipe_modules/step], [recipe\_engine/time][recipe_engine/recipe_modules/time]
diff --git a/infra/bots/cfg.json b/infra/bots/cfg.json
index dbc3914bb5..e3a5d48ba8 100644
--- a/infra/bots/cfg.json
+++ b/infra/bots/cfg.json
@@ -1,6 +1,7 @@
 {
   "gs_bucket_gm": "skia-infra-gm",
   "gs_bucket_nano": "skia-perf",
+  "gs_bucket_coverage": "skia-coverage",
   "pool": "Skia",
   "no_upload": [
     "ASAN",
diff --git a/infra/bots/gen_tasks.go b/infra/bots/gen_tasks.go
index 53e0ac20fe..436b457c95 100644
--- a/infra/bots/gen_tasks.go
+++ b/infra/bots/gen_tasks.go
@@ -50,10 +50,11 @@ var (
 
 	// General configuration information.
 	CONFIG struct {
-		GsBucketGm   string   `json:"gs_bucket_gm"`
-		GsBucketNano string   `json:"gs_bucket_nano"`
-		NoUpload     []string `json:"no_upload"`
-		Pool         string   `json:"pool"`
+		GsBucketCoverage string   `json:"gs_bucket_coverage"`
+		GsBucketGm       string   `json:"gs_bucket_gm"`
+		GsBucketNano     string   `json:"gs_bucket_nano"`
+		NoUpload         []string `json:"no_upload"`
+		Pool             string   `json:"pool"`
 	}
 
 	// alternateSwarmDimensions can be set in an init function to override the default swarming bot
@@ -743,7 +744,8 @@ func test(b *specs.TasksCfgBuilder, name string, parts map[string]string, compil
 	}
 	b.MustAddTask(name, s)
 
-	// Upload results if necessary.
+	// Upload results if necessary. TODO(kjlubick): If we do coverage analysis at the same
+	// time as normal tests (which would be nice), cfg.json needs to have Coverage removed.
 	if doUpload(name) {
 		uploadName := fmt.Sprintf("%s%s%s", PREFIX_UPLOAD, jobNameSchema.Sep, name)
 		b.MustAddTask(uploadName, &specs.TaskSpec{
@@ -765,7 +767,38 @@ func test(b *specs.TasksCfgBuilder, name string, parts map[string]string, compil
 			Priority: 0.8,
 		})
 		return uploadName
+	} else if strings.Contains(name, "Coverage") {
+		uploadName := fmt.Sprintf("%s%s%s", "Upload", jobNameSchema.Sep, name)
+		// We need clang_linux to get access to the llvm-profdata and llvm-cov binaries
+		// which are used to deal with the raw coverage data output by the Test step.
+		pkgs := []*specs.CipdPackage{}
+		pkgs = append(pkgs, b.MustGetCipdPackageFromAsset("clang_linux"))
+		b.MustAddTask(uploadName, &specs.TaskSpec{
+			// A dependency on compileTaskName makes the TaskScheduler link the
+			// isolated output of the compile step to the input of the upload step,
+			// which gives us access to the instrumented binary. The binary is
+			// needed to figure out symbol names and line numbers.
+			Dependencies: []string{name, compileTaskName},
+			Dimensions:   linuxGceDimensions(),
+			CipdPackages: pkgs,
+			ExtraArgs: []string{
+				"--workdir", "../../..", "upload_coverage_results",
+				fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO),
+				fmt.Sprintf("buildername=%s", name),
+				fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR),
+				fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION),
+				fmt.Sprintf("patch_repo=%s", specs.PLACEHOLDER_PATCH_REPO),
+				fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE),
+				fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE),
+				fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET),
+				fmt.Sprintf("gs_bucket=%s", CONFIG.GsBucketCoverage),
+			},
+			Isolate:  relpath("upload_coverage_results.isolate"),
+			Priority: 0.8,
+		})
+		return uploadName
 	}
+
 	return name
 }
 
diff --git a/infra/bots/jobs.json b/infra/bots/jobs.json
index 1692e9a42f..4ce280a423 100644
--- a/infra/bots/jobs.json
+++ b/infra/bots/jobs.json
@@ -21,6 +21,7 @@
   "Build-Debian9-Clang-x86-Release-Android_Vulkan",
   "Build-Debian9-Clang-x86_64-Debug",
   "Build-Debian9-Clang-x86_64-Debug-ASAN",
+  "Build-Debian9-Clang-x86_64-Debug-Coverage",
   "Build-Debian9-Clang-x86_64-Debug-MSAN",
   "Build-Debian9-Clang-x86_64-Debug-SK_USE_DISCARDABLE_SCALEDIMAGECACHE",
   "Build-Debian9-Clang-x86_64-Debug-UBSAN_float_cast_overflow",
@@ -320,6 +321,7 @@
   "Test-Debian9-Clang-GCE-CPU-AVX2-x86-Debug",
   "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug",
   "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-ASAN",
+  "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-Coverage",
   "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-MSAN",
   "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-SK_USE_DISCARDABLE_SCALEDIMAGECACHE",
   "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-UBSAN_float_cast_overflow",
diff --git a/infra/bots/recipe_modules/flavor/examples/full.expected/Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-Coverage.json b/infra/bots/recipe_modules/flavor/examples/full.expected/Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-Coverage.json
new file mode 100644
index 0000000000..3d194e9b16
--- /dev/null
+++ b/infra/bots/recipe_modules/flavor/examples/full.expected/Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-Coverage.json
@@ -0,0 +1,214 @@
+[
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "[START_DIR]/skia/bin/fetch-gn"
+    ],
+    "cwd": "[START_DIR]/skia",
+    "env": {
+      "BUILDTYPE": "Debug",
+      "CHROME_HEADLESS": "1",
+      "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
+      "SKIA_OUT": "[START_DIR]/out"
+    },
+    "infra_step": true,
+    "name": "fetch-gn"
+  },
+  {
+    "cmd": [
+      "[START_DIR]/skia/bin/gn",
+      "gen",
+      "[START_DIR]/out/Debug",
+      "--args=cc=\"[START_DIR]/clang_linux/bin/clang\" cxx=\"[START_DIR]/clang_linux/bin/clang++\" extra_cflags=[\"-B[START_DIR]/clang_linux/bin\", \"-fprofile-instr-generate\", \"-fcoverage-mapping\"] extra_ldflags=[\"-B[START_DIR]/clang_linux/bin\", \"-fuse-ld=lld\", \"-fprofile-instr-generate\", \"-fcoverage-mapping\"] skia_use_system_freetype2=false"
+    ],
+    "cwd": "[START_DIR]/skia",
+    "env": {
+      "BUILDTYPE": "Debug",
+      "CHROME_HEADLESS": "1",
+      "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
+      "SKIA_OUT": "[START_DIR]/out"
+    },
+    "name": "gn gen"
+  },
+  {
+    "cmd": [
+      "ninja",
+      "-C",
+      "[START_DIR]/out/Debug"
+    ],
+    "cwd": "[START_DIR]/skia",
+    "env": {
+      "BUILDTYPE": "Debug",
+      "CHROME_HEADLESS": "1",
+      "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
+      "SKIA_OUT": "[START_DIR]/out"
+    },
+    "name": "ninja"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "rmtree",
+      "results_dir"
+    ],
+    "infra_step": true,
+    "name": "rmtree results_dir"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "ensure-directory",
+      "--mode",
+      "0777",
+      "results_dir"
+    ],
+    "infra_step": true,
+    "name": "makedirs results_dir"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "rmtree",
+      "device_results_dir"
+    ],
+    "infra_step": true,
+    "name": "rmtree device_results_dir"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "ensure-directory",
+      "--mode",
+      "0777",
+      "device_results_dir"
+    ],
+    "infra_step": true,
+    "name": "makedirs device_results_dir"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "copy",
+      "[START_DIR]/skia/infra/bots/assets/skp/VERSION",
+      "/path/to/tmp/"
+    ],
+    "infra_step": true,
+    "name": "Get downloaded SKP VERSION"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "copy",
+      "42",
+      "[START_DIR]/tmp/SKP_VERSION"
+    ],
+    "infra_step": true,
+    "name": "write SKP_VERSION"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "copy",
+      "[START_DIR]/skia/infra/bots/assets/skimage/VERSION",
+      "/path/to/tmp/"
+    ],
+    "infra_step": true,
+    "name": "Get downloaded skimage VERSION"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "copy",
+      "42",
+      "[START_DIR]/tmp/SK_IMAGE_VERSION"
+    ],
+    "infra_step": true,
+    "name": "write SK_IMAGE_VERSION"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "copy",
+      "[START_DIR]/skia/infra/bots/assets/svg/VERSION",
+      "/path/to/tmp/"
+    ],
+    "infra_step": true,
+    "name": "Get downloaded SVG VERSION"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "copy",
+      "42",
+      "[START_DIR]/tmp/SVG_VERSION"
+    ],
+    "infra_step": true,
+    "name": "write SVG_VERSION"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[skia::flavor]/resources/symbolize_stack_trace.py",
+      "[START_DIR]",
+      "catchsegv",
+      "[START_DIR]/out/Debug/dm",
+      "--some-flag"
+    ],
+    "cwd": "[START_DIR]/skia",
+    "env": {
+      "BUILDTYPE": "Debug",
+      "CHROME_HEADLESS": "1",
+      "LLVM_PROFILE_FILE": "[CUSTOM_[SWARM_OUT_DIR]]/output.profraw",
+      "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
+      "SKIA_OUT": "[START_DIR]/out"
+    },
+    "name": "symbolized dm"
+  },
+  {
+    "name": "$result",
+    "recipe_result": null,
+    "status_code": 0
+  }
+]
+\ No newline at end of file
diff --git a/infra/bots/recipe_modules/flavor/examples/full.py b/infra/bots/recipe_modules/flavor/examples/full.py
index c0598f9b0d..e8bf10decf 100644
--- a/infra/bots/recipe_modules/flavor/examples/full.py
+++ b/infra/bots/recipe_modules/flavor/examples/full.py
@@ -84,6 +84,7 @@ TEST_BUILDERS = [
   'Perf-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-ASAN',
   'Perf-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-UBSAN_float_cast_overflow',
   'Perf-Ubuntu14-GCC-GCE-CPU-AVX2-x86_64-Release-CT_BENCH_1k_SKPs',
+  'Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-Coverage',
   'Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-TSAN',
   'Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Release',
   'Test-Ubuntu16-Clang-NUC6i5SYK-GPU-IntelIris540-x86_64-Debug-Vulkan',
diff --git a/infra/bots/recipe_modules/flavor/gn_flavor.py b/infra/bots/recipe_modules/flavor/gn_flavor.py
index a88cf1f0ff..b1855de98e 100644
--- a/infra/bots/recipe_modules/flavor/gn_flavor.py
+++ b/infra/bots/recipe_modules/flavor/gn_flavor.py
@@ -59,7 +59,15 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils):
       cxx  = emscripten_sdk + '/emscripten/incoming/em++'
       extra_cflags.append('-Wno-unknown-warning-option')
 
-    if compiler != 'MSVC' and configuration == 'Debug':
+    if extra_config == 'Coverage':
+      # See https://clang.llvm.org/docs/SourceBasedCodeCoverage.html for
+      # more info on using llvm to gather coverage information.
+      extra_cflags.append('-fprofile-instr-generate')
+      extra_cflags.append('-fcoverage-mapping')
+      extra_ldflags.append('-fprofile-instr-generate')
+      extra_ldflags.append('-fcoverage-mapping')
+
+    elif compiler != 'MSVC' and configuration == 'Debug':
       extra_cflags.append('-O1')
 
     if extra_config == 'Exceptions':
@@ -126,6 +134,8 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils):
         'skia_use_icu':        'false',
         'skia_enable_gpu':     'false',
       })
+    if extra_config == 'Coverage':
+      args['skia_use_system_freetype2'] = 'false'
 
     sanitize = ''
     if extra_config == 'UBSAN_float_cast_overflow':
@@ -228,6 +238,13 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils):
       # If we're in a signal handler, we're already crashing...
       env['TSAN_OPTIONS'] = 'report_signal_unsafe=0'
 
+    if 'Coverage' == extra_config:
+      # This is the output file for the coverage data. Just running the binary
+      # will produce the output. The output_file is in the swarming_out_dir and
+      # thus will be an isolated output of the Test step.
+      env['LLVM_PROFILE_FILE'] = self.m.path.join(self.m.vars.swarming_out_dir,
+                                                  'output.profraw')
+
     if path:
       env['PATH'] = '%%(PATH)s:%s' % ':'.join('%s' % p for p in path)
     if ld_library_path:
diff --git a/infra/bots/recipe_modules/gsutil/__init__.py b/infra/bots/recipe_modules/gsutil/__init__.py
new file mode 100644
index 0000000000..94595a0214
--- /dev/null
+++ b/infra/bots/recipe_modules/gsutil/__init__.py
@@ -0,0 +1,10 @@
+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+DEPS = [
+  'recipe_engine/context',
+  'recipe_engine/step',
+  'run',
+  'vars',
+]
diff --git a/infra/bots/recipe_modules/gsutil/api.py b/infra/bots/recipe_modules/gsutil/api.py
new file mode 100644
index 0000000000..babf85270b
--- /dev/null
+++ b/infra/bots/recipe_modules/gsutil/api.py
@@ -0,0 +1,38 @@
+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+from recipe_engine import recipe_api
+
+UPLOAD_ATTEMPTS = 5
+
+class GSUtilApi(recipe_api.RecipeApi):
+  def cp(self, name, src, dst, extra_args=None):
+    """Attempt to upload or download files to/from Google Cloud Storage (GCS).
+
+    Args:
+      name: string. Will be used to fill out the step name.
+      src: string. Absolute path for a local file or gcs file (e.g. gs://...)
+      dst: string. Same as src.
+      extra_args: optional list of args to be passed to gsutil. e.g. [-Z] asks
+        all files be compressed with gzip after upload and before download.
+
+    If the operation fails, it will be retried multiple times.
+    """
+    cmd = ['gsutil', 'cp']
+    if extra_args:
+      cmd.extend(extra_args)
+    cmd.extend([src, dst])
+
+    name = 'upload %s' % name
+    for i in xrange(UPLOAD_ATTEMPTS):
+      step_name = name
+      if i > 0:
+        step_name += ' (attempt %d)' % (i+1)
+      try:
+        self.m.step(step_name, cmd=cmd)
+        break
+      except self.m.step.StepFailure:
+        if i == UPLOAD_ATTEMPTS - 1:
+          raise
diff --git a/infra/bots/recipe_modules/gsutil/examples/full.expected/failed_all_uploads.json b/infra/bots/recipe_modules/gsutil/examples/full.expected/failed_all_uploads.json
new file mode 100644
index 0000000000..2307ef7d90
--- /dev/null
+++ b/infra/bots/recipe_modules/gsutil/examples/full.expected/failed_all_uploads.json
@@ -0,0 +1,78 @@
+[
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-Z",
+      "/foo/file",
+      "gs://bar-bucket/file"
+    ],
+    "name": "upload test file",
+    "~followup_annotations": [
+      "step returned non-zero exit code: 1",
+      "@@@STEP_FAILURE@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-Z",
+      "/foo/file",
+      "gs://bar-bucket/file"
+    ],
+    "name": "upload test file (attempt 2)",
+    "~followup_annotations": [
+      "step returned non-zero exit code: 1",
+      "@@@STEP_FAILURE@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-Z",
+      "/foo/file",
+      "gs://bar-bucket/file"
+    ],
+    "name": "upload test file (attempt 3)",
+    "~followup_annotations": [
+      "step returned non-zero exit code: 1",
+      "@@@STEP_FAILURE@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-Z",
+      "/foo/file",
+      "gs://bar-bucket/file"
+    ],
+    "name": "upload test file (attempt 4)",
+    "~followup_annotations": [
+      "step returned non-zero exit code: 1",
+      "@@@STEP_FAILURE@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-Z",
+      "/foo/file",
+      "gs://bar-bucket/file"
+    ],
+    "name": "upload test file (attempt 5)",
+    "~followup_annotations": [
+      "step returned non-zero exit code: 1",
+      "@@@STEP_FAILURE@@@"
+    ]
+  },
+  {
+    "name": "$result",
+    "reason": "Step('upload test file (attempt 5)') failed with return_code 1",
+    "recipe_result": null,
+    "status_code": 1
+  }
+]
+\ No newline at end of file
diff --git a/infra/bots/recipe_modules/gsutil/examples/full.expected/failed_one_upload.json b/infra/bots/recipe_modules/gsutil/examples/full.expected/failed_one_upload.json
new file mode 100644
index 0000000000..ab20ee6052
--- /dev/null
+++ b/infra/bots/recipe_modules/gsutil/examples/full.expected/failed_one_upload.json
@@ -0,0 +1,31 @@
+[
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-Z",
+      "/foo/file",
+      "gs://bar-bucket/file"
+    ],
+    "name": "upload test file",
+    "~followup_annotations": [
+      "step returned non-zero exit code: 1",
+      "@@@STEP_FAILURE@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-Z",
+      "/foo/file",
+      "gs://bar-bucket/file"
+    ],
+    "name": "upload test file (attempt 2)"
+  },
+  {
+    "name": "$result",
+    "recipe_result": null,
+    "status_code": 0
+  }
+]
+\ No newline at end of file
diff --git a/infra/bots/recipe_modules/gsutil/examples/full.expected/gsutil_tests.json b/infra/bots/recipe_modules/gsutil/examples/full.expected/gsutil_tests.json
new file mode 100644
index 0000000000..c29e1c4612
--- /dev/null
+++ b/infra/bots/recipe_modules/gsutil/examples/full.expected/gsutil_tests.json
@@ -0,0 +1,17 @@
+[
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-Z",
+      "/foo/file",
+      "gs://bar-bucket/file"
+    ],
+    "name": "upload test file"
+  },
+  {
+    "name": "$result",
+    "recipe_result": null,
+    "status_code": 0
+  }
+]
+\ No newline at end of file
diff --git a/infra/bots/recipe_modules/gsutil/examples/full.py b/infra/bots/recipe_modules/gsutil/examples/full.py
new file mode 100644
index 0000000000..8ed9745dcc
--- /dev/null
+++ b/infra/bots/recipe_modules/gsutil/examples/full.py
@@ -0,0 +1,55 @@
+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+# Recipe which runs the Skia gsutils tests.
+
+
+DEPS = [
+  'gsutil',
+  'recipe_engine/path',
+  'recipe_engine/properties',
+  'recipe_engine/python',
+  'recipe_engine/step',
+  'run',
+  'vars',
+]
+
+
+def RunSteps(api):
+  api.gsutil.cp('test file', '/foo/file', 'gs://bar-bucket/file', ['-Z'])
+
+def GenTests(api):
+  yield (
+      api.test('gsutil_tests') +
+      api.properties(buildername='Housekeeper-PerCommit-InfraTests',
+                     repository='https://skia.googlesource.com/skia.git',
+                     revision='abc123',
+                     path_config='kitchen',
+                     swarm_out_dir='[SWARM_OUT_DIR]')
+  )
+
+  yield (
+    api.test('failed_one_upload') +
+      api.properties(buildername='Housekeeper-PerCommit-InfraTests',
+                     repository='https://skia.googlesource.com/skia.git',
+                     revision='abc123',
+                     path_config='kitchen',
+                     swarm_out_dir='[SWARM_OUT_DIR]') +
+    api.step_data('upload test file', retcode=1)
+  )
+
+  yield (
+    api.test('failed_all_uploads') +
+      api.properties(buildername='Housekeeper-PerCommit-InfraTests',
+                     repository='https://skia.googlesource.com/skia.git',
+                     revision='abc123',
+                     path_config='kitchen',
+                     swarm_out_dir='[SWARM_OUT_DIR]') +
+    api.step_data('upload test file', retcode=1) +
+    api.step_data('upload test file (attempt 2)', retcode=1) +
+    api.step_data('upload test file (attempt 3)', retcode=1) +
+    api.step_data('upload test file (attempt 4)', retcode=1) +
+    api.step_data('upload test file (attempt 5)', retcode=1)
+  )
diff --git a/infra/bots/recipe_modules/infra/api.py b/infra/bots/recipe_modules/infra/api.py
index 932541fbf0..ee86cd9cdf 100644
--- a/infra/bots/recipe_modules/infra/api.py
+++ b/infra/bots/recipe_modules/infra/api.py
@@ -8,6 +8,7 @@ from recipe_engine import recipe_api
 
 INFRA_GO_PKG = 'go.skia.org/infra'
 UPDATE_GO_ATTEMPTS = 5
+UPLOAD_ATTEMPTS = 5
 
 
 class InfraApi(recipe_api.RecipeApi):
diff --git a/infra/bots/recipe_modules/vars/api.py b/infra/bots/recipe_modules/vars/api.py
index 1102b6955a..eff8f21693 100644
--- a/infra/bots/recipe_modules/vars/api.py
+++ b/infra/bots/recipe_modules/vars/api.py
@@ -37,15 +37,22 @@ class SkiaVarsApi(recipe_api.RecipeApi):
     self.gclient_env = {}
     self.is_compile_bot = self.builder_name.startswith('Build-')
 
+    self.persistent_checkout = False
     # Compile bots keep a persistent checkout.
-    self.persistent_checkout = (self.is_compile_bot or
-                                'RecreateSKPs' in self.builder_name or
-                                'UpdateMetaConfig' in self.builder_name or
-                                '-CT_' in self.builder_name or
-                                'Presubmit' in self.builder_name or
-                                'InfraTests' in self.builder_name or
-                                self.builder_name == "Housekeeper-PerCommit" or
-                                'CheckGeneratedFiles' in self.builder_name)
+    if self.is_compile_bot:
+      self.persistent_checkout = True
+    if 'Housekeeper' in self.builder_name:
+      self.persistent_checkout = True
+    if '-CT_' in self.builder_name:
+      self.persistent_checkout = True
+    # We need the source code for the Coverage's Upload step to be in the
+    # same absolute location as when we compiled it so we can map the
+    # coverage data to actual line numbers. We ensure this by making sure
+    # we have a checkout on the Coverage's Upload step and that the Upload
+    # step runs on the same bots that Compile.
+    if 'Coverage' in self.builder_name and 'Upload' in self.builder_name:
+      self.persistent_checkout = True
+
     if self.persistent_checkout:
       if 'Win' in self.builder_name:
         self.checkout_root = self.make_path('C:\\', 'b', 'work')
diff --git a/infra/bots/recipe_modules/vars/examples/full.expected/Perf-Ubuntu14-GCC-GCE-CPU-AVX2-x86_64-Release-CT_BENCH_1k_SKPs.json b/infra/bots/recipe_modules/vars/examples/full.expected/Perf-Ubuntu14-GCC-GCE-CPU-AVX2-x86_64-Release-CT_BENCH_1k_SKPs.json
new file mode 100644
index 0000000000..4594f9e6b9
--- /dev/null
+++ b/infra/bots/recipe_modules/vars/examples/full.expected/Perf-Ubuntu14-GCC-GCE-CPU-AVX2-x86_64-Release-CT_BENCH_1k_SKPs.json
@@ -0,0 +1,35 @@
+[
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "import os\nprint os.environ.get('SWARMING_BOT_ID', '')\n"
+    ],
+    "name": "get swarming bot id",
+    "stdout": "/path/to/tmp/",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@import os@@@",
+      "@@@STEP_LOG_LINE@python.inline@print os.environ.get('SWARMING_BOT_ID', '')@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "import os\nprint os.environ.get('SWARMING_TASK_ID', '')\n"
+    ],
+    "name": "get swarming task id",
+    "stdout": "/path/to/tmp/",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@import os@@@",
+      "@@@STEP_LOG_LINE@python.inline@print os.environ.get('SWARMING_TASK_ID', '')@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "name": "$result",
+    "recipe_result": null,
+    "status_code": 0
+  }
+]
+\ No newline at end of file
diff --git a/infra/bots/recipe_modules/vars/examples/full.expected/Upload-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-Coverage.json b/infra/bots/recipe_modules/vars/examples/full.expected/Upload-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-Coverage.json
new file mode 100644
index 0000000000..4594f9e6b9
--- /dev/null
+++ b/infra/bots/recipe_modules/vars/examples/full.expected/Upload-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-Coverage.json
@@ -0,0 +1,35 @@
+[
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "import os\nprint os.environ.get('SWARMING_BOT_ID', '')\n"
+    ],
+    "name": "get swarming bot id",
+    "stdout": "/path/to/tmp/",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@import os@@@",
+      "@@@STEP_LOG_LINE@python.inline@print os.environ.get('SWARMING_BOT_ID', '')@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "import os\nprint os.environ.get('SWARMING_TASK_ID', '')\n"
+    ],
+    "name": "get swarming task id",
+    "stdout": "/path/to/tmp/",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@import os@@@",
+      "@@@STEP_LOG_LINE@python.inline@print os.environ.get('SWARMING_TASK_ID', '')@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "name": "$result",
+    "recipe_result": null,
+    "status_code": 0
+  }
+]
+\ No newline at end of file
diff --git a/infra/bots/recipe_modules/vars/examples/full.py b/infra/bots/recipe_modules/vars/examples/full.py
index 0f38909cd7..e875a2353d 100644
--- a/infra/bots/recipe_modules/vars/examples/full.py
+++ b/infra/bots/recipe_modules/vars/examples/full.py
@@ -29,6 +29,8 @@ TEST_BUILDERS = [
   'Housekeeper-Weekly-RecreateSKPs',
   'Perf-Chromecast-GCC-Chorizo-CPU-Cortex_A7-arm-Debug',
   'Perf-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-ASAN',
+  'Perf-Ubuntu14-GCC-GCE-CPU-AVX2-x86_64-Release-CT_BENCH_1k_SKPs',
+  'Upload-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-Coverage',
 ]
 
 
diff --git a/infra/bots/recipes/compile.expected/Build-Debian9-Clang-x86_64-Debug-Coverage.json b/infra/bots/recipes/compile.expected/Build-Debian9-Clang-x86_64-Debug-Coverage.json
new file mode 100644
index 0000000000..4ac41c7345
--- /dev/null
+++ b/infra/bots/recipes/compile.expected/Build-Debian9-Clang-x86_64-Debug-Coverage.json
@@ -0,0 +1,181 @@
+[
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "ensure-directory",
+      "--mode",
+      "0777",
+      "[CUSTOM_/_B_WORK]"
+    ],
+    "infra_step": true,
+    "name": "makedirs checkout_path"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "remove",
+      "[CUSTOM_/_B_WORK]/.gclient_entries"
+    ],
+    "infra_step": true,
+    "name": "remove [CUSTOM_/_B_WORK]/.gclient_entries"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
+      "--spec-path",
+      "cache_dir = '[CUSTOM_/_B_CACHE]'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': False, 'name': 'skia', 'url': 'https://skia.googlesource.com/skia.git'}]",
+      "--patch_root",
+      "skia",
+      "--revision_mapping_file",
+      "{\"got_revision\": \"skia\"}",
+      "--git-cache-dir",
+      "[CUSTOM_/_B_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
+      "--output_json",
+      "/path/to/tmp/json",
+      "--revision",
+      "skia@abc123"
+    ],
+    "cwd": "[CUSTOM_/_B_WORK]",
+    "env_prefixes": {
+      "PATH": [
+        "RECIPE_PACKAGE_REPO[depot_tools]"
+      ]
+    },
+    "infra_step": true,
+    "name": "bot_update",
+    "~followup_annotations": [
+      "@@@STEP_TEXT@Some step text@@@",
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"did_run\": true, @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"fixed_revisions\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"skia\": \"abc123\"@@@",
+      "@@@STEP_LOG_LINE@json.output@  }, @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"manifest\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"skia\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"repository\": \"https://fake.org/skia.git\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  }, @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"patch_failure\": false, @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"patch_root\": \"skia\", @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"properties\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"got_revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\", @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"got_revision_cp\": \"refs/heads/master@{#164710}\"@@@",
+      "@@@STEP_LOG_LINE@json.output@  }, @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"root\": \"skia\", @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"step_text\": \"Some step text\"@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@SET_BUILD_PROPERTY@got_revision@\"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@",
+      "@@@SET_BUILD_PROPERTY@got_revision_cp@\"refs/heads/master@{#164710}\"@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "[CUSTOM_/_B_WORK]/skia/bin/fetch-gn"
+    ],
+    "cwd": "[CUSTOM_/_B_WORK]/skia",
+    "env": {
+      "BUILDTYPE": "Debug",
+      "CC": "/usr/bin/clang",
+      "CHROME_HEADLESS": "1",
+      "CXX": "/usr/bin/clang++",
+      "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
+      "SKIA_OUT": "[CUSTOM_/_B_WORK]/skia/out/Build-Debian9-Clang-x86_64-Debug-Coverage"
+    },
+    "infra_step": true,
+    "name": "fetch-gn"
+  },
+  {
+    "cmd": [
+      "[CUSTOM_/_B_WORK]/skia/bin/gn",
+      "gen",
+      "[CUSTOM_/_B_WORK]/skia/out/Build-Debian9-Clang-x86_64-Debug-Coverage/Debug",
+      "--args=cc=\"[START_DIR]/clang_linux/bin/clang\" cxx=\"[START_DIR]/clang_linux/bin/clang++\" extra_cflags=[\"-B[START_DIR]/clang_linux/bin\", \"-fprofile-instr-generate\", \"-fcoverage-mapping\"] extra_ldflags=[\"-B[START_DIR]/clang_linux/bin\", \"-fuse-ld=lld\", \"-fprofile-instr-generate\", \"-fcoverage-mapping\"] skia_use_system_freetype2=false target_cpu=\"x86_64\""
+    ],
+    "cwd": "[CUSTOM_/_B_WORK]/skia",
+    "env": {
+      "BUILDTYPE": "Debug",
+      "CC": "/usr/bin/clang",
+      "CHROME_HEADLESS": "1",
+      "CXX": "/usr/bin/clang++",
+      "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
+      "SKIA_OUT": "[CUSTOM_/_B_WORK]/skia/out/Build-Debian9-Clang-x86_64-Debug-Coverage"
+    },
+    "name": "gn gen"
+  },
+  {
+    "cmd": [
+      "ninja",
+      "-C",
+      "[CUSTOM_/_B_WORK]/skia/out/Build-Debian9-Clang-x86_64-Debug-Coverage/Debug"
+    ],
+    "cwd": "[CUSTOM_/_B_WORK]/skia",
+    "env": {
+      "BUILDTYPE": "Debug",
+      "CC": "/usr/bin/clang",
+      "CHROME_HEADLESS": "1",
+      "CXX": "/usr/bin/clang++",
+      "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
+      "SKIA_OUT": "[CUSTOM_/_B_WORK]/skia/out/Build-Debian9-Clang-x86_64-Debug-Coverage"
+    },
+    "name": "ninja"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "import errno\nimport glob\nimport os\nimport shutil\nimport sys\n\nsrc = sys.argv[1]\ndst = sys.argv[2]\nbuild_products_whitelist = ['dm', 'dm.exe', 'dm.app', 'nanobench.app', 'get_images_from_skps', 'get_images_from_skps.exe', 'nanobench', 'nanobench.exe', 'skpbench', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'lib/*.so', 'iOSShell.app', 'iOSShell.ipa', 'visualbench', 'visualbench.exe', 'vulkan-1.dll']\n\ntry:\n  os.makedirs(dst)\nexcept OSError as e:\n  if e.errno != errno.EEXIST:\n    raise\n\nfor pattern in build_products_whitelist:\n  path = os.path.join(src, pattern)\n  for f in glob.glob(path):\n    dst_path = os.path.join(dst, os.path.relpath(f, src))\n    if not os.path.isdir(os.path.dirname(dst_path)):\n      os.makedirs(os.path.dirname(dst_path))\n    print 'Copying build product %s to %s' % (f, dst_path)\n    shutil.move(f, dst_path)\n",
+      "[CUSTOM_/_B_WORK]/skia/out/Build-Debian9-Clang-x86_64-Debug-Coverage/Debug",
+      "[CUSTOM_[SWARM_OUT_DIR]]/out/Debug"
+    ],
+    "infra_step": true,
+    "name": "copy build products",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@import errno@@@",
+      "@@@STEP_LOG_LINE@python.inline@import glob@@@",
+      "@@@STEP_LOG_LINE@python.inline@import os@@@",
+      "@@@STEP_LOG_LINE@python.inline@import shutil@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@src = sys.argv[1]@@@",
+      "@@@STEP_LOG_LINE@python.inline@dst = sys.argv[2]@@@",
+      "@@@STEP_LOG_LINE@python.inline@build_products_whitelist = ['dm', 'dm.exe', 'dm.app', 'nanobench.app', 'get_images_from_skps', 'get_images_from_skps.exe', 'nanobench', 'nanobench.exe', 'skpbench', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'lib/*.so', 'iOSShell.app', 'iOSShell.ipa', 'visualbench', 'visualbench.exe', 'vulkan-1.dll']@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@try:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  os.makedirs(dst)@@@",
+      "@@@STEP_LOG_LINE@python.inline@except OSError as e:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  if e.errno != errno.EEXIST:@@@",
+      "@@@STEP_LOG_LINE@python.inline@    raise@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@for pattern in build_products_whitelist:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  path = os.path.join(src, pattern)@@@",
+      "@@@STEP_LOG_LINE@python.inline@  for f in glob.glob(path):@@@",
+      "@@@STEP_LOG_LINE@python.inline@    dst_path = os.path.join(dst, os.path.relpath(f, src))@@@",
+      "@@@STEP_LOG_LINE@python.inline@    if not os.path.isdir(os.path.dirname(dst_path)):@@@",
+      "@@@STEP_LOG_LINE@python.inline@      os.makedirs(os.path.dirname(dst_path))@@@",
+      "@@@STEP_LOG_LINE@python.inline@    print 'Copying build product %s to %s' % (f, dst_path)@@@",
+      "@@@STEP_LOG_LINE@python.inline@    shutil.move(f, dst_path)@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "name": "$result",
+    "recipe_result": null,
+    "status_code": 0
+  }
+]
+\ No newline at end of file
diff --git a/infra/bots/recipes/compile.py b/infra/bots/recipes/compile.py
index 2a8b25600e..bde164d1e3 100644
--- a/infra/bots/recipes/compile.py
+++ b/infra/bots/recipes/compile.py
@@ -82,6 +82,7 @@ TEST_BUILDERS = [
   'Build-Debian9-Clang-mipsel-Debug-Android',
   'Build-Debian9-Clang-x86_64-Debug',
   'Build-Debian9-Clang-x86_64-Debug-ASAN',
+  'Build-Debian9-Clang-x86_64-Debug-Coverage',
   'Build-Debian9-Clang-x86_64-Debug-MSAN',
   'Build-Debian9-Clang-x86_64-Debug-SK_USE_DISCARDABLE_SCALEDIMAGECACHE',
   'Build-Debian9-Clang-x86_64-Release-Fast',
diff --git a/infra/bots/recipes/test.expected/Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-Coverage.json b/infra/bots/recipes/test.expected/Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-Coverage.json
new file mode 100644
index 0000000000..fcb65182cd
--- /dev/null
+++ b/infra/bots/recipes/test.expected/Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-Coverage.json
@@ -0,0 +1,475 @@
+[
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "copy",
+      "[START_DIR]/skia/infra/bots/assets/skp/VERSION",
+      "/path/to/tmp/"
+    ],
+    "infra_step": true,
+    "name": "Get downloaded SKP VERSION"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "copy",
+      "42",
+      "[START_DIR]/tmp/SKP_VERSION"
+    ],
+    "infra_step": true,
+    "name": "write SKP_VERSION"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "copy",
+      "[START_DIR]/skia/infra/bots/assets/skimage/VERSION",
+      "/path/to/tmp/"
+    ],
+    "infra_step": true,
+    "name": "Get downloaded skimage VERSION"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "copy",
+      "42",
+      "[START_DIR]/tmp/SK_IMAGE_VERSION"
+    ],
+    "infra_step": true,
+    "name": "write SK_IMAGE_VERSION"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "copy",
+      "[START_DIR]/skia/infra/bots/assets/svg/VERSION",
+      "/path/to/tmp/"
+    ],
+    "infra_step": true,
+    "name": "Get downloaded SVG VERSION"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "copy",
+      "42",
+      "[START_DIR]/tmp/SVG_VERSION"
+    ],
+    "infra_step": true,
+    "name": "write SVG_VERSION"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "import os\nprint os.environ.get('SWARMING_BOT_ID', '')\n"
+    ],
+    "name": "get swarming bot id",
+    "stdout": "/path/to/tmp/",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@import os@@@",
+      "@@@STEP_LOG_LINE@python.inline@print os.environ.get('SWARMING_BOT_ID', '')@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "import os\nprint os.environ.get('SWARMING_TASK_ID', '')\n"
+    ],
+    "name": "get swarming task id",
+    "stdout": "/path/to/tmp/",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@import os@@@",
+      "@@@STEP_LOG_LINE@python.inline@print os.environ.get('SWARMING_TASK_ID', '')@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[skia::flavor]/resources/symbolize_stack_trace.py",
+      "[START_DIR]",
+      "catchsegv",
+      "[START_DIR]/out/Debug/dm",
+      "--resourcePath",
+      "[START_DIR]/skia/resources",
+      "--skps",
+      "[START_DIR]/skp",
+      "--images",
+      "[START_DIR]/skimage/dm",
+      "--colorImages",
+      "[START_DIR]/skimage/colorspace",
+      "--nameByHash",
+      "--properties",
+      "gitHash",
+      "abc123",
+      "builder",
+      "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-Coverage",
+      "swarming_bot_id",
+      "skia-bot-123",
+      "swarming_task_id",
+      "123456",
+      "--svgs",
+      "[START_DIR]/svg",
+      "--key",
+      "arch",
+      "x86_64",
+      "compiler",
+      "Clang",
+      "configuration",
+      "Debug",
+      "cpu_or_gpu",
+      "CPU",
+      "cpu_or_gpu_value",
+      "AVX2",
+      "extra_config",
+      "Coverage",
+      "model",
+      "GCE",
+      "os",
+      "Debian9",
+      "--dont_write",
+      "pdf",
+      "--randomProcessorTest",
+      "--nogpu",
+      "--config",
+      "8888",
+      "srgb",
+      "pdf",
+      "565",
+      "f16",
+      "sp-8888",
+      "2ndpic-8888",
+      "lite-8888",
+      "gbr-8888",
+      "serialize-8888",
+      "tiles_rt-8888",
+      "pic-8888",
+      "--src",
+      "tests",
+      "gm",
+      "image",
+      "colorImage",
+      "--blacklist",
+      "f16",
+      "_",
+      "_",
+      "dstreadshuffle",
+      "gbr-8888",
+      "image",
+      "_",
+      "_",
+      "gbr-8888",
+      "colorImage",
+      "_",
+      "_",
+      "serialize-8888",
+      "gm",
+      "_",
+      "bleed_image",
+      "serialize-8888",
+      "gm",
+      "_",
+      "c_gms",
+      "serialize-8888",
+      "gm",
+      "_",
+      "colortype",
+      "serialize-8888",
+      "gm",
+      "_",
+      "colortype_xfermodes",
+      "serialize-8888",
+      "gm",
+      "_",
+      "drawfilter",
+      "serialize-8888",
+      "gm",
+      "_",
+      "fontmgr_bounds_0.75_0",
+      "serialize-8888",
+      "gm",
+      "_",
+      "fontmgr_bounds_1_-0.25",
+      "serialize-8888",
+      "gm",
+      "_",
+      "fontmgr_bounds",
+      "serialize-8888",
+      "gm",
+      "_",
+      "fontmgr_match",
+      "serialize-8888",
+      "gm",
+      "_",
+      "fontmgr_iter",
+      "serialize-8888",
+      "gm",
+      "_",
+      "imagemasksubset",
+      "serialize-8888",
+      "gm",
+      "_",
+      "bitmapfilters",
+      "serialize-8888",
+      "gm",
+      "_",
+      "bitmapshaders",
+      "serialize-8888",
+      "gm",
+      "_",
+      "bleed",
+      "serialize-8888",
+      "gm",
+      "_",
+      "bleed_alpha_bmp",
+      "serialize-8888",
+      "gm",
+      "_",
+      "bleed_alpha_bmp_shader",
+      "serialize-8888",
+      "gm",
+      "_",
+      "convex_poly_clip",
+      "serialize-8888",
+      "gm",
+      "_",
+      "extractalpha",
+      "serialize-8888",
+      "gm",
+      "_",
+      "filterbitmap_checkerboard_32_32_g8",
+      "serialize-8888",
+      "gm",
+      "_",
+      "filterbitmap_image_mandrill_64",
+      "serialize-8888",
+      "gm",
+      "_",
+      "shadows",
+      "serialize-8888",
+      "gm",
+      "_",
+      "simpleaaclip_aaclip",
+      "serialize-8888",
+      "gm",
+      "_",
+      "composeshader_bitmap",
+      "serialize-8888",
+      "gm",
+      "_",
+      "scaled_tilemodes_npot",
+      "serialize-8888",
+      "gm",
+      "_",
+      "scaled_tilemodes",
+      "serialize-8888",
+      "gm",
+      "_",
+      "typefacerendering_pfaMac",
+      "serialize-8888",
+      "gm",
+      "_",
+      "parsedpaths",
+      "serialize-8888",
+      "gm",
+      "_",
+      "ImageGeneratorExternal_rect",
+      "serialize-8888",
+      "gm",
+      "_",
+      "ImageGeneratorExternal_shader",
+      "serialize-8888",
+      "gm",
+      "_",
+      "shadow_utils",
+      "serialize-8888",
+      "gm",
+      "_",
+      "makecolorspace",
+      "serialize-8888",
+      "gm",
+      "_",
+      "bleed_alpha_image",
+      "serialize-8888",
+      "gm",
+      "_",
+      "bleed_alpha_image_shader",
+      "sp-8888",
+      "gm",
+      "_",
+      "drawfilter",
+      "pic-8888",
+      "gm",
+      "_",
+      "drawfilter",
+      "2ndpic-8888",
+      "gm",
+      "_",
+      "drawfilter",
+      "lite-8888",
+      "gm",
+      "_",
+      "drawfilter",
+      "sp-8888",
+      "gm",
+      "_",
+      "image-cacherator-from-picture",
+      "pic-8888",
+      "gm",
+      "_",
+      "image-cacherator-from-picture",
+      "2ndpic-8888",
+      "gm",
+      "_",
+      "image-cacherator-from-picture",
+      "serialize-8888",
+      "gm",
+      "_",
+      "image-cacherator-from-picture",
+      "sp-8888",
+      "gm",
+      "_",
+      "image-cacherator-from-raster",
+      "pic-8888",
+      "gm",
+      "_",
+      "image-cacherator-from-raster",
+      "2ndpic-8888",
+      "gm",
+      "_",
+      "image-cacherator-from-raster",
+      "serialize-8888",
+      "gm",
+      "_",
+      "image-cacherator-from-raster",
+      "sp-8888",
+      "gm",
+      "_",
+      "image-cacherator-from-ctable",
+      "pic-8888",
+      "gm",
+      "_",
+      "image-cacherator-from-ctable",
+      "2ndpic-8888",
+      "gm",
+      "_",
+      "image-cacherator-from-ctable",
+      "serialize-8888",
+      "gm",
+      "_",
+      "image-cacherator-from-ctable",
+      "sp-8888",
+      "gm",
+      "_",
+      "gamut",
+      "pic-8888",
+      "gm",
+      "_",
+      "gamut",
+      "lite-8888",
+      "gm",
+      "_",
+      "gamut",
+      "2ndpic-8888",
+      "gm",
+      "_",
+      "gamut",
+      "serialize-8888",
+      "gm",
+      "_",
+      "gamut",
+      "sp-8888",
+      "gm",
+      "_",
+      "complexclip4_bw",
+      "pic-8888",
+      "gm",
+      "_",
+      "complexclip4_bw",
+      "lite-8888",
+      "gm",
+      "_",
+      "complexclip4_bw",
+      "2ndpic-8888",
+      "gm",
+      "_",
+      "complexclip4_bw",
+      "serialize-8888",
+      "gm",
+      "_",
+      "complexclip4_bw",
+      "sp-8888",
+      "gm",
+      "_",
+      "complexclip4_aa",
+      "pic-8888",
+      "gm",
+      "_",
+      "complexclip4_aa",
+      "lite-8888",
+      "gm",
+      "_",
+      "complexclip4_aa",
+      "2ndpic-8888",
+      "gm",
+      "_",
+      "complexclip4_aa",
+      "serialize-8888",
+      "gm",
+      "_",
+      "complexclip4_aa",
+      "tiles_rt-8888",
+      "gm",
+      "_",
+      "complexclip4_bw",
+      "tiles_rt-8888",
+      "gm",
+      "_",
+      "complexclip4_aa",
+      "--verbose"
+    ],
+    "cwd": "[START_DIR]/skia",
+    "env": {
+      "BUILDTYPE": "Debug",
+      "CHROME_HEADLESS": "1",
+      "LLVM_PROFILE_FILE": "[CUSTOM_[SWARM_OUT_DIR]]/output.profraw",
+      "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
+      "SKIA_OUT": "[START_DIR]/out"
+    },
+    "name": "symbolized dm"
+  },
+  {
+    "name": "$result",
+    "recipe_result": null,
+    "status_code": 0
+  }
+]
+\ No newline at end of file
diff --git a/infra/bots/recipes/test.py b/infra/bots/recipes/test.py
index 6aa15dc69a..6ea5701a76 100644
--- a/infra/bots/recipes/test.py
+++ b/infra/bots/recipes/test.py
@@ -872,6 +872,7 @@ TEST_BUILDERS = [
   'Test-ChromeOS-Clang-Chromebook_CB5_312T-GPU-PowerVRGX6250-arm-Debug',
   'Test-Chromecast-GCC-Chorizo-GPU-Cortex_A7-arm-Release',
   'Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-ASAN',
+  'Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-Coverage',
   'Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-MSAN',
   ('Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug'
    '-SK_USE_DISCARDABLE_SCALEDIMAGECACHE'),
diff --git a/infra/bots/recipes/upload_coverage_results.expected/alternate_bucket.json b/infra/bots/recipes/upload_coverage_results.expected/alternate_bucket.json
new file mode 100644
index 0000000000..30fe085fcb
--- /dev/null
+++ b/infra/bots/recipes/upload_coverage_results.expected/alternate_bucket.json
@@ -0,0 +1,110 @@
+[
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-Z",
+      "[START_DIR]/output.profraw",
+      "gs://skia-coverage-alt/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug.profraw"
+    ],
+    "name": "upload raw data"
+  },
+  {
+    "cmd": [
+      "[START_DIR]/clang_linux/bin/llvm-profdata",
+      "merge",
+      "-sparse",
+      "[START_DIR]/output.profraw",
+      "-o",
+      "[START_DIR]/output.profdata"
+    ],
+    "name": "merge and index"
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-Z",
+      "[START_DIR]/output.profdata",
+      "gs://skia-coverage-alt/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug.profdata"
+    ],
+    "name": "upload parsed data"
+  },
+  {
+    "cmd": [
+      "[START_DIR]/clang_linux/bin/llvm-cov",
+      "show",
+      "[START_DIR]/out/Debug/dm",
+      "-instr-profile=[START_DIR]/output.profdata",
+      "-use-color=0",
+      "-format=text",
+      "-output-dir=[START_DIR]/coverage_text"
+    ],
+    "name": "create text summary"
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-Z",
+      "[START_DIR]/coverage_text/index.txt",
+      "gs://skia-coverage-alt/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug.summary"
+    ],
+    "name": "upload coverage summary"
+  },
+  {
+    "cmd": [
+      "tar",
+      "-cvf",
+      "[START_DIR]/coverage.text.tar",
+      "[START_DIR]/coverage_text"
+    ],
+    "name": "create text coverage archive"
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-Z",
+      "[START_DIR]/coverage.text.tar",
+      "gs://skia-coverage-alt/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug.text.tar"
+    ],
+    "name": "upload text coverage data"
+  },
+  {
+    "cmd": [
+      "[START_DIR]/clang_linux/bin/llvm-cov",
+      "show",
+      "[START_DIR]/out/Debug/dm",
+      "-instr-profile=[START_DIR]/output.profdata",
+      "-use-color=1",
+      "-format=html",
+      "-output-dir=[START_DIR]/coverage_html"
+    ],
+    "name": "create html summary"
+  },
+  {
+    "cmd": [
+      "tar",
+      "-cvf",
+      "[START_DIR]/coverage.html.tar",
+      "[START_DIR]/coverage_html"
+    ],
+    "name": "create html coverage archive"
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-Z",
+      "[START_DIR]/coverage.html.tar",
+      "gs://skia-coverage-alt/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug.html.tar"
+    ],
+    "name": "upload html coverage data"
+  },
+  {
+    "name": "$result",
+    "recipe_result": null,
+    "status_code": 0
+  }
+]
+\ No newline at end of file
diff --git a/infra/bots/recipes/upload_coverage_results.expected/failed_all.json b/infra/bots/recipes/upload_coverage_results.expected/failed_all.json
new file mode 100644
index 0000000000..e276f05639
--- /dev/null
+++ b/infra/bots/recipes/upload_coverage_results.expected/failed_all.json
@@ -0,0 +1,78 @@
+[
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-Z",
+      "[START_DIR]/output.profraw",
+      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug.profraw"
+    ],
+    "name": "upload raw data",
+    "~followup_annotations": [
+      "step returned non-zero exit code: 1",
+      "@@@STEP_FAILURE@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-Z",
+      "[START_DIR]/output.profraw",
+      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug.profraw"
+    ],
+    "name": "upload raw data (attempt 2)",
+    "~followup_annotations": [
+      "step returned non-zero exit code: 1",
+      "@@@STEP_FAILURE@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-Z",
+      "[START_DIR]/output.profraw",
+      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug.profraw"
+    ],
+    "name": "upload raw data (attempt 3)",
+    "~followup_annotations": [
+      "step returned non-zero exit code: 1",
+      "@@@STEP_FAILURE@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-Z",
+      "[START_DIR]/output.profraw",
+      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug.profraw"
+    ],
+    "name": "upload raw data (attempt 4)",
+    "~followup_annotations": [
+      "step returned non-zero exit code: 1",
+      "@@@STEP_FAILURE@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-Z",
+      "[START_DIR]/output.profraw",
+      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug.profraw"
+    ],
+    "name": "upload raw data (attempt 5)",
+    "~followup_annotations": [
+      "step returned non-zero exit code: 1",
+      "@@@STEP_FAILURE@@@"
+    ]
+  },
+  {
+    "name": "$result",
+    "reason": "Step('upload raw data (attempt 5)') failed with return_code 1",
+    "recipe_result": null,
+    "status_code": 1
+  }
+]
+\ No newline at end of file
diff --git a/infra/bots/recipes/upload_coverage_results.expected/failed_once.json b/infra/bots/recipes/upload_coverage_results.expected/failed_once.json
new file mode 100644
index 0000000000..2312af4b85
--- /dev/null
+++ b/infra/bots/recipes/upload_coverage_results.expected/failed_once.json
@@ -0,0 +1,124 @@
+[
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-Z",
+      "[START_DIR]/output.profraw",
+      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug.profraw"
+    ],
+    "name": "upload raw data",
+    "~followup_annotations": [
+      "step returned non-zero exit code: 1",
+      "@@@STEP_FAILURE@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-Z",
+      "[START_DIR]/output.profraw",
+      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug.profraw"
+    ],
+    "name": "upload raw data (attempt 2)"
+  },
+  {
+    "cmd": [
+      "[START_DIR]/clang_linux/bin/llvm-profdata",
+      "merge",
+      "-sparse",
+      "[START_DIR]/output.profraw",
+      "-o",
+      "[START_DIR]/output.profdata"
+    ],
+    "name": "merge and index"
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-Z",
+      "[START_DIR]/output.profdata",
+      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug.profdata"
+    ],
+    "name": "upload parsed data"
+  },
+  {
+    "cmd": [
+      "[START_DIR]/clang_linux/bin/llvm-cov",
+      "show",
+      "[START_DIR]/out/Debug/dm",
+      "-instr-profile=[START_DIR]/output.profdata",
+      "-use-color=0",
+      "-format=text",
+      "-output-dir=[START_DIR]/coverage_text"
+    ],
+    "name": "create text summary"
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-Z",
+      "[START_DIR]/coverage_text/index.txt",
+      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug.summary"
+    ],
+    "name": "upload coverage summary"
+  },
+  {
+    "cmd": [
+      "tar",
+      "-cvf",
+      "[START_DIR]/coverage.text.tar",
+      "[START_DIR]/coverage_text"
+    ],
+    "name": "create text coverage archive"
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-Z",
+      "[START_DIR]/coverage.text.tar",
+      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug.text.tar"
+    ],
+    "name": "upload text coverage data"
+  },
+  {
+    "cmd": [
+      "[START_DIR]/clang_linux/bin/llvm-cov",
+      "show",
+      "[START_DIR]/out/Debug/dm",
+      "-instr-profile=[START_DIR]/output.profdata",
+      "-use-color=1",
+      "-format=html",
+      "-output-dir=[START_DIR]/coverage_html"
+    ],
+    "name": "create html summary"
+  },
+  {
+    "cmd": [
+      "tar",
+      "-cvf",
+      "[START_DIR]/coverage.html.tar",
+      "[START_DIR]/coverage_html"
+    ],
+    "name": "create html coverage archive"
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-Z",
+      "[START_DIR]/coverage.html.tar",
+      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug.html.tar"
+    ],
+    "name": "upload html coverage data"
+  },
+  {
+    "name": "$result",
+    "recipe_result": null,
+    "status_code": 0
+  }
+]
+\ No newline at end of file
diff --git a/infra/bots/recipes/upload_coverage_results.expected/normal_bot.json b/infra/bots/recipes/upload_coverage_results.expected/normal_bot.json
new file mode 100644
index 0000000000..cb3ec71592
--- /dev/null
+++ b/infra/bots/recipes/upload_coverage_results.expected/normal_bot.json
@@ -0,0 +1,110 @@
+[
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-Z",
+      "[START_DIR]/output.profraw",
+      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug.profraw"
+    ],
+    "name": "upload raw data"
+  },
+  {
+    "cmd": [
+      "[START_DIR]/clang_linux/bin/llvm-profdata",
+      "merge",
+      "-sparse",
+      "[START_DIR]/output.profraw",
+      "-o",
+      "[START_DIR]/output.profdata"
+    ],
+    "name": "merge and index"
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-Z",
+      "[START_DIR]/output.profdata",
+      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug.profdata"
+    ],
+    "name": "upload parsed data"
+  },
+  {
+    "cmd": [
+      "[START_DIR]/clang_linux/bin/llvm-cov",
+      "show",
+      "[START_DIR]/out/Debug/dm",
+      "-instr-profile=[START_DIR]/output.profdata",
+      "-use-color=0",
+      "-format=text",
+      "-output-dir=[START_DIR]/coverage_text"
+    ],
+    "name": "create text summary"
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-Z",
+      "[START_DIR]/coverage_text/index.txt",
+      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug.summary"
+    ],
+    "name": "upload coverage summary"
+  },
+  {
+    "cmd": [
+      "tar",
+      "-cvf",
+      "[START_DIR]/coverage.text.tar",
+      "[START_DIR]/coverage_text"
+    ],
+    "name": "create text coverage archive"
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-Z",
+      "[START_DIR]/coverage.text.tar",
+      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug.text.tar"
+    ],
+    "name": "upload text coverage data"
+  },
+  {
+    "cmd": [
+      "[START_DIR]/clang_linux/bin/llvm-cov",
+      "show",
+      "[START_DIR]/out/Debug/dm",
+      "-instr-profile=[START_DIR]/output.profdata",
+      "-use-color=1",
+      "-format=html",
+      "-output-dir=[START_DIR]/coverage_html"
+    ],
+    "name": "create html summary"
+  },
+  {
+    "cmd": [
+      "tar",
+      "-cvf",
+      "[START_DIR]/coverage.html.tar",
+      "[START_DIR]/coverage_html"
+    ],
+    "name": "create html coverage archive"
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-Z",
+      "[START_DIR]/coverage.html.tar",
+      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug.html.tar"
+    ],
+    "name": "upload html coverage data"
+  },
+  {
+    "name": "$result",
+    "recipe_result": null,
+    "status_code": 0
+  }
+]
+\ No newline at end of file
diff --git a/infra/bots/recipes/upload_coverage_results.expected/trybot.json b/infra/bots/recipes/upload_coverage_results.expected/trybot.json
new file mode 100644
index 0000000000..65584f6486
--- /dev/null
+++ b/infra/bots/recipes/upload_coverage_results.expected/trybot.json
@@ -0,0 +1,110 @@
+[
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-Z",
+      "[START_DIR]/output.profraw",
+      "gs://skia-coverage/trybot/456789/12/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug.profraw"
+    ],
+    "name": "upload raw data"
+  },
+  {
+    "cmd": [
+      "[START_DIR]/clang_linux/bin/llvm-profdata",
+      "merge",
+      "-sparse",
+      "[START_DIR]/output.profraw",
+      "-o",
+      "[START_DIR]/output.profdata"
+    ],
+    "name": "merge and index"
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-Z",
+      "[START_DIR]/output.profdata",
+      "gs://skia-coverage/trybot/456789/12/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug.profdata"
+    ],
+    "name": "upload parsed data"
+  },
+  {
+    "cmd": [
+      "[START_DIR]/clang_linux/bin/llvm-cov",
+      "show",
+      "[START_DIR]/out/Debug/dm",
+      "-instr-profile=[START_DIR]/output.profdata",
+      "-use-color=0",
+      "-format=text",
+      "-output-dir=[START_DIR]/coverage_text"
+    ],
+    "name": "create text summary"
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-Z",
+      "[START_DIR]/coverage_text/index.txt",
+      "gs://skia-coverage/trybot/456789/12/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug.summary"
+    ],
+    "name": "upload coverage summary"
+  },
+  {
+    "cmd": [
+      "tar",
+      "-cvf",
+      "[START_DIR]/coverage.text.tar",
+      "[START_DIR]/coverage_text"
+    ],
+    "name": "create text coverage archive"
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-Z",
+      "[START_DIR]/coverage.text.tar",
+      "gs://skia-coverage/trybot/456789/12/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug.text.tar"
+    ],
+    "name": "upload text coverage data"
+  },
+  {
+    "cmd": [
+      "[START_DIR]/clang_linux/bin/llvm-cov",
+      "show",
+      "[START_DIR]/out/Debug/dm",
+      "-instr-profile=[START_DIR]/output.profdata",
+      "-use-color=1",
+      "-format=html",
+      "-output-dir=[START_DIR]/coverage_html"
+    ],
+    "name": "create html summary"
+  },
+  {
+    "cmd": [
+      "tar",
+      "-cvf",
+      "[START_DIR]/coverage.html.tar",
+      "[START_DIR]/coverage_html"
+    ],
+    "name": "create html coverage archive"
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-Z",
+      "[START_DIR]/coverage.html.tar",
+      "gs://skia-coverage/trybot/456789/12/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug.html.tar"
+    ],
+    "name": "upload html coverage data"
+  },
+  {
+    "name": "$result",
+    "recipe_result": null,
+    "status_code": 0
+  }
+]
+\ No newline at end of file
diff --git a/infra/bots/recipes/upload_coverage_results.py b/infra/bots/recipes/upload_coverage_results.py
new file mode 100644
index 0000000000..fe78c86c82
--- /dev/null
+++ b/infra/bots/recipes/upload_coverage_results.py
@@ -0,0 +1,181 @@
+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+# Recipe for uploading Coverage results.
+
+
+import calendar
+
+
+DEPS = [
+  'recipe_engine/json',
+  'recipe_engine/path',
+  'recipe_engine/properties',
+  'recipe_engine/step',
+  'recipe_engine/time',
+  'gsutil',
+]
+
+
+TRY_JOB_FOLDER = 'trybot/%s/%s/' # % (issue_number, patchset_number)
+COMMIT_FOLDER = 'commit/%s/'      # % (git_revision)
+
+RAW_FILE = '%s.profraw'
+PARSED_FILE = '%s.profdata'
+SUMMARY_FILE = '%s.summary'
+# Text is an easier format to read with machines (e.g. for Gerrit).
+COVERAGE_TEXT_FILE = '%s.text.tar'
+# HTML is a quick and dirty browsable format. (e.g. for coverage.skia.org)
+COVERAGE_HTML_FILE = '%s.html.tar'
+
+def RunSteps(api):
+  # See https://clang.llvm.org/docs/SourceBasedCodeCoverage.html for a
+  # detailed explanation of getting code coverage from LLVM.
+  # Since we have already compiled the binary with the special flags
+  # and run the executable to generate an output.profraw, we
+  # need to merge and index the data, create the coverage output,
+  # and then upload the results to GCS. We also upload the intermediate
+  # results to GCS so we can regenerate reports if needed.
+  builder_name = api.properties['buildername']
+  bucket = api.properties['gs_bucket']
+
+  # The raw data is brought in as an isolated input.
+  raw_data = api.path['start_dir'].join('output.profraw')
+  # The instrumented executable is brought in as an isolated input.
+  executable = api.path['start_dir'].join('out','Debug','dm')
+  # clang_dir is brought in via CIPD.
+  clang_dir = api.path['start_dir'].join('clang_linux', 'bin')
+
+  revision = api.properties['revision']
+  path = COMMIT_FOLDER % revision
+
+  issue = api.properties.get('patch_issue')
+  patchset = api.properties.get('patch_set')
+  if issue and patchset:
+    path = TRY_JOB_FOLDER % (issue, patchset)
+
+  gcs_file = RAW_FILE % builder_name
+  api.gsutil.cp('raw data', raw_data,
+                   'gs://%s/%s%s' % (bucket, path, gcs_file), ['-Z'])
+
+  # Merge and Index the data.
+  indexed_data = api.path['start_dir'].join('output.profdata')
+  api.step('merge and index',
+           cmd=[clang_dir.join('llvm-profdata'),
+               'merge',
+               '-sparse',
+               raw_data,
+               '-o',
+               indexed_data ])
+
+  gcs_file = PARSED_FILE % builder_name
+  api.gsutil.cp('parsed data', indexed_data,
+                   'gs://%s/%s%s' % (bucket, path, gcs_file), ['-Z'])
+
+  # Create text coverage output
+  output_data = api.path['start_dir'].join('coverage_text')
+  api.step('create text summary',
+           cmd=[clang_dir.join('llvm-cov'),
+               'show',
+               executable,
+               '-instr-profile=' + str(indexed_data),
+               '-use-color=0',
+               '-format=text',
+               '-output-dir=' + str(output_data)])
+
+  # Upload the summary by itself so we can get easier access to it (instead of
+  # downloading and untarring all the coverage data.
+  gcs_file = SUMMARY_FILE % builder_name
+  api.gsutil.cp('coverage summary', output_data.join('index.txt'),
+                   'gs://%s/%s%s' % (bucket, path, gcs_file), ['-Z'])
+
+  tar_file = api.path['start_dir'].join('coverage.text.tar')
+
+  # Tar and upload the coverage data. We tar it to ease downloading/ingestion,
+  # otherwise, there is a 1:1 mapping of source code files -> coverage files.
+  api.step('create text coverage archive', cmd=['tar', '-cvf',
+                                           tar_file, output_data])
+
+  gcs_file = COVERAGE_TEXT_FILE % builder_name
+  api.gsutil.cp('text coverage data', tar_file,
+                   'gs://%s/%s%s' % (bucket, path, gcs_file), ['-Z'])
+
+  # Create html coverage output
+  output_data = api.path['start_dir'].join('coverage_html')
+  api.step('create html summary',
+           cmd=[clang_dir.join('llvm-cov'),
+               'show',
+               executable,
+               '-instr-profile=' + str(indexed_data),
+               '-use-color=1',
+               '-format=html',
+               '-output-dir=' + str(output_data)])
+
+  tar_file = api.path['start_dir'].join('coverage.html.tar')
+
+  # Tar and upload the coverage data. We tar it to ease downloading/ingestion,
+  # otherwise, there is a 1:1 mapping of source code files -> coverage files.
+  api.step('create html coverage archive',
+           cmd=['tar', '-cvf', tar_file, output_data])
+
+  gcs_file = COVERAGE_HTML_FILE % builder_name
+  api.gsutil.cp('html coverage data', tar_file,
+                   'gs://%s/%s%s' % (bucket, path, gcs_file), ['-Z'])
+
+
+def GenTests(api):
+  builder = 'Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug'
+  yield (
+    api.test('normal_bot') +
+    api.properties(buildername=builder,
+                   gs_bucket='skia-coverage',
+                   revision='abc123',
+                   path_config='kitchen')
+  )
+
+  yield (
+    api.test('alternate_bucket') +
+    api.properties(buildername=builder,
+                   gs_bucket='skia-coverage-alt',
+                   revision='abc123',
+                   path_config='kitchen')
+  )
+
+  yield (
+    api.test('failed_once') +
+    api.properties(buildername=builder,
+                   gs_bucket='skia-coverage',
+                   revision='abc123',
+                   path_config='kitchen') +
+    api.step_data('upload raw data', retcode=1)
+  )
+
+  yield (
+    api.test('failed_all') +
+    api.properties(buildername=builder,
+                   gs_bucket='skia-coverage',
+                   revision='abc123',
+                   path_config='kitchen') +
+    api.step_data('upload raw data', retcode=1) +
+    api.step_data('upload raw data (attempt 2)', retcode=1) +
+    api.step_data('upload raw data (attempt 3)', retcode=1) +
+    api.step_data('upload raw data (attempt 4)', retcode=1) +
+    api.step_data('upload raw data (attempt 5)', retcode=1)
+  )
+
+  yield (
+      api.test('trybot') +
+      api.properties(
+          buildername=builder,
+          gs_bucket='skia-coverage',
+          revision='abc123',
+          path_config='kitchen',
+          patch_storage='gerrit') +
+      api.properties.tryserver(
+          buildername=builder,
+          gerrit_project='skia',
+          gerrit_url='https://skia-review.googlesource.com/',
+      )
+  )
diff --git a/infra/bots/recipes/upload_dm_results.py b/infra/bots/recipes/upload_dm_results.py
index 3ec2828e5d..a46c787d26 100644
--- a/infra/bots/recipes/upload_dm_results.py
+++ b/infra/bots/recipes/upload_dm_results.py
@@ -16,34 +16,15 @@ DEPS = [
   'recipe_engine/properties',
   'recipe_engine/step',
   'recipe_engine/time',
+  'gsutil',
 ]
 
 
 GS_BUCKET_IMAGES = 'skia-infra-gm'
 DM_JSON = 'dm.json'
-UPLOAD_ATTEMPTS = 5
 VERBOSE_LOG = 'verbose.log'
 
 
-def cp(api, name, src, dst, extra_args=None):
-  cmd = ['gsutil', 'cp']
-  if extra_args:
-    cmd.extend(extra_args)
-  cmd.extend([src, dst])
-
-  name = 'upload %s' % name
-  for i in xrange(UPLOAD_ATTEMPTS):
-    step_name = name
-    if i > 0:
-      step_name += ' (attempt %d)' % (i+1)
-    try:
-      api.step(step_name, cmd=cmd)
-      break
-    except api.step.StepFailure:
-      if i == UPLOAD_ATTEMPTS - 1:
-        raise
-
-
 def RunSteps(api):
   builder_name = api.properties['buildername']
   revision = api.properties['revision']
@@ -71,7 +52,8 @@ def RunSteps(api):
     # For some reason, glob returns results_dir when it should return nothing.
     files_to_upload = [f for f in files_to_upload if str(f).endswith(ext)]
     if len(files_to_upload) > 0:
-      cp(api, 'images', results_dir.join('*%s' % ext), image_dest_path)
+      api.gsutil.cp('images', results_dir.join('*%s' % ext),
+                       image_dest_path)
 
   # Upload the JSON summary and verbose.log.
   now = api.time.utcnow()
@@ -95,8 +77,8 @@ def RunSteps(api):
   summary_dest_path = 'gs://%s/%s' % (api.properties['gs_bucket'],
                                       summary_dest_path)
 
-  cp(api, 'JSON and logs', tmp_dir.join('*'), summary_dest_path,
-     ['-z', 'json,log'])
+  api.gsutil.cp('JSON and logs', tmp_dir.join('*'), summary_dest_path,
+                   ['-z', 'json,log'])
 
 
 def GenTests(api):
diff --git a/infra/bots/tasks.json b/infra/bots/tasks.json
index 09a8f184b9..553689e846 100644
--- a/infra/bots/tasks.json
+++ b/infra/bots/tasks.json
@@ -132,6 +132,12 @@
         "Build-Debian9-Clang-x86_64-Debug-ASAN"
       ]
     },
+    "Build-Debian9-Clang-x86_64-Debug-Coverage": {
+      "priority": 0.8,
+      "tasks": [
+        "Build-Debian9-Clang-x86_64-Debug-Coverage"
+      ]
+    },
     "Build-Debian9-Clang-x86_64-Debug-MSAN": {
       "priority": 0.8,
       "tasks": [
@@ -1935,6 +1941,12 @@
         "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-ASAN"
       ]
     },
+    "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-Coverage": {
+      "priority": 0.8,
+      "tasks": [
+        "Upload-Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-Coverage"
+      ]
+    },
     "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-MSAN": {
       "priority": 0.8,
       "tasks": [
@@ -3393,6 +3405,36 @@
       "isolate": "compile_skia.isolate",
       "priority": 0.8
     },
+    "Build-Debian9-Clang-x86_64-Debug-Coverage": {
+      "cipd_packages": [
+        {
+          "name": "skia/bots/clang_linux",
+          "path": "clang_linux",
+          "version": "version:10"
+        }
+      ],
+      "dimensions": [
+        "cpu:x86-64-Haswell_GCE",
+        "gpu:none",
+        "os:Debian-9.1",
+        "pool:Skia"
+      ],
+      "extra_args": [
+        "--workdir",
+        "../../..",
+        "compile",
+        "repository=<(REPO)",
+        "buildername=Build-Debian9-Clang-x86_64-Debug-Coverage",
+        "swarm_out_dir=${ISOLATED_OUTDIR}",
+        "revision=<(REVISION)",
+        "patch_repo=<(PATCH_REPO)",
+        "patch_storage=<(PATCH_STORAGE)",
+        "patch_issue=<(ISSUE)",
+        "patch_set=<(PATCHSET)"
+      ],
+      "isolate": "compile_skia.isolate",
+      "priority": 0.8
+    },
     "Build-Debian9-Clang-x86_64-Debug-MSAN": {
       "cipd_packages": [
         {
@@ -14703,6 +14745,54 @@
       "max_attempts": 1,
       "priority": 0.8
     },
+    "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-Coverage": {
+      "cipd_packages": [
+        {
+          "name": "skia/bots/skimage",
+          "path": "skimage",
+          "version": "version:32"
+        },
+        {
+          "name": "skia/bots/skp",
+          "path": "skp",
+          "version": "version:89"
+        },
+        {
+          "name": "skia/bots/svg",
+          "path": "svg",
+          "version": "version:9"
+        }
+      ],
+      "dependencies": [
+        "Build-Debian9-Clang-x86_64-Debug-Coverage",
+        "Housekeeper-PerCommit-BundleRecipes"
+      ],
+      "dimensions": [
+        "cpu:x86-64-Haswell_GCE",
+        "gpu:none",
+        "os:Debian-9.1",
+        "pool:Skia"
+      ],
+      "execution_timeout_ns": 14400000000000,
+      "expiration_ns": 72000000000000,
+      "extra_args": [
+        "--workdir",
+        "../../..",
+        "test",
+        "repository=<(REPO)",
+        "buildername=Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-Coverage",
+        "swarm_out_dir=${ISOLATED_OUTDIR}",
+        "revision=<(REVISION)",
+        "patch_repo=<(PATCH_REPO)",
+        "patch_storage=<(PATCH_STORAGE)",
+        "patch_issue=<(ISSUE)",
+        "patch_set=<(PATCHSET)"
+      ],
+      "io_timeout_ns": 2400000000000,
+      "isolate": "test_skia_bundled_unix.isolate",
+      "max_attempts": 1,
+      "priority": 0.8
+    },
     "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-MSAN": {
       "cipd_packages": [
         {
@@ -24383,6 +24473,41 @@
       "isolate": "upload_dm_results.isolate",
       "priority": 0.8
     },
+    "Upload-Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-Coverage": {
+      "cipd_packages": [
+        {
+          "name": "skia/bots/clang_linux",
+          "path": "clang_linux",
+          "version": "version:10"
+        }
+      ],
+      "dependencies": [
+        "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-Coverage",
+        "Build-Debian9-Clang-x86_64-Debug-Coverage"
+      ],
+      "dimensions": [
+        "cpu:x86-64-Haswell_GCE",
+        "gpu:none",
+        "os:Debian-9.1",
+        "pool:Skia"
+      ],
+      "extra_args": [
+        "--workdir",
+        "../../..",
+        "upload_coverage_results",
+        "repository=<(REPO)",
+        "buildername=Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-Coverage",
+        "swarm_out_dir=${ISOLATED_OUTDIR}",
+        "revision=<(REVISION)",
+        "patch_repo=<(PATCH_REPO)",
+        "patch_storage=<(PATCH_STORAGE)",
+        "patch_issue=<(ISSUE)",
+        "patch_set=<(PATCHSET)",
+        "gs_bucket=skia-coverage"
+      ],
+      "isolate": "upload_coverage_results.isolate",
+      "priority": 0.8
+    },
     "Upload-Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-SK_USE_DISCARDABLE_SCALEDIMAGECACHE": {
       "dependencies": [
         "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-SK_USE_DISCARDABLE_SCALEDIMAGECACHE"
diff --git a/infra/bots/upload_coverage_results.isolate b/infra/bots/upload_coverage_results.isolate
new file mode 100644
index 0000000000..767fa3af9e
--- /dev/null
+++ b/infra/bots/upload_coverage_results.isolate
@@ -0,0 +1,10 @@
+{
+  'includes': [
+    'swarm_recipe.isolate',
+  ],
+  'variables': {
+    'files': [
+      '../../../.gclient',
+    ],
+  },
+}
author	Kevin Lubick <kjlubick@google.com>	2017-10-09 15:26:19 -0400
committer	Skia Commit-Bot <skia-commit-bot@chromium.org>	2017-10-09 19:58:34 +0000
commit	c795a4c8862dbab914561fadf7a3567c55362ae4 (patch)
tree	725e0fc4209029005230fdd4db48035d3aebd52a /infra
parent	df007e1a7ae808ad41eb2bd01f6a658c5b438285 (diff)