Shard GCE Coverage bot to decrease latency

Bug: skia:
Change-Id: I28d28f514b46ad0c30109beba90ae8a1bd8fc4e3
Reviewed-on: https://skia-review.googlesource.com/59961
Reviewed-by: Eric Boren <borenet@google.com>
Commit-Queue: Kevin Lubick <kjlubick@google.com>

9 files changed, 292 insertions, 73 deletions

diff --git a/infra/bots/recipes/test.expected/Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-Coverage.json b/infra/bots/recipes/test.expected/Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-shard_00_10-Coverage.json
index e82bb870ce..079594d7d0 100644
--- a/infra/bots/recipes/test.expected/Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-Coverage.json
+++ b/infra/bots/recipes/test.expected/Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-shard_00_10-Coverage.json
@@ -132,7 +132,7 @@
       "gitHash",
       "abc123",
       "builder",
-      "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-Coverage",
+      "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-shard_00_10-Coverage",
       "swarming_bot_id",
       "skia-bot-123",
       "swarming_task_id",
@@ -160,6 +160,10 @@
       "pdf",
       "--randomProcessorTest",
       "--nogpu",
+      "--shard",
+      "00",
+      "--shards",
+      "10",
       "--config",
       "8888",
       "srgb",
@@ -461,7 +465,7 @@
     "env": {
       "BUILDTYPE": "Debug",
       "CHROME_HEADLESS": "1",
-      "LLVM_PROFILE_FILE": "[CUSTOM_[SWARM_OUT_DIR]]/output.profraw",
+      "LLVM_PROFILE_FILE": "[CUSTOM_[SWARM_OUT_DIR]]/shard_00_10.profraw",
       "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
       "SKIA_OUT": "[START_DIR]/out"
     },
diff --git a/infra/bots/recipes/test.expected/Test-Ubuntu16-Clang-NUC6i5SYK-GPU-IntelIris540-x86_64-Debug-All-Vulkan_Coverage.json b/infra/bots/recipes/test.expected/Test-Ubuntu16-Clang-NUC6i5SYK-GPU-IntelIris540-x86_64-Debug-All-Vulkan_Coverage.json
index 696ee41622..263aa104bc 100644
--- a/infra/bots/recipes/test.expected/Test-Ubuntu16-Clang-NUC6i5SYK-GPU-IntelIris540-x86_64-Debug-All-Vulkan_Coverage.json
+++ b/infra/bots/recipes/test.expected/Test-Ubuntu16-Clang-NUC6i5SYK-GPU-IntelIris540-x86_64-Debug-All-Vulkan_Coverage.json
@@ -311,7 +311,7 @@
       "CHROME_HEADLESS": "1",
       "LD_LIBRARY_PATH": "[START_DIR]/linux_vulkan_intel_driver_debug:[START_DIR]/linux_vulkan_sdk/lib",
       "LIBGL_DRIVERS_PATH": "[START_DIR]/linux_vulkan_intel_driver_debug",
-      "LLVM_PROFILE_FILE": "[CUSTOM_[SWARM_OUT_DIR]]/output.profraw",
+      "LLVM_PROFILE_FILE": "[CUSTOM_[SWARM_OUT_DIR]]/All.profraw",
       "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]:[START_DIR]/linux_vulkan_sdk/bin",
       "SKIA_OUT": "[START_DIR]/out",
       "VK_ICD_FILENAMES": "[START_DIR]/linux_vulkan_intel_driver_debug/intel_icd.x86_64.json"
diff --git a/infra/bots/recipes/test.py b/infra/bots/recipes/test.py
index 61c0efbb2d..92ab274a16 100644
--- a/infra/bots/recipes/test.py
+++ b/infra/bots/recipes/test.py
@@ -229,6 +229,16 @@ def dm_flags(api, bot):
       configs = [c for c in configs if c == 'gl' or c == 'gles']
       args.extend(['--pr', 'ccpr', '--cachePathMasks', 'false'])
 
+  tf = api.vars.builder_cfg.get('test_filter')
+  if 'All' != tf:
+    # Expected format: shard_XX_YY
+    parts = tf.split('_')
+    if len(parts) == 3:
+      args.extend(['--shard', parts[1]])
+      args.extend(['--shards', parts[2]])
+    else:
+      raise Exception('Invalid task name - bad shards') #pragma: nocover
+
   args.append('--config')
   args.extend(configs)
 
@@ -876,7 +886,7 @@ TEST_BUILDERS = [
   'Test-ChromeOS-Clang-Chromebook_CB5_312T-GPU-PowerVRGX6250-arm-Debug-All',
   'Test-Chromecast-GCC-Chorizo-GPU-Cortex_A7-arm-Release-All',
   'Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-ASAN',
-  'Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-Coverage',
+  'Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-shard_00_10-Coverage',
   'Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-MSAN',
   ('Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All'
    '-SK_USE_DISCARDABLE_SCALEDIMAGECACHE'),
diff --git a/infra/bots/recipes/upload_coverage_results.expected/alternate_bucket.json b/infra/bots/recipes/upload_coverage_results.expected/alternate_bucket.json
index 8cee6d4e84..3e2bbfed52 100644
--- a/infra/bots/recipes/upload_coverage_results.expected/alternate_bucket.json
+++ b/infra/bots/recipes/upload_coverage_results.expected/alternate_bucket.json
@@ -1,22 +1,55 @@
 [
   {
     "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "glob",
+      "[START_DIR]",
+      "*.profraw"
+    ],
+    "infra_step": true,
+    "name": "find raw inputs",
+    "stdout": "/path/to/tmp/",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@glob@[START_DIR]/a.raw@@@",
+      "@@@STEP_LOG_LINE@glob@[START_DIR]/b.raw@@@",
+      "@@@STEP_LOG_LINE@glob@[START_DIR]/c.raw@@@",
+      "@@@STEP_LOG_END@glob@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "tar",
+      "-zcvf",
+      "[START_DIR]/raw_data.profraw.tar.gz",
+      "[START_DIR]/a.raw",
+      "[START_DIR]/b.raw",
+      "[START_DIR]/c.raw"
+    ],
+    "name": "create raw data archive"
+  },
+  {
+    "cmd": [
       "gsutil",
       "cp",
-      "-Z",
-      "[START_DIR]/output.profraw",
-      "gs://skia-coverage-alt/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profraw"
+      "[START_DIR]/raw_data.profraw.tar.gz",
+      "gs://skia-coverage-alt/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profraw.tar.gz"
     ],
-    "name": "upload raw data"
+    "name": "upload raw data archive"
   },
   {
     "cmd": [
       "[START_DIR]/clang_linux/bin/llvm-profdata",
       "merge",
       "-sparse",
-      "[START_DIR]/output.profraw",
       "-o",
-      "[START_DIR]/output.profdata"
+      "[START_DIR]/output.profdata",
+      "[START_DIR]/a.raw",
+      "[START_DIR]/b.raw",
+      "[START_DIR]/c.raw"
     ],
     "name": "merge and index"
   },
diff --git a/infra/bots/recipes/upload_coverage_results.expected/failed_all.json b/infra/bots/recipes/upload_coverage_results.expected/failed_all.json
index 27f91fc4dd..96be589d3b 100644
--- a/infra/bots/recipes/upload_coverage_results.expected/failed_all.json
+++ b/infra/bots/recipes/upload_coverage_results.expected/failed_all.json
@@ -1,13 +1,67 @@
 [
   {
     "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "glob",
+      "[START_DIR]",
+      "*.profraw"
+    ],
+    "infra_step": true,
+    "name": "find raw inputs",
+    "stdout": "/path/to/tmp/",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@glob@[START_DIR]/a.raw@@@",
+      "@@@STEP_LOG_LINE@glob@[START_DIR]/b.raw@@@",
+      "@@@STEP_LOG_LINE@glob@[START_DIR]/c.raw@@@",
+      "@@@STEP_LOG_END@glob@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "tar",
+      "-zcvf",
+      "[START_DIR]/raw_data.profraw.tar.gz",
+      "[START_DIR]/a.raw",
+      "[START_DIR]/b.raw",
+      "[START_DIR]/c.raw"
+    ],
+    "name": "create raw data archive"
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "[START_DIR]/raw_data.profraw.tar.gz",
+      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profraw.tar.gz"
+    ],
+    "name": "upload raw data archive"
+  },
+  {
+    "cmd": [
+      "[START_DIR]/clang_linux/bin/llvm-profdata",
+      "merge",
+      "-sparse",
+      "-o",
+      "[START_DIR]/output.profdata",
+      "[START_DIR]/a.raw",
+      "[START_DIR]/b.raw",
+      "[START_DIR]/c.raw"
+    ],
+    "name": "merge and index"
+  },
+  {
+    "cmd": [
       "gsutil",
       "cp",
       "-Z",
-      "[START_DIR]/output.profraw",
-      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profraw"
+      "[START_DIR]/output.profdata",
+      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profdata"
     ],
-    "name": "upload raw data",
+    "name": "upload parsed data",
     "~followup_annotations": [
       "step returned non-zero exit code: 1",
       "@@@STEP_FAILURE@@@"
@@ -18,10 +72,10 @@
       "gsutil",
       "cp",
       "-Z",
-      "[START_DIR]/output.profraw",
-      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profraw"
+      "[START_DIR]/output.profdata",
+      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profdata"
     ],
-    "name": "upload raw data (attempt 2)",
+    "name": "upload parsed data (attempt 2)",
     "~followup_annotations": [
       "step returned non-zero exit code: 1",
       "@@@STEP_FAILURE@@@"
@@ -32,10 +86,10 @@
       "gsutil",
       "cp",
       "-Z",
-      "[START_DIR]/output.profraw",
-      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profraw"
+      "[START_DIR]/output.profdata",
+      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profdata"
     ],
-    "name": "upload raw data (attempt 3)",
+    "name": "upload parsed data (attempt 3)",
     "~followup_annotations": [
       "step returned non-zero exit code: 1",
       "@@@STEP_FAILURE@@@"
@@ -46,10 +100,10 @@
       "gsutil",
       "cp",
       "-Z",
-      "[START_DIR]/output.profraw",
-      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profraw"
+      "[START_DIR]/output.profdata",
+      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profdata"
     ],
-    "name": "upload raw data (attempt 4)",
+    "name": "upload parsed data (attempt 4)",
     "~followup_annotations": [
       "step returned non-zero exit code: 1",
       "@@@STEP_FAILURE@@@"
@@ -60,10 +114,10 @@
       "gsutil",
       "cp",
       "-Z",
-      "[START_DIR]/output.profraw",
-      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profraw"
+      "[START_DIR]/output.profdata",
+      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profdata"
     ],
-    "name": "upload raw data (attempt 5)",
+    "name": "upload parsed data (attempt 5)",
     "~followup_annotations": [
       "step returned non-zero exit code: 1",
       "@@@STEP_FAILURE@@@"
@@ -71,7 +125,7 @@
   },
   {
     "name": "$result",
-    "reason": "Step('upload raw data (attempt 5)') failed with return_code 1",
+    "reason": "Step('upload parsed data (attempt 5)') failed with return_code 1",
     "recipe_result": null,
     "status_code": 1
   }
diff --git a/infra/bots/recipes/upload_coverage_results.expected/failed_once.json b/infra/bots/recipes/upload_coverage_results.expected/failed_once.json
index 1752047b3c..a0dc0bf034 100644
--- a/infra/bots/recipes/upload_coverage_results.expected/failed_once.json
+++ b/infra/bots/recipes/upload_coverage_results.expected/failed_once.json
@@ -1,36 +1,55 @@
 [
   {
     "cmd": [
-      "gsutil",
-      "cp",
-      "-Z",
-      "[START_DIR]/output.profraw",
-      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profraw"
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "glob",
+      "[START_DIR]",
+      "*.profraw"
     ],
-    "name": "upload raw data",
+    "infra_step": true,
+    "name": "find raw inputs",
+    "stdout": "/path/to/tmp/",
     "~followup_annotations": [
-      "step returned non-zero exit code: 1",
-      "@@@STEP_FAILURE@@@"
+      "@@@STEP_LOG_LINE@glob@[START_DIR]/a.raw@@@",
+      "@@@STEP_LOG_LINE@glob@[START_DIR]/b.raw@@@",
+      "@@@STEP_LOG_LINE@glob@[START_DIR]/c.raw@@@",
+      "@@@STEP_LOG_END@glob@@@"
     ]
   },
   {
     "cmd": [
+      "tar",
+      "-zcvf",
+      "[START_DIR]/raw_data.profraw.tar.gz",
+      "[START_DIR]/a.raw",
+      "[START_DIR]/b.raw",
+      "[START_DIR]/c.raw"
+    ],
+    "name": "create raw data archive"
+  },
+  {
+    "cmd": [
       "gsutil",
       "cp",
-      "-Z",
-      "[START_DIR]/output.profraw",
-      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profraw"
+      "[START_DIR]/raw_data.profraw.tar.gz",
+      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profraw.tar.gz"
     ],
-    "name": "upload raw data (attempt 2)"
+    "name": "upload raw data archive"
   },
   {
     "cmd": [
       "[START_DIR]/clang_linux/bin/llvm-profdata",
       "merge",
       "-sparse",
-      "[START_DIR]/output.profraw",
       "-o",
-      "[START_DIR]/output.profdata"
+      "[START_DIR]/output.profdata",
+      "[START_DIR]/a.raw",
+      "[START_DIR]/b.raw",
+      "[START_DIR]/c.raw"
     ],
     "name": "merge and index"
   },
@@ -42,7 +61,21 @@
       "[START_DIR]/output.profdata",
       "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profdata"
     ],
-    "name": "upload parsed data"
+    "name": "upload parsed data",
+    "~followup_annotations": [
+      "step returned non-zero exit code: 1",
+      "@@@STEP_FAILURE@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-Z",
+      "[START_DIR]/output.profdata",
+      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profdata"
+    ],
+    "name": "upload parsed data (attempt 2)"
   },
   {
     "cmd": [
diff --git a/infra/bots/recipes/upload_coverage_results.expected/normal_bot.json b/infra/bots/recipes/upload_coverage_results.expected/normal_bot.json
index 84b148243b..ced66fbe6f 100644
--- a/infra/bots/recipes/upload_coverage_results.expected/normal_bot.json
+++ b/infra/bots/recipes/upload_coverage_results.expected/normal_bot.json
@@ -1,22 +1,55 @@
 [
   {
     "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "glob",
+      "[START_DIR]",
+      "*.profraw"
+    ],
+    "infra_step": true,
+    "name": "find raw inputs",
+    "stdout": "/path/to/tmp/",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@glob@[START_DIR]/a.raw@@@",
+      "@@@STEP_LOG_LINE@glob@[START_DIR]/b.raw@@@",
+      "@@@STEP_LOG_LINE@glob@[START_DIR]/c.raw@@@",
+      "@@@STEP_LOG_END@glob@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "tar",
+      "-zcvf",
+      "[START_DIR]/raw_data.profraw.tar.gz",
+      "[START_DIR]/a.raw",
+      "[START_DIR]/b.raw",
+      "[START_DIR]/c.raw"
+    ],
+    "name": "create raw data archive"
+  },
+  {
+    "cmd": [
       "gsutil",
       "cp",
-      "-Z",
-      "[START_DIR]/output.profraw",
-      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profraw"
+      "[START_DIR]/raw_data.profraw.tar.gz",
+      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profraw.tar.gz"
     ],
-    "name": "upload raw data"
+    "name": "upload raw data archive"
   },
   {
     "cmd": [
       "[START_DIR]/clang_linux/bin/llvm-profdata",
       "merge",
       "-sparse",
-      "[START_DIR]/output.profraw",
       "-o",
-      "[START_DIR]/output.profdata"
+      "[START_DIR]/output.profdata",
+      "[START_DIR]/a.raw",
+      "[START_DIR]/b.raw",
+      "[START_DIR]/c.raw"
     ],
     "name": "merge and index"
   },
diff --git a/infra/bots/recipes/upload_coverage_results.expected/trybot.json b/infra/bots/recipes/upload_coverage_results.expected/trybot.json
index 2e12fddea1..938b3280b4 100644
--- a/infra/bots/recipes/upload_coverage_results.expected/trybot.json
+++ b/infra/bots/recipes/upload_coverage_results.expected/trybot.json
@@ -1,22 +1,55 @@
 [
   {
     "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "glob",
+      "[START_DIR]",
+      "*.profraw"
+    ],
+    "infra_step": true,
+    "name": "find raw inputs",
+    "stdout": "/path/to/tmp/",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@glob@[START_DIR]/a.raw@@@",
+      "@@@STEP_LOG_LINE@glob@[START_DIR]/b.raw@@@",
+      "@@@STEP_LOG_LINE@glob@[START_DIR]/c.raw@@@",
+      "@@@STEP_LOG_END@glob@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "tar",
+      "-zcvf",
+      "[START_DIR]/raw_data.profraw.tar.gz",
+      "[START_DIR]/a.raw",
+      "[START_DIR]/b.raw",
+      "[START_DIR]/c.raw"
+    ],
+    "name": "create raw data archive"
+  },
+  {
+    "cmd": [
       "gsutil",
       "cp",
-      "-Z",
-      "[START_DIR]/output.profraw",
-      "gs://skia-coverage/trybot/456789/12/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profraw"
+      "[START_DIR]/raw_data.profraw.tar.gz",
+      "gs://skia-coverage/trybot/456789/12/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profraw.tar.gz"
     ],
-    "name": "upload raw data"
+    "name": "upload raw data archive"
   },
   {
     "cmd": [
       "[START_DIR]/clang_linux/bin/llvm-profdata",
       "merge",
       "-sparse",
-      "[START_DIR]/output.profraw",
       "-o",
-      "[START_DIR]/output.profdata"
+      "[START_DIR]/output.profdata",
+      "[START_DIR]/a.raw",
+      "[START_DIR]/b.raw",
+      "[START_DIR]/c.raw"
     ],
     "name": "merge and index"
   },
diff --git a/infra/bots/recipes/upload_coverage_results.py b/infra/bots/recipes/upload_coverage_results.py
index f083dd9672..fd9a24ec8e 100644
--- a/infra/bots/recipes/upload_coverage_results.py
+++ b/infra/bots/recipes/upload_coverage_results.py
@@ -10,21 +10,26 @@ import calendar
 
 
 DEPS = [
+  'gsutil',
+  'recipe_engine/file',
   'recipe_engine/json',
   'recipe_engine/path',
   'recipe_engine/properties',
+  'recipe_engine/python',
+  'recipe_engine/raw_io',
   'recipe_engine/step',
   'recipe_engine/time',
-  'gsutil',
 ]
 
 
 TRY_JOB_FOLDER = 'trybot/%s/%s/' # % (issue_number, patchset_number)
 COMMIT_FOLDER = 'commit/%s/'      # % (git_revision)
 
-RAW_FILE = '%s.profraw'
+RAW_FILE = '*.profraw'
 PARSED_FILE = '%s.profdata'
 SUMMARY_FILE = '%s.summary'
+
+COVERAGE_RAW_ARCHIVE = '%s.profraw.tar.gz'
 # Text is an easier format to read with machines (e.g. for Gerrit).
 COVERAGE_TEXT_FILE = '%s.text.tar'
 # HTML is a quick and dirty browsable format. (e.g. for coverage.skia.org)
@@ -41,8 +46,13 @@ def RunSteps(api):
   builder_name = api.properties['buildername']
   bucket = api.properties['gs_bucket']
 
-  # The raw data is brought in as an isolated input.
-  raw_data = api.path['start_dir'].join('output.profraw')
+  # The raw data files are brought in as isolated inputs. It is possible
+  # for there to be 1 if the coverage task wasn't broken up.
+  raw_inputs = api.file.glob_paths('find raw inputs', api.path['start_dir'],
+                                   RAW_FILE,
+                                   test_data=['a.raw', 'b.raw', 'c.raw'])
+
+
   # The instrumented executable is brought in as an isolated input.
   executable = api.path['start_dir'].join('out','Debug','dm')
   # clang_dir is brought in via CIPD.
@@ -56,19 +66,28 @@ def RunSteps(api):
   if issue and patchset:
     path = TRY_JOB_FOLDER % (issue, patchset)
 
-  gcs_file = RAW_FILE % builder_name
-  api.gsutil.cp('raw data', raw_data,
-                   'gs://%s/%s%s' % (bucket, path, gcs_file), ['-Z'])
+  # Upload the raw files, tarred together to decrease upload time and
+  # improve compression.
+  tar_file = api.path['start_dir'].join('raw_data.profraw.tar.gz')
+  cmd = ['tar', '-zcvf', tar_file]
+  cmd.extend(raw_inputs)
+  api.step('create raw data archive', cmd=cmd)
+
+  gcs_file = COVERAGE_RAW_ARCHIVE % builder_name
+  api.gsutil.cp('raw data archive', tar_file,
+                'gs://%s/%s%s' % (bucket, path, gcs_file))
 
-  # Merge and Index the data.
+  # Merge all the raw data files together, then index the data.
+  # This creates one cohesive
   indexed_data = api.path['start_dir'].join('output.profdata')
+  cmd = [clang_dir.join('llvm-profdata'),
+         'merge',
+         '-sparse',
+         '-o',
+         indexed_data]
+  cmd.extend(raw_inputs)
   api.step('merge and index',
-           cmd=[clang_dir.join('llvm-profdata'),
-               'merge',
-               '-sparse',
-               raw_data,
-               '-o',
-               indexed_data ])
+           cmd=cmd)
 
   gcs_file = PARSED_FILE % builder_name
   api.gsutil.cp('parsed data', indexed_data,
@@ -149,7 +168,7 @@ def GenTests(api):
                    gs_bucket='skia-coverage',
                    revision='abc123',
                    path_config='kitchen') +
-    api.step_data('upload raw data', retcode=1)
+    api.step_data('upload parsed data', retcode=1)
   )
 
   yield (
@@ -158,11 +177,11 @@ def GenTests(api):
                    gs_bucket='skia-coverage',
                    revision='abc123',
                    path_config='kitchen') +
-    api.step_data('upload raw data', retcode=1) +
-    api.step_data('upload raw data (attempt 2)', retcode=1) +
-    api.step_data('upload raw data (attempt 3)', retcode=1) +
-    api.step_data('upload raw data (attempt 4)', retcode=1) +
-    api.step_data('upload raw data (attempt 5)', retcode=1)
+    api.step_data('upload parsed data', retcode=1) +
+    api.step_data('upload parsed data (attempt 2)', retcode=1) +
+    api.step_data('upload parsed data (attempt 3)', retcode=1) +
+    api.step_data('upload parsed data (attempt 4)', retcode=1) +
+    api.step_data('upload parsed data (attempt 5)', retcode=1)
   )
 
   yield (