Add Mac NDK asset, and fetch NDK on Android compile bots.

BUG=skia:
GOLD_TRYBOT_URL= https://gold.skia.org/search?issue=2284813002

Review-Url: https://codereview.chromium.org/2284813002

11 files changed, 674 insertions, 5 deletions

diff --git a/infra/bots/assets/android_ndk_darwin/VERSION b/infra/bots/assets/android_ndk_darwin/VERSION
new file mode 100644
index 0000000000..c227083464
--- /dev/null
+++ b/infra/bots/assets/android_ndk_darwin/VERSION
@@ -0,0 +1 @@
+0
\ No newline at end of file
diff --git a/infra/bots/assets/android_ndk_darwin/common.py b/infra/bots/assets/android_ndk_darwin/common.py
new file mode 100755
index 0000000000..4920c9b4fb
--- /dev/null
+++ b/infra/bots/assets/android_ndk_darwin/common.py
@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+#
+# Copyright 2016 Google Inc.
+#
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+"""Common vars used by scripts in this directory."""
+
+
+import os
+import sys
+
+FILE_DIR = os.path.dirname(os.path.abspath(__file__))
+INFRA_BOTS_DIR = os.path.realpath(os.path.join(FILE_DIR, os.pardir, os.pardir))
+
+sys.path.insert(0, INFRA_BOTS_DIR)
+from assets import assets
+
+ASSET_NAME = os.path.basename(FILE_DIR)
+
+
+def run(cmd):
+  """Run a command, eg. "upload" or "download". """
+  assets.main([cmd, ASSET_NAME] + sys.argv[1:])
diff --git a/infra/bots/assets/android_ndk_darwin/create.py b/infra/bots/assets/android_ndk_darwin/create.py
new file mode 100755
index 0000000000..818ec5ac37
--- /dev/null
+++ b/infra/bots/assets/android_ndk_darwin/create.py
@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+#
+# Copyright 2016 Google Inc.
+#
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+"""Create the asset."""
+
+
+import argparse
+import glob
+import os.path
+import shutil
+import subprocess
+
+NDK_VER = "android-ndk-r12b"
+NDK_URL = \
+    "https://dl.google.com/android/repository/%s-darwin-x86_64.zip" % NDK_VER
+
+def create_asset(target_dir):
+  """Create the asset."""
+  subprocess.check_call(["curl", NDK_URL, "-o", "ndk.zip"])
+  subprocess.check_call(["unzip", "ndk.zip", "-d", target_dir])
+  for f in glob.glob(os.path.join(target_dir, NDK_VER, "*")):
+    shutil.move(f, target_dir)
+  subprocess.check_call(["rm", "ndk.zip"])
+
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--target_dir', '-t', required=True)
+  args = parser.parse_args()
+  create_asset(args.target_dir)
+
+
+if __name__ == '__main__':
+  main()
diff --git a/infra/bots/assets/android_ndk_darwin/create_and_upload.py b/infra/bots/assets/android_ndk_darwin/create_and_upload.py
new file mode 100755
index 0000000000..1356447477
--- /dev/null
+++ b/infra/bots/assets/android_ndk_darwin/create_and_upload.py
@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+#
+# Copyright 2016 Google Inc.
+#
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+"""Create the asset and upload it."""
+
+
+import argparse
+import common
+import os
+import subprocess
+import sys
+import utils
+
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--gsutil')
+  args = parser.parse_args()
+
+  with utils.tmp_dir():
+    cwd = os.getcwd()
+    create_script = os.path.join(common.FILE_DIR, 'create.py')
+    upload_script = os.path.join(common.FILE_DIR, 'upload.py')
+
+    try:
+      subprocess.check_call(['python', create_script, '-t', cwd])
+      cmd = ['python', upload_script, '-t', cwd]
+      if args.gsutil:
+        cmd.extend(['--gsutil', args.gsutil])
+      subprocess.check_call(cmd)
+    except subprocess.CalledProcessError:
+      # Trap exceptions to avoid printing two stacktraces.
+      sys.exit(1)
+
+
+if __name__ == '__main__':
+  main()
diff --git a/infra/bots/assets/android_ndk_darwin/download.py b/infra/bots/assets/android_ndk_darwin/download.py
new file mode 100755
index 0000000000..96cc87d43f
--- /dev/null
+++ b/infra/bots/assets/android_ndk_darwin/download.py
@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+#
+# Copyright 2016 Google Inc.
+#
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+"""Download the current version of the asset."""
+
+
+import common
+
+
+if __name__ == '__main__':
+  common.run('download')
diff --git a/infra/bots/assets/android_ndk_darwin/upload.py b/infra/bots/assets/android_ndk_darwin/upload.py
new file mode 100755
index 0000000000..ba7fc8b6a1
--- /dev/null
+++ b/infra/bots/assets/android_ndk_darwin/upload.py
@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+#
+# Copyright 2016 Google Inc.
+#
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+"""Upload a new version of the asset."""
+
+
+import common
+
+
+if __name__ == '__main__':
+  common.run('upload')
diff --git a/infra/bots/recipes/swarm_trigger.expected/Build-Mac-Clang-Arm64-Release-Android.json b/infra/bots/recipes/swarm_trigger.expected/Build-Mac-Clang-Arm64-Release-Android.json
new file mode 100644
index 0000000000..49c8306c26
--- /dev/null
+++ b/infra/bots/recipes/swarm_trigger.expected/Build-Mac-Clang-Arm64-Release-Android.json
@@ -0,0 +1,488 @@
+[
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport json\nimport sys\n\nwith open(sys.argv[1]) as f:\n  content = json.load(f)\n\nprint json.dumps(content, indent=2)\n",
+      "{\"buildername\": \"Build-Mac-Clang-Arm64-Release-Android\", \"buildnumber\": 5, \"mastername\": \"client.skia\", \"path_config\": \"kitchen\", \"recipe\": \"swarm_trigger\", \"revision\": \"abc123\", \"slavename\": \"skiabot-linux-swarm-000\"}"
+    ],
+    "name": "print properties",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import json@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1]) as f:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  content = json.load(f)@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@print json.dumps(content, indent=2)@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "git",
+      "rev-parse",
+      "HEAD"
+    ],
+    "cwd": "[SLAVE_BUILD]/skia",
+    "name": "git rev-parse",
+    "stdout": "/path/to/tmp/"
+  },
+  {
+    "cmd": [
+      "python",
+      "-c",
+      "\"print 'abc123'\""
+    ],
+    "name": "got_revision",
+    "~followup_annotations": [
+      "@@@SET_BUILD_PROPERTY@got_revision@\"abc123\"@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "",
+      "[SLAVE_BUILD]/.gclient"
+    ],
+    "name": "write .gclient"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "import os\nfor r, _, files in os.walk(os.getcwd()):\n  for fname in files:\n    f = os.path.join(r, fname)\n    if os.path.isfile(f):\n      if os.access(f, os.X_OK):\n        os.chmod(f, 0755)\n      else:\n        os.chmod(f, 0644)\n"
+    ],
+    "cwd": "[SLAVE_BUILD]/skia",
+    "name": "fix filemodes",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@import os@@@",
+      "@@@STEP_LOG_LINE@python.inline@for r, _, files in os.walk(os.getcwd()):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  for fname in files:@@@",
+      "@@@STEP_LOG_LINE@python.inline@    f = os.path.join(r, fname)@@@",
+      "@@@STEP_LOG_LINE@python.inline@    if os.path.isfile(f):@@@",
+      "@@@STEP_LOG_LINE@python.inline@      if os.access(f, os.X_OK):@@@",
+      "@@@STEP_LOG_LINE@python.inline@        os.chmod(f, 0755)@@@",
+      "@@@STEP_LOG_LINE@python.inline@      else:@@@",
+      "@@@STEP_LOG_LINE@python.inline@        os.chmod(f, 0644)@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[depot_tools::git]/resources/git_setup.py",
+      "--path",
+      "[SLAVE_BUILD]/swarming.client",
+      "--url",
+      "https://chromium.googlesource.com/external/swarming.client.git"
+    ],
+    "name": "git setup (swarming_client)"
+  },
+  {
+    "cmd": [
+      "git",
+      "retry",
+      "fetch",
+      "origin",
+      "master"
+    ],
+    "cwd": "[SLAVE_BUILD]/swarming.client",
+    "env": {
+      "PATH": "RECIPE_PACKAGE_REPO[depot_tools]:%(PATH)s"
+    },
+    "name": "git fetch (swarming_client)"
+  },
+  {
+    "cmd": [
+      "git",
+      "checkout",
+      "-f",
+      "FETCH_HEAD"
+    ],
+    "cwd": "[SLAVE_BUILD]/swarming.client",
+    "name": "git checkout (swarming_client)"
+  },
+  {
+    "cmd": [
+      "git",
+      "rev-parse",
+      "HEAD"
+    ],
+    "cwd": "[SLAVE_BUILD]/swarming.client",
+    "name": "read revision",
+    "stdout": "/path/to/tmp/",
+    "~followup_annotations": [
+      "@@@STEP_TEXT@<br/>checked out 'deadbeef'<br/>@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "git",
+      "clean",
+      "-f",
+      "-d",
+      "-x"
+    ],
+    "cwd": "[SLAVE_BUILD]/swarming.client",
+    "name": "git clean (swarming_client)"
+  },
+  {
+    "cmd": [
+      "git",
+      "submodule",
+      "sync"
+    ],
+    "cwd": "[SLAVE_BUILD]/swarming.client",
+    "name": "submodule sync (swarming_client)"
+  },
+  {
+    "cmd": [
+      "git",
+      "submodule",
+      "update",
+      "--init",
+      "--recursive"
+    ],
+    "cwd": "[SLAVE_BUILD]/swarming.client",
+    "name": "submodule update (swarming_client)"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "[SLAVE_BUILD]/swarming.client/swarming.py",
+      "--version"
+    ],
+    "name": "swarming.py --version",
+    "stdout": "/path/to/tmp/",
+    "~followup_annotations": [
+      "@@@STEP_TEXT@0.8.6@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "download_from_google_storage",
+      "--no_resume",
+      "--platform=linux*",
+      "--no_auth",
+      "--bucket",
+      "chromium-luci",
+      "-d",
+      "[SLAVE_BUILD]/skia/infra/bots/tools/luci-go/linux64"
+    ],
+    "env": {
+      "PATH": "RECIPE_PACKAGE_REPO[depot_tools]:%(PATH)s"
+    },
+    "name": "download luci-go linux"
+  },
+  {
+    "cmd": [
+      "download_from_google_storage",
+      "--no_resume",
+      "--platform=darwin",
+      "--no_auth",
+      "--bucket",
+      "chromium-luci",
+      "-d",
+      "[SLAVE_BUILD]/skia/infra/bots/tools/luci-go/mac64"
+    ],
+    "env": {
+      "PATH": "RECIPE_PACKAGE_REPO[depot_tools]:%(PATH)s"
+    },
+    "name": "download luci-go mac"
+  },
+  {
+    "cmd": [
+      "download_from_google_storage",
+      "--no_resume",
+      "--platform=win32",
+      "--no_auth",
+      "--bucket",
+      "chromium-luci",
+      "-d",
+      "[SLAVE_BUILD]/skia/infra/bots/tools/luci-go/win64"
+    ],
+    "env": {
+      "PATH": "RECIPE_PACKAGE_REPO[depot_tools]:%(PATH)s"
+    },
+    "name": "download luci-go win"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n  chromium_utils.RemoveDirectory(sys.argv[1])\n",
+      "[SLAVE_BUILD]/luci-go"
+    ],
+    "env": {
+      "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts"
+    },
+    "name": "rmtree luci-go",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import os, sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  chromium_utils.RemoveDirectory(sys.argv[1])@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copytree(sys.argv[1], sys.argv[2], symlinks=bool(sys.argv[3]))\n",
+      "[SLAVE_BUILD]/skia/infra/bots/tools/luci-go",
+      "[SLAVE_BUILD]/luci-go",
+      "0"
+    ],
+    "name": "Copy Go binary"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[build::gsutil]/resources/gsutil_wrapper.py",
+      "--",
+      "RECIPE_PACKAGE_REPO[depot_tools]/gsutil.py",
+      "----",
+      "help"
+    ],
+    "name": "gsutil help"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "[SLAVE_BUILD]/skia/infra/bots/assets/android_sdk/VERSION",
+      "/path/to/tmp/"
+    ],
+    "name": "read android_sdk VERSION"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "[SLAVE_BUILD]/skia/infra/bots/assets/android_ndk_darwin/VERSION",
+      "/path/to/tmp/"
+    ],
+    "name": "read android_ndk_darwin VERSION"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n  if os.path.exists(path):\n    print \"%s exists but is not a dir\" % path\n    sys.exit(1)\n  os.makedirs(path, mode)\n",
+      "[SLAVE_BUILD]/swarming_temp_dir",
+      "511"
+    ],
+    "name": "makedirs swarming tmp dir",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys, os@@@",
+      "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@",
+      "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@",
+      "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  if os.path.exists(path):@@@",
+      "@@@STEP_LOG_LINE@python.inline@    print \"%s exists but is not a dir\" % path@@@",
+      "@@@STEP_LOG_LINE@python.inline@    sys.exit(1)@@@",
+      "@@@STEP_LOG_LINE@python.inline@  os.makedirs(path, mode)@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "{\n    \"args\": [\n        \"--isolate\", \n        \"[SLAVE_BUILD]/skia/infra/bots/compile_skia.isolate\", \n        \"--isolated\", \n        \"[SLAVE_BUILD]/swarming_temp_dir/skia-task-compile_skia.isolated\", \n        \"--config-variable\", \n        \"OS\", \n        \"Mac\", \n        \"--blacklist\", \n        \".git\", \n        \"--blacklist\", \n        \"out\", \n        \"--blacklist\", \n        \"*.pyc\", \n        \"--blacklist\", \n        \".recipe_deps\", \n        \"--extra-variable\", \n        \"WORKDIR\", \n        \"[SLAVE_BUILD]\"\n    ], \n    \"dir\": \"[SLAVE_BUILD]\", \n    \"version\": 1\n}",
+      "[SLAVE_BUILD]/swarming_temp_dir/compile_skia.isolated.gen.json"
+    ],
+    "name": "Write compile_skia.isolated.gen.json"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[build::isolate]/resources/isolate.py",
+      "[SLAVE_BUILD]/swarming.client",
+      "batcharchive",
+      "--dump-json",
+      "/path/to/tmp/json",
+      "--isolate-server",
+      "https://isolateserver.appspot.com",
+      "--verbose",
+      "[SLAVE_BUILD]/swarming_temp_dir/compile_skia.isolated.gen.json"
+    ],
+    "name": "isolate tests",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"compile_skia\": \"[dummy hash for compile_skia]\"@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"compile_skia\": \"[dummy hash for compile_skia]\"}@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "[SLAVE_BUILD]/swarming.client/swarming.py",
+      "trigger",
+      "--swarming",
+      "https://chromium-swarm.appspot.com",
+      "--isolate-server",
+      "https://isolateserver.appspot.com",
+      "--priority",
+      "90",
+      "--shards",
+      "1",
+      "--task-name",
+      "compile_skia/Mac/[dummy has/Build-Mac-Clang-Arm64-Release-Android/5",
+      "--dump-json",
+      "/path/to/tmp/json",
+      "--expiration",
+      "72000",
+      "--io-timeout",
+      "2400",
+      "--hard-timeout",
+      "14400",
+      "--dimension",
+      "gpu",
+      "none",
+      "--dimension",
+      "os",
+      "Mac",
+      "--dimension",
+      "pool",
+      "Skia",
+      "--tag",
+      "allow_milo:1",
+      "--tag",
+      "buildername:Build-Mac-Clang-Arm64-Release-Android",
+      "--tag",
+      "buildnumber:5",
+      "--tag",
+      "data:[dummy hash for compile_skia]",
+      "--tag",
+      "master:client.skia",
+      "--tag",
+      "name:compile_skia",
+      "--tag",
+      "os:Mac",
+      "--tag",
+      "revision:abc123",
+      "--tag",
+      "slavename:skiabot-linux-swarm-000",
+      "--tag",
+      "stepname:compile_skia on Mac",
+      "--idempotent",
+      "--cipd-package",
+      "android_sdk:skia/bots/android_sdk:version:0",
+      "--cipd-package",
+      "android_ndk_darwin:skia/bots/android_ndk_darwin:version:0",
+      "[dummy hash for compile_skia]",
+      "--",
+      "--workdir",
+      "../../..",
+      "swarm_compile",
+      "buildername=Build-Mac-Clang-Arm64-Release-Android",
+      "mastername=client.skia.compile",
+      "buildnumber=1",
+      "slavename=skiabot-dummy-compile-slave",
+      "reason=Triggered by Skia swarm_trigger Recipe",
+      "swarm_out_dir=${ISOLATED_OUTDIR}",
+      "revision=abc123"
+    ],
+    "name": "[trigger] compile_skia on Mac",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"base_task_name\": \"compile_skia/Mac/[dummy has/Build-Mac-Clang-Arm64-Release-Android/5\", @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"tasks\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"compile_skia/Mac/[dummy has/Build-Mac-Clang-Arm64-Release-Android/5\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"shard_index\": 0, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"task_id\": \"10000\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@STEP_LINK@shard #0@https://chromium-swarm.appspot.com/user/task/10000@@@",
+      "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/10000@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "[SLAVE_BUILD]/swarming.client/swarming.py",
+      "collect",
+      "--swarming",
+      "https://chromium-swarm.appspot.com",
+      "--decorate",
+      "--print-status-updates",
+      "--json",
+      "{\"base_task_name\": \"compile_skia/Mac/[dummy has/Build-Mac-Clang-Arm64-Release-Android/5\", \"tasks\": {\"compile_skia/Mac/[dummy has/Build-Mac-Clang-Arm64-Release-Android/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}",
+      "--task-summary-json",
+      "/path/to/tmp/json"
+    ],
+    "name": "compile_skia on Mac",
+    "~followup_annotations": [
+      "@@@STEP_TEXT@swarming pending 71s@@@",
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"shards\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@    {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"abandoned_ts\": null, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"bot_id\": \"vm30\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"completed_ts\": \"2014-09-25T01:42:00.123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"created_ts\": \"2014-09-25T01:41:00.123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"durations\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@        5.7, @@@",
+      "@@@STEP_LOG_LINE@json.output@        31.5@@@",
+      "@@@STEP_LOG_LINE@json.output@      ], @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"exit_codes\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@        0, @@@",
+      "@@@STEP_LOG_LINE@json.output@        0@@@",
+      "@@@STEP_LOG_LINE@json.output@      ], @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"failure\": false, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"id\": \"148aa78d7aa0000\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"internal_failure\": false, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"isolated_out\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"isolated\": \"abc123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"isolatedserver\": \"https://isolateserver.appspot.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"namespace\": \"default-gzip\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"view_url\": \"blah\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"modified_ts\": \"2014-09-25 01:42:00\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"name\": \"heartbeat-canary-2014-09-25_01:41:55-os=Windows\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"outputs\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"Heart beat succeeded on win32.\\n\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"Foo\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      ], @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"started_ts\": \"2014-09-25T01:42:11.123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"state\": 112, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"try_number\": 1, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"user\": \"unknown\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  ]@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@STEP_LINK@shard #0 isolated out@blah@@@",
+      "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/148aa78d7aa0000@@@"
+    ]
+  },
+  {
+    "name": "$result",
+    "recipe_result": null,
+    "status_code": 0
+  }
+]
\ No newline at end of file
diff --git a/infra/bots/recipes/swarm_trigger.expected/Build-Ubuntu-GCC-Arm64-Debug-Android_Vulkan.json b/infra/bots/recipes/swarm_trigger.expected/Build-Ubuntu-GCC-Arm64-Debug-Android_Vulkan.json
index 3f670903bf..8f64f9d38a 100644
--- a/infra/bots/recipes/swarm_trigger.expected/Build-Ubuntu-GCC-Arm64-Debug-Android_Vulkan.json
+++ b/infra/bots/recipes/swarm_trigger.expected/Build-Ubuntu-GCC-Arm64-Debug-Android_Vulkan.json
@@ -271,6 +271,16 @@
     "cmd": [
       "python",
       "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "[SLAVE_BUILD]/skia/infra/bots/assets/android_ndk_linux/VERSION",
+      "/path/to/tmp/"
+    ],
+    "name": "read android_ndk_linux VERSION"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
       "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n  if os.path.exists(path):\n    print \"%s exists but is not a dir\" % path\n    sys.exit(1)\n  os.makedirs(path, mode)\n",
       "[SLAVE_BUILD]/swarming_temp_dir",
       "511"
@@ -378,6 +388,8 @@
       "--idempotent",
       "--cipd-package",
       "android_sdk:skia/bots/android_sdk:version:0",
+      "--cipd-package",
+      "android_ndk_linux:skia/bots/android_ndk_linux:version:0",
       "[dummy hash for compile_skia]",
       "--",
       "--workdir",
diff --git a/infra/bots/recipes/swarm_trigger.expected/Test-Android-GCC-NVIDIA_Shield-GPU-TegraX1-Arm64-Debug-Vulkan.json b/infra/bots/recipes/swarm_trigger.expected/Test-Android-GCC-NVIDIA_Shield-GPU-TegraX1-Arm64-Debug-Vulkan.json
index c3e02c47ad..405be3f18e 100644
--- a/infra/bots/recipes/swarm_trigger.expected/Test-Android-GCC-NVIDIA_Shield-GPU-TegraX1-Arm64-Debug-Vulkan.json
+++ b/infra/bots/recipes/swarm_trigger.expected/Test-Android-GCC-NVIDIA_Shield-GPU-TegraX1-Arm64-Debug-Vulkan.json
@@ -271,6 +271,16 @@
     "cmd": [
       "python",
       "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "[SLAVE_BUILD]/skia/infra/bots/assets/android_ndk_linux/VERSION",
+      "/path/to/tmp/"
+    ],
+    "name": "read android_ndk_linux VERSION"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
       "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n  if os.path.exists(path):\n    print \"%s exists but is not a dir\" % path\n    sys.exit(1)\n  os.makedirs(path, mode)\n",
       "[SLAVE_BUILD]/swarming_temp_dir",
       "511"
@@ -378,6 +388,8 @@
       "--idempotent",
       "--cipd-package",
       "android_sdk:skia/bots/android_sdk:version:0",
+      "--cipd-package",
+      "android_ndk_linux:skia/bots/android_ndk_linux:version:0",
       "[dummy hash for compile_skia]",
       "--",
       "--workdir",
diff --git a/infra/bots/recipes/swarm_trigger.expected/Test-Android-GCC-Nexus7v2-GPU-Tegra3-Arm7-Release.json b/infra/bots/recipes/swarm_trigger.expected/Test-Android-GCC-Nexus7v2-GPU-Tegra3-Arm7-Release.json
index f21e61385a..780f02a1e9 100644
--- a/infra/bots/recipes/swarm_trigger.expected/Test-Android-GCC-Nexus7v2-GPU-Tegra3-Arm7-Release.json
+++ b/infra/bots/recipes/swarm_trigger.expected/Test-Android-GCC-Nexus7v2-GPU-Tegra3-Arm7-Release.json
@@ -271,6 +271,16 @@
     "cmd": [
       "python",
       "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "[SLAVE_BUILD]/skia/infra/bots/assets/android_ndk_linux/VERSION",
+      "/path/to/tmp/"
+    ],
+    "name": "read android_ndk_linux VERSION"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
       "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n  if os.path.exists(path):\n    print \"%s exists but is not a dir\" % path\n    sys.exit(1)\n  os.makedirs(path, mode)\n",
       "[SLAVE_BUILD]/swarming_temp_dir",
       "511"
@@ -378,6 +388,8 @@
       "--idempotent",
       "--cipd-package",
       "android_sdk:skia/bots/android_sdk:version:0",
+      "--cipd-package",
+      "android_ndk_linux:skia/bots/android_ndk_linux:version:0",
       "[dummy hash for compile_skia]",
       "--",
       "--workdir",
diff --git a/infra/bots/recipes/swarm_trigger.py b/infra/bots/recipes/swarm_trigger.py
index f49cd299f8..f512f770d6 100644
--- a/infra/bots/recipes/swarm_trigger.py
+++ b/infra/bots/recipes/swarm_trigger.py
@@ -34,8 +34,7 @@ DEPS = [
 TEST_BUILDERS = {
   'client.skia': {
     'skiabot-linux-swarm-000': [
-      'Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind',
-      'Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Coverage-Trybot',
+      'Build-Mac-Clang-Arm64-Release-Android',
       'Build-Mac-Clang-x86_64-Release',
       'Build-Ubuntu-GCC-Arm64-Debug-Android_Vulkan',
       'Build-Ubuntu-GCC-x86_64-Debug',
@@ -44,20 +43,22 @@ TEST_BUILDERS = {
       'Build-Ubuntu-GCC-x86_64-Release-Trybot',
       'Build-Win-MSVC-x86_64-Release',
       'Build-Win-MSVC-x86_64-Release-Vulkan',
-      'Housekeeper-PerCommit',
       'Housekeeper-Nightly-RecreateSKPs_Canary',
+      'Housekeeper-PerCommit',
       'Infra-PerCommit',
       'Perf-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Trybot',
       'Perf-Ubuntu-GCC-Golo-GPU-GT610-x86_64-Release-CT_BENCH_1k_SKPs',
-      'Test-Android-GCC-Nexus7v2-GPU-Tegra3-Arm7-Release',
       'Test-Android-GCC-NVIDIA_Shield-GPU-TegraX1-Arm64-Debug-Vulkan',
-      'Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Release',
+      'Test-Android-GCC-Nexus7v2-GPU-Tegra3-Arm7-Release',
       'Test-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release',
+      'Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Coverage-Trybot',
       'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug',
       'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-MSAN',
       'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared',
+      'Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind',
       'Test-Win8-MSVC-ShuttleA-GPU-HD7770-x86_64-Release',
       'Test-Win8-MSVC-ShuttleB-CPU-AVX2-x86_64-Release',
+      'Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Release',
     ],
   },
 }
@@ -355,6 +356,10 @@ def compile_steps_swarm(api, builder_cfg, got_revision, infrabots_dir):
   # Android bots require a toolchain.
   if 'Android' in api.properties['buildername']:
     cipd_packages.append(cipd_pkg(api, infrabots_dir, 'android_sdk'))
+    if 'Mac' in api.properties['buildername']:
+      cipd_packages.append(cipd_pkg(api, infrabots_dir, 'android_ndk_darwin'))
+    else:
+      cipd_packages.append(cipd_pkg(api, infrabots_dir, 'android_ndk_linux'))
 
   # Windows bots require a toolchain.
   if 'Win' in builder_name: