Add a GPU calmbench bot

Bug: skia:
Change-Id: I354c295ce04f5e478e88f724e0b17ce90496a4cc
Reviewed-on: https://skia-review.googlesource.com/65341
Commit-Queue: Yuqian Li <liyuqian@google.com>
Reviewed-by: Eric Boren <borenet@google.com>

6 files changed, 298 insertions, 26 deletions

diff --git a/infra/bots/recipes/calmbench.expected/Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All.json b/infra/bots/recipes/calmbench.expected/Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All.json
new file mode 100644
index 0000000000..ed467f8012
--- /dev/null
+++ b/infra/bots/recipes/calmbench.expected/Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All.json
@@ -0,0 +1,256 @@
+[
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "ensure-directory",
+      "--mode",
+      "0777",
+      "[CUSTOM_/_B_WORK]"
+    ],
+    "infra_step": true,
+    "name": "makedirs checkout_path"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "remove",
+      "[CUSTOM_/_B_WORK]/.gclient_entries"
+    ],
+    "infra_step": true,
+    "name": "remove [CUSTOM_/_B_WORK]/.gclient_entries"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
+      "--spec-path",
+      "cache_dir = '[CUSTOM_/_B_CACHE]'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': False, 'name': 'skia', 'url': 'https://skia.googlesource.com/skia.git'}]",
+      "--patch_root",
+      "skia",
+      "--revision_mapping_file",
+      "{\"got_revision\": \"skia\"}",
+      "--git-cache-dir",
+      "[CUSTOM_/_B_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
+      "--output_json",
+      "/path/to/tmp/json",
+      "--revision",
+      "skia@abc123"
+    ],
+    "cwd": "[CUSTOM_/_B_WORK]",
+    "env_prefixes": {
+      "PATH": [
+        "RECIPE_PACKAGE_REPO[depot_tools]"
+      ]
+    },
+    "infra_step": true,
+    "name": "bot_update",
+    "~followup_annotations": [
+      "@@@STEP_TEXT@Some step text@@@",
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"did_run\": true, @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"fixed_revisions\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"skia\": \"abc123\"@@@",
+      "@@@STEP_LOG_LINE@json.output@  }, @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"manifest\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"skia\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"repository\": \"https://fake.org/skia.git\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  }, @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"patch_failure\": false, @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"patch_root\": \"skia\", @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"properties\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"got_revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\", @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"got_revision_cp\": \"refs/heads/master@{#164710}\"@@@",
+      "@@@STEP_LOG_LINE@json.output@  }, @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"root\": \"skia\", @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"source_manifest\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"git_checkout\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"skia\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"repo_url\": \"https://fake.org/skia.git\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }@@@",
+      "@@@STEP_LOG_LINE@json.output@    }, @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"version\": 0@@@",
+      "@@@STEP_LOG_LINE@json.output@  }, @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"step_text\": \"Some step text\"@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@SET_BUILD_PROPERTY@got_revision@\"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@",
+      "@@@SET_BUILD_PROPERTY@got_revision_cp@\"refs/heads/master@{#164710}\"@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "ensure-directory",
+      "--mode",
+      "0777",
+      "[START_DIR]/tmp"
+    ],
+    "infra_step": true,
+    "name": "makedirs tmp_dir"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "copy",
+      "[CUSTOM_/_B_WORK]/skia/infra/bots/assets/skp/VERSION",
+      "/path/to/tmp/"
+    ],
+    "infra_step": true,
+    "name": "Get downloaded SKP VERSION"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "copy",
+      "42",
+      "[START_DIR]/tmp/SKP_VERSION"
+    ],
+    "infra_step": true,
+    "name": "write SKP_VERSION"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "copy",
+      "[CUSTOM_/_B_WORK]/skia/infra/bots/assets/svg/VERSION",
+      "/path/to/tmp/"
+    ],
+    "infra_step": true,
+    "name": "Get downloaded SVG VERSION"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "copy",
+      "42",
+      "[START_DIR]/tmp/SVG_VERSION"
+    ],
+    "infra_step": true,
+    "name": "write SVG_VERSION"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "[CUSTOM_/_B_WORK]/skia/bin/fetch-gn"
+    ],
+    "cwd": "[CUSTOM_/_B_WORK]/skia",
+    "env": {
+      "BUILDTYPE": "Release",
+      "CHROME_HEADLESS": "1",
+      "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
+      "SKIA_OUT": "[START_DIR]/out"
+    },
+    "infra_step": true,
+    "name": "fetch-gn"
+  },
+  {
+    "cmd": [
+      "[CUSTOM_/_B_WORK]/skia/bin/gn",
+      "gen",
+      "[START_DIR]/out/Release",
+      "--args=cc=\"[START_DIR]/clang_linux/bin/clang\" cxx=\"[START_DIR]/clang_linux/bin/clang++\" extra_cflags=[\"-B[START_DIR]/clang_linux/bin\"] extra_ldflags=[\"-B[START_DIR]/clang_linux/bin\", \"-fuse-ld=lld\"] is_debug=false"
+    ],
+    "cwd": "[CUSTOM_/_B_WORK]/skia",
+    "env": {
+      "BUILDTYPE": "Release",
+      "CHROME_HEADLESS": "1",
+      "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
+      "SKIA_OUT": "[START_DIR]/out"
+    },
+    "name": "gn gen"
+  },
+  {
+    "cmd": [
+      "ninja",
+      "-C",
+      "[START_DIR]/out/Release"
+    ],
+    "cwd": "[CUSTOM_/_B_WORK]/skia",
+    "env": {
+      "BUILDTYPE": "Release",
+      "CHROME_HEADLESS": "1",
+      "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
+      "SKIA_OUT": "[START_DIR]/out"
+    },
+    "name": "ninja"
+  },
+  {
+    "cmd": [
+      "python",
+      "tools/calmbench/calmbench.py",
+      "modified",
+      "--config",
+      "8888",
+      "--ninjadir",
+      "[START_DIR]/out/Release",
+      "--extraarg",
+      "--svgs [START_DIR]/svg --skps [START_DIR]/skp --mpd false",
+      "--writedir",
+      "[CUSTOM_[SWARM_OUT_DIR]]",
+      "--concise",
+      "--githash",
+      "9046e2e693bb92a76e972b694580e5d17ad10748",
+      "--key",
+      "arch",
+      "x86_64",
+      "compiler",
+      "Clang",
+      "cpu_or_gpu",
+      "CPU",
+      "cpu_or_gpu_value",
+      "AVX2",
+      "model",
+      "GCE",
+      "os",
+      "Debian9"
+    ],
+    "cwd": "[CUSTOM_/_B_WORK]/skia",
+    "env": {
+      "BUILDTYPE": "Release",
+      "CHROME_HEADLESS": "1",
+      "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
+      "SKIA_OUT": "[START_DIR]/out"
+    },
+    "name": "Run calmbench"
+  },
+  {
+    "name": "$result",
+    "recipe_result": null,
+    "status_code": 0
+  }
+]
\ No newline at end of file
diff --git a/infra/bots/recipes/calmbench.expected/Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release.json b/infra/bots/recipes/calmbench.expected/Calmbench-Ubuntu17-Clang-Golo-GPU-QuadroP400-x86_64-Release-All.json
index 1dd78d7d96..31657bebd3 100644
--- a/infra/bots/recipes/calmbench.expected/Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release.json
+++ b/infra/bots/recipes/calmbench.expected/Calmbench-Ubuntu17-Clang-Golo-GPU-QuadroP400-x86_64-Release-All.json
@@ -214,6 +214,8 @@
       "python",
       "tools/calmbench/calmbench.py",
       "modified",
+      "--config",
+      "gl",
       "--ninjadir",
       "[START_DIR]/out/Release",
       "--extraarg",
@@ -229,13 +231,13 @@
       "compiler",
       "Clang",
       "cpu_or_gpu",
-      "CPU",
+      "GPU",
       "cpu_or_gpu_value",
-      "AVX2",
+      "QuadroP400",
       "model",
-      "GCE",
+      "Golo",
       "os",
-      "Debian9"
+      "Ubuntu17"
     ],
     "cwd": "[CUSTOM_/_B_WORK]/skia",
     "env": {
diff --git a/infra/bots/recipes/calmbench.py b/infra/bots/recipes/calmbench.py
index f61d5c7d6a..17e3dd864a 100644
--- a/infra/bots/recipes/calmbench.py
+++ b/infra/bots/recipes/calmbench.py
@@ -31,8 +31,17 @@ def RunSteps(api):
   with api.context(cwd=api.vars.skia_dir):
     extra_arg = '--svgs %s --skps %s' % (api.flavor.device_dirs.svg_dir,
                                          api.flavor.device_dirs.skp_dir)
-    command = [ # TODO liyuqian: handle GPU config in the future
+
+    # measuring multi-picture-draw in our multi-threaded CPU test is inaccurate
+    if api.vars.builder_cfg.get('cpu_or_gpu') == 'CPU':
+      extra_arg += ' --mpd false'
+      config = "8888"
+    else:
+      config = "gl"
+
+    command = [
         'python', 'tools/calmbench/calmbench.py', 'modified',
+        '--config', config,
         '--ninjadir', api.vars.skia_out.join("Release"),
         '--extraarg', extra_arg,
         '--writedir', api.vars.swarming_out_dir,
@@ -50,21 +59,26 @@ def RunSteps(api):
   api.run.check_failure()
 
 def GenTests(api):
-  builder = "Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release"
-  test = (
-    api.test(builder) +
-    api.properties(buildername=builder,
-                   repository='https://skia.googlesource.com/skia.git',
-                   revision='abc123',
-                   path_config='kitchen',
-                   swarm_out_dir='[SWARM_OUT_DIR]') +
-    api.path.exists(
-        api.path['start_dir'].join('skia'),
-        api.path['start_dir'].join('skia', 'infra', 'bots', 'assets',
-                                   'svg', 'VERSION'),
-        api.path['start_dir'].join('skia', 'infra', 'bots', 'assets',
-                                   'skp', 'VERSION'),
+  builders = [
+    "Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All",
+    "Calmbench-Ubuntu17-Clang-Golo-GPU-QuadroP400-x86_64-Release-All",
+  ]
+
+  for builder in builders:
+    test = (
+      api.test(builder) +
+      api.properties(buildername=builder,
+                     repository='https://skia.googlesource.com/skia.git',
+                     revision='abc123',
+                     path_config='kitchen',
+                     swarm_out_dir='[SWARM_OUT_DIR]') +
+      api.path.exists(
+          api.path['start_dir'].join('skia'),
+          api.path['start_dir'].join('skia', 'infra', 'bots', 'assets',
+                                     'svg', 'VERSION'),
+          api.path['start_dir'].join('skia', 'infra', 'bots', 'assets',
+                                     'skp', 'VERSION'),
+      )
     )
-  )
 
-  yield test
+    yield test
diff --git a/infra/bots/recipes/upload_calmbench_results.expected/normal_bot.json b/infra/bots/recipes/upload_calmbench_results.expected/normal_bot.json
index 3b6d06d691..355a830add 100644
--- a/infra/bots/recipes/upload_calmbench_results.expected/normal_bot.json
+++ b/infra/bots/recipes/upload_calmbench_results.expected/normal_bot.json
@@ -151,7 +151,7 @@
       "-z",
       "json",
       "[START_DIR]/bench_modified_master.json",
-      "gs://skia-calmbench/calmbench-v1/2012/05/14/12/Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release/bench_modified_master_9046e2e693bb92a76e972b694580e5d17ad10748_1337000001.json"
+      "gs://skia-calmbench/calmbench-v1/2012/05/14/12/Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All/bench_modified_master_9046e2e693bb92a76e972b694580e5d17ad10748_1337000001.json"
     ],
     "infra_step": true,
     "name": "upload json"
@@ -163,7 +163,7 @@
       "-z",
       "csv",
       "[START_DIR]/bench_modified_master.csv",
-      "gs://skia-calmbench/calmbench-v1/2012/05/14/12/Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release/bench_modified_master_9046e2e693bb92a76e972b694580e5d17ad10748_1337000001.csv"
+      "gs://skia-calmbench/calmbench-v1/2012/05/14/12/Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All/bench_modified_master_9046e2e693bb92a76e972b694580e5d17ad10748_1337000001.csv"
     ],
     "infra_step": true,
     "name": "upload csv"
diff --git a/infra/bots/recipes/upload_calmbench_results.expected/trybot.json b/infra/bots/recipes/upload_calmbench_results.expected/trybot.json
index 3da6a499dc..6c400038bf 100644
--- a/infra/bots/recipes/upload_calmbench_results.expected/trybot.json
+++ b/infra/bots/recipes/upload_calmbench_results.expected/trybot.json
@@ -155,7 +155,7 @@
       "-z",
       "json",
       "[START_DIR]/bench_modified_master.json",
-      "gs://skia-calmbench/trybot/calmbench-v1/2012/05/14/12/Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release/456789/12/bench_modified_master_9046e2e693bb92a76e972b694580e5d17ad10748_1337000001.json"
+      "gs://skia-calmbench/trybot/calmbench-v1/2012/05/14/12/Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All/456789/12/bench_modified_master_9046e2e693bb92a76e972b694580e5d17ad10748_1337000001.json"
     ],
     "infra_step": true,
     "name": "upload json"
@@ -167,7 +167,7 @@
       "-z",
       "csv",
       "[START_DIR]/bench_modified_master.csv",
-      "gs://skia-calmbench/trybot/calmbench-v1/2012/05/14/12/Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release/456789/12/bench_modified_master_9046e2e693bb92a76e972b694580e5d17ad10748_1337000001.csv"
+      "gs://skia-calmbench/trybot/calmbench-v1/2012/05/14/12/Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All/456789/12/bench_modified_master_9046e2e693bb92a76e972b694580e5d17ad10748_1337000001.csv"
     ],
     "infra_step": true,
     "name": "upload csv"
diff --git a/infra/bots/recipes/upload_calmbench_results.py b/infra/bots/recipes/upload_calmbench_results.py
index cab067461b..df1e26c1fa 100644
--- a/infra/bots/recipes/upload_calmbench_results.py
+++ b/infra/bots/recipes/upload_calmbench_results.py
@@ -73,7 +73,7 @@ def RunSteps(api):
 
 
 def GenTests(api):
-  builder = 'Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release'
+  builder = 'Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All'
   yield (
     api.test('normal_bot') +
     api.properties(buildername=builder,