Output calmbench json in nanobench format

Meanwhile, we also make the bot name more similar to the perf bot.

Bug: skia:
Change-Id: I8b0e6aa2d927aa4a0ce57606345589adb6e920b3
Reviewed-on: https://skia-review.googlesource.com/64082
Reviewed-by: Eric Boren <borenet@google.com>
Reviewed-by: Joe Gregorio <jcgregorio@google.com>
Commit-Queue: Yuqian Li <liyuqian@google.com>

12 files changed, 89 insertions, 25 deletions

diff --git a/infra/bots/jobs.json b/infra/bots/jobs.json
index fca02967e3..de1721932b 100644
--- a/infra/bots/jobs.json
+++ b/infra/bots/jobs.json
@@ -89,7 +89,7 @@
   "Build-Win-MSVC-x86_64-Release",
   "Build-Win-MSVC-x86_64-Release-ANGLE",
   "Build-Win-MSVC-x86_64-Release-Vulkan",
-  "Calmbench-Debian9-Clang-x86_64-Release",
+  "Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release",
   "Housekeeper-Nightly-RecreateSKPs_Canary",
   "Housekeeper-Nightly-UpdateMetaConfig",
   "Housekeeper-PerCommit",
diff --git a/infra/bots/recipe_modules/builder_name_schema/builder_name_schema.json b/infra/bots/recipe_modules/builder_name_schema/builder_name_schema.json
index 7db2eb1db6..cc58acbc3a 100644
--- a/infra/bots/recipe_modules/builder_name_schema/builder_name_schema.json
+++ b/infra/bots/recipe_modules/builder_name_schema/builder_name_schema.json
@@ -52,7 +52,10 @@
     "Calmbench": [
       "os",
       "compiler",
-      "target_arch",
+      "model",
+      "cpu_or_gpu",
+      "cpu_or_gpu_value",
+      "arch",
       "configuration"
     ]
   },
diff --git a/infra/bots/recipe_modules/vars/examples/full.expected/Calmbench-Debian9-Clang-x86_64-Release.json b/infra/bots/recipe_modules/vars/examples/full.expected/Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release.json
index 4594f9e6b9..4594f9e6b9 100644
--- a/infra/bots/recipe_modules/vars/examples/full.expected/Calmbench-Debian9-Clang-x86_64-Release.json
+++ b/infra/bots/recipe_modules/vars/examples/full.expected/Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release.json
diff --git a/infra/bots/recipe_modules/vars/examples/full.py b/infra/bots/recipe_modules/vars/examples/full.py
index da1149c8a6..aab1e401fb 100644
--- a/infra/bots/recipe_modules/vars/examples/full.py
+++ b/infra/bots/recipe_modules/vars/examples/full.py
@@ -31,7 +31,7 @@ TEST_BUILDERS = [
   'Perf-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-ASAN',
   'Perf-Ubuntu14-GCC-GCE-CPU-AVX2-x86_64-Release-All-CT_BENCH_1k_SKPs',
   'Upload-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-Coverage',
-  'Calmbench-Debian9-Clang-x86_64-Release'
+  'Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release'
 ]
 
 
diff --git a/infra/bots/recipes/calmbench.expected/Calmbench-Debian9-Clang-x86_64-Release.json b/infra/bots/recipes/calmbench.expected/Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release.json
index 52bcbe5008..d9915eaeb5 100644
--- a/infra/bots/recipes/calmbench.expected/Calmbench-Debian9-Clang-x86_64-Release.json
+++ b/infra/bots/recipes/calmbench.expected/Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release.json
@@ -174,7 +174,7 @@
       "[CUSTOM_/_B_WORK]/skia/bin/gn",
       "gen",
       "[START_DIR]/out/Release",
-      "--args=cc=\"[START_DIR]/clang_linux/bin/clang\" cxx=\"[START_DIR]/clang_linux/bin/clang++\" extra_cflags=[\"-B[START_DIR]/clang_linux/bin\"] extra_ldflags=[\"-B[START_DIR]/clang_linux/bin\", \"-fuse-ld=lld\"] is_debug=false target_cpu=\"x86_64\""
+      "--args=cc=\"[START_DIR]/clang_linux/bin/clang\" cxx=\"[START_DIR]/clang_linux/bin/clang++\" extra_cflags=[\"-B[START_DIR]/clang_linux/bin\"] extra_ldflags=[\"-B[START_DIR]/clang_linux/bin\", \"-fuse-ld=lld\"] is_debug=false"
     ],
     "cwd": "[CUSTOM_/_B_WORK]/skia",
     "env": {
@@ -211,7 +211,22 @@
       "--svgs [START_DIR]/svg --skps [START_DIR]/skp",
       "--writedir",
       "[CUSTOM_[SWARM_OUT_DIR]]",
-      "--concise"
+      "--concise",
+      "--githash",
+      "9046e2e693bb92a76e972b694580e5d17ad10748",
+      "--key",
+      "arch",
+      "x86_64",
+      "compiler",
+      "Clang",
+      "cpu_or_gpu",
+      "CPU",
+      "cpu_or_gpu_value",
+      "AVX2",
+      "model",
+      "GCE",
+      "os",
+      "Debian9"
     ],
     "cwd": "[CUSTOM_/_B_WORK]/skia",
     "env": {
diff --git a/infra/bots/recipes/calmbench.py b/infra/bots/recipes/calmbench.py
index cceceb6f53..f61d5c7d6a 100644
--- a/infra/bots/recipes/calmbench.py
+++ b/infra/bots/recipes/calmbench.py
@@ -31,18 +31,26 @@ def RunSteps(api):
   with api.context(cwd=api.vars.skia_dir):
     extra_arg = '--svgs %s --skps %s' % (api.flavor.device_dirs.svg_dir,
                                          api.flavor.device_dirs.skp_dir)
-    command = [
+    command = [ # TODO liyuqian: handle GPU config in the future
         'python', 'tools/calmbench/calmbench.py', 'modified',
         '--ninjadir', api.vars.skia_out.join("Release"),
         '--extraarg', extra_arg,
         '--writedir', api.vars.swarming_out_dir,
-        '--concise'
+        '--concise',
+        '--githash', api.vars.got_revision,
     ]
+
+    keys_blacklist = ['configuration', 'role', 'test_filter']
+    command.append('--key')
+    for k in sorted(api.vars.builder_cfg.keys()):
+      if not k in keys_blacklist:
+        command.extend([k, api.vars.builder_cfg[k]])
+
     api.run(api.step, 'Run calmbench', cmd=command)
   api.run.check_failure()
 
 def GenTests(api):
-  builder = "Calmbench-Debian9-Clang-x86_64-Release"
+  builder = "Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release"
   test = (
     api.test(builder) +
     api.properties(buildername=builder,
diff --git a/infra/bots/recipes/upload_calmbench_results.expected/normal_bot.json b/infra/bots/recipes/upload_calmbench_results.expected/normal_bot.json
index 400f56e15c..d757cff52b 100644
--- a/infra/bots/recipes/upload_calmbench_results.expected/normal_bot.json
+++ b/infra/bots/recipes/upload_calmbench_results.expected/normal_bot.json
@@ -142,7 +142,7 @@
       "-z",
       "json",
       "[START_DIR]/bench_modified_master.json",
-      "gs://skia-calmbench/calmbench-v1/2012/05/14/12/Calmbench-Debian9-Clang-x86_64-Release/bench_modified_master_9046e2e693bb92a76e972b694580e5d17ad10748_1337000001.json"
+      "gs://skia-calmbench/calmbench-v1/2012/05/14/12/Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release/bench_modified_master_9046e2e693bb92a76e972b694580e5d17ad10748_1337000001.json"
     ],
     "infra_step": true,
     "name": "upload json"
@@ -154,7 +154,7 @@
       "-z",
       "csv",
       "[START_DIR]/bench_modified_master.csv",
-      "gs://skia-calmbench/calmbench-v1/2012/05/14/12/Calmbench-Debian9-Clang-x86_64-Release/bench_modified_master_9046e2e693bb92a76e972b694580e5d17ad10748_1337000001.csv"
+      "gs://skia-calmbench/calmbench-v1/2012/05/14/12/Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release/bench_modified_master_9046e2e693bb92a76e972b694580e5d17ad10748_1337000001.csv"
     ],
     "infra_step": true,
     "name": "upload csv"
diff --git a/infra/bots/recipes/upload_calmbench_results.expected/trybot.json b/infra/bots/recipes/upload_calmbench_results.expected/trybot.json
index 12ee743d38..a20079d996 100644
--- a/infra/bots/recipes/upload_calmbench_results.expected/trybot.json
+++ b/infra/bots/recipes/upload_calmbench_results.expected/trybot.json
@@ -146,7 +146,7 @@
       "-z",
       "json",
       "[START_DIR]/bench_modified_master.json",
-      "gs://skia-calmbench/trybot/calmbench-v1/2012/05/14/12/Calmbench-Debian9-Clang-x86_64-Release/456789/12/bench_modified_master_9046e2e693bb92a76e972b694580e5d17ad10748_1337000001.json"
+      "gs://skia-calmbench/trybot/calmbench-v1/2012/05/14/12/Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release/456789/12/bench_modified_master_9046e2e693bb92a76e972b694580e5d17ad10748_1337000001.json"
     ],
     "infra_step": true,
     "name": "upload json"
@@ -158,7 +158,7 @@
       "-z",
       "csv",
       "[START_DIR]/bench_modified_master.csv",
-      "gs://skia-calmbench/trybot/calmbench-v1/2012/05/14/12/Calmbench-Debian9-Clang-x86_64-Release/456789/12/bench_modified_master_9046e2e693bb92a76e972b694580e5d17ad10748_1337000001.csv"
+      "gs://skia-calmbench/trybot/calmbench-v1/2012/05/14/12/Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release/456789/12/bench_modified_master_9046e2e693bb92a76e972b694580e5d17ad10748_1337000001.csv"
     ],
     "infra_step": true,
     "name": "upload csv"
diff --git a/infra/bots/recipes/upload_calmbench_results.py b/infra/bots/recipes/upload_calmbench_results.py
index 2815b162be..cab067461b 100644
--- a/infra/bots/recipes/upload_calmbench_results.py
+++ b/infra/bots/recipes/upload_calmbench_results.py
@@ -73,7 +73,7 @@ def RunSteps(api):
 
 
 def GenTests(api):
-  builder = 'Calmbench-Debian9-Clang-x86_64-Release'
+  builder = 'Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release'
   yield (
     api.test('normal_bot') +
     api.properties(buildername=builder,
diff --git a/infra/bots/tasks.json b/infra/bots/tasks.json
index fea6aea8f0..c8b761f604 100644
--- a/infra/bots/tasks.json
+++ b/infra/bots/tasks.json
@@ -545,10 +545,10 @@
         "Build-Win-MSVC-x86_64-Release-Vulkan"
       ]
     },
-    "Calmbench-Debian9-Clang-x86_64-Release": {
+    "Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release": {
       "priority": 0.8,
       "tasks": [
-        "Upload-Calmbench-Debian9-Clang-x86_64-Release"
+        "Upload-Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release"
       ]
     },
     "Housekeeper-Nightly-RecreateSKPs_Canary": {
@@ -5519,7 +5519,7 @@
       "isolate": "compile_skia.isolate",
       "priority": 0.8
     },
-    "Calmbench-Debian9-Clang-x86_64-Release": {
+    "Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release": {
       "cipd_packages": [
         {
           "name": "skia/bots/clang_linux",
@@ -5542,7 +5542,7 @@
         "../../..",
         "calmbench",
         "repository=<(REPO)",
-        "buildername=Calmbench-Debian9-Clang-x86_64-Release",
+        "buildername=Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release",
         "swarm_out_dir=${ISOLATED_OUTDIR}",
         "revision=<(REVISION)",
         "patch_repo=<(PATCH_REPO)",
@@ -22939,9 +22939,9 @@
       "max_attempts": 1,
       "priority": 0.8
     },
-    "Upload-Calmbench-Debian9-Clang-x86_64-Release": {
+    "Upload-Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release": {
       "dependencies": [
-        "Calmbench-Debian9-Clang-x86_64-Release"
+        "Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release"
       ],
       "dimensions": [
         "cpu:x86-64-Haswell_GCE",
@@ -22954,7 +22954,7 @@
         "../../..",
         "upload_calmbench_results",
         "repository=<(REPO)",
-        "buildername=Calmbench-Debian9-Clang-x86_64-Release",
+        "buildername=Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release",
         "swarm_out_dir=${ISOLATED_OUTDIR}",
         "revision=<(REVISION)",
         "patch_repo=<(PATCH_REPO)",
diff --git a/tools/calmbench/ab.py b/tools/calmbench/ab.py
index e417414820..2a3944d577 100644
--- a/tools/calmbench/ab.py
+++ b/tools/calmbench/ab.py
@@ -78,6 +78,11 @@ def parse_args():
       help="If set, no verbose thread info will be printed.")
   parser.set_defaults(concise=False)
 
+  # Additional args for bots
+  BHELP = "bot specific options"
+  parser.add_argument('--githash', type=str, default=[], nargs='+', help=BHELP)
+  parser.add_argument('--keys', type=str, default=[], nargs='+', help=BHELP)
+
   args = parser.parse_args()
   args.skip_b = args.skip_b == "true"
   args.noinit = args.noinit == "true"
@@ -252,6 +257,13 @@ def format_r(r):
   return ('%6.2f' % percentage(r)) + "%"
 
 
+def normalize_r(r):
+  if r > 1.0:
+    return r - 1.0
+  else:
+    return 1.0 - 1/r
+
+
 def test():
   args = parse_args()
 
@@ -296,10 +308,26 @@ def test():
                 (format_r(r), suspect)
 
   with open("%s/bench_%s_%s.json" % (args.outdir, args.a, args.b), 'w') as f:
-    f.write(json.dumps(map(
-        lambda bench: {bench: regression(bench)},
-        suspects
-    )))
+    results = {}
+    for bench in timesA:
+      r = regression(bench) if bench in suspects else 1.0
+      results[bench] = {
+        args.config: {
+          "signed_regression": normalize_r(r),
+          "lower_quantile_ms": get_lower_upper(timesA[bench])[0] * 1e-6,
+          "upper_quantile_ms": get_lower_upper(timesA[bench])[1] * 1e-6
+        }
+      }
+
+    output = {"results": results}
+    if args.githash:
+      output["gitHash"] = args.githash
+    if args.keys:
+      keys = {}
+      for i in range(len(args.keys) / 2):
+        keys[args.keys[i * 2]] = args.keys[i * 2 + 1]
+      output["key"] = keys
+    f.write(json.dumps(output, indent=4))
     print ("\033[36mJSON results available in %s\033[0m" % f.name)
 
   with open("%s/bench_%s_%s.csv" % (args.outdir, args.a, args.b), 'w') as out:
diff --git a/tools/calmbench/calmbench.py b/tools/calmbench/calmbench.py
index 50208cc3fd..b786b5f346 100644
--- a/tools/calmbench/calmbench.py
+++ b/tools/calmbench/calmbench.py
@@ -88,7 +88,7 @@ def parse_args():
     ['--baseline',  str, 'master', baseline_help],
     ['--basearg',   str, '', basearg_help],
     ['--reps',      int, 2, reps_help],
-    ['--threads',   int, default_threads, threads_help]
+    ['--threads',   int, default_threads, threads_help],
   ]
 
   for d in definitions:
@@ -108,6 +108,11 @@ def parse_args():
   parser.set_defaults(noinit=False);
   parser.set_defaults(concise=False);
 
+  # Additional args for bots
+  BHELP = "bot specific options"
+  parser.add_argument('--githash', type=str, help=BHELP)
+  parser.add_argument('--keys', type=str, default=[], nargs='+', help=BHELP)
+
   args = parser.parse_args()
   if not args.basearg:
     args.basearg = args.extraarg
@@ -189,6 +194,11 @@ def main():
     "true" if args.noinit else "false"
   ]
 
+  if args.githash:
+    command += ['--githash', args.githash]
+  if args.keys:
+    command += (['--keys'] + args.keys)
+
   if args.concise:
     command.append("--concise")