Add a new calmbench tool for noiseless nanobench

This tool can quickly check all nanobench tests including svgs
and skps (<5 minutes for CPU, ~30 minutes for GPU) and find
significant performance regressions without much noise.

This tool is not only faster (lower latency to get regression
alerts), but also more sensitive compared to our k-means and
step-fitting bots (especially for changes that only affect very
few benches). It may still miss some regressions, but the
regressions reported should be valid with very high probability.

Bug: skia:
Change-Id: I02115e6c5ab630e4c56b2087ffeb5cae1d4a618e
Reviewed-on: https://skia-review.googlesource.com/50060
Commit-Queue: Yuqian Li <liyuqian@google.com>
Reviewed-by: Eric Boren <borenet@google.com>

1 files changed, 174 insertions, 0 deletions

diff --git a/tools/calmbench/calmbench.py b/tools/calmbench/calmbench.py
new file mode 100644
index 0000000000..4d1319de1a
--- /dev/null
+++ b/tools/calmbench/calmbench.py
@@ -0,0 +1,174 @@
+#!/usr/bin/pyton
+
+# Copyright 2017 Google Inc.
+#
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+import sys
+import subprocess
+import multiprocessing
+
+from argparse import ArgumentParser
+
+
+README = """
+Simply run
+\033[36m
+    python {0} TEST_GIT_BRANCH
+\033[0m
+to see if TEST_GIT_BRANCH has performance regressions against master in 8888.
+
+To compare a specific config with svg and skp resources included, add --config
+and --extraarg option. For exampe,
+\033[36m
+    python {0} TEST_GIT_BRANCH --config gl \\
+        --extraarg "--svgs ~/Desktop/bots/svgs --skps ~/Desktop/bots/skps"
+\033[0m
+For more options, please see
+
+    python {0} --help
+""".format(__file__)
+
+
+CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
+AB_SCRIPT = "ab.py"
+
+
+def parse_args():
+  if len(sys.argv) <= 1 or sys.argv[1] == '-h' or sys.argv[1] == '--help':
+    print README
+
+  parser = ArgumentParser(
+    description='Noiselessly (hence calm) becnhmark a git branch against ' +
+                'another baseline branch (e.g., master) using multiple ' +
+                ' nanobench runs.'
+  )
+
+  default_threads = max(1, multiprocessing.cpu_count() / 2);
+  default_skiadir = os.path.normpath(CURRENT_DIR + "/../../")
+
+  config_help = (
+      'nanobench config; we currently support only one config '
+      'at a time (default: %(default)s)')
+  reps_help = (
+      'initial repititions of the nanobench run; this may be '
+      'overridden when we have many threads (default: %(default)s)')
+  extraarg_help = (
+      'nanobench args (example: --svgs ~/Desktop/bots/svgs --skps '
+      '~/Desktop/bots/skps)')
+  baseline_help = (
+      'baseline branch to compare against (default: %(default)s)')
+  basearg_help = (
+      'nanobench arg for the baseline branch; if not given, we use '
+      ' the same arg for both the test branch and the baseline branch')
+  threads_help = (
+      'number of threads to be used (default: %(default)s); '
+      'for GPU config, this will always be 1')
+  no_compile_help = (
+      'whether NOT to compile nanobench and copy it to WRITEDIR '
+      '(i.e., reuse previous nanobench compiled)')
+  skip_base_help = (
+      'whether NOT to run nanobench on baseline branch '
+      '(i.e., reuse previous baseline measurements)')
+  noinit_help = (
+      'whether to skip initial nanobench runs (default: %(default)s)')
+
+  definitions = [
+    # argname, type, default value, help
+    ['--config',    str, '8888', config_help],
+    ['--skiadir',   str, default_skiadir, 'default: %(default)s'],
+    ['--ninjadir',  str, 'out/Release', 'default: %(default)s'],
+    ['--writedir',  str, '/var/tmp', 'default: %(default)s'],
+    ['--extraarg',  str, '', extraarg_help],
+    ['--baseline',  str, 'master', baseline_help],
+    ['--basearg',   str, '', basearg_help],
+    ['--reps',      int, 2, reps_help],
+    ['--threads',   int, default_threads, threads_help]
+  ]
+
+  for d in definitions:
+    parser.add_argument(d[0], type=d[1], default=d[2], help=d[3])
+
+  parser.add_argument('branch', type=str, help="the test branch to benchmark")
+  parser.add_argument('--no-compile', dest='no_compile', action="store_true",
+      help=no_compile_help)
+  parser.add_argument('--skip-base', dest='skipbase', action="store_true",
+      help=skip_base_help)
+  parser.add_argument('--noinit', dest='noinit', action="store_true",
+      help=noinit_help)
+  parser.set_defaults(no_compile=False);
+  parser.set_defaults(skipbase=False);
+  parser.set_defaults(noinit=False);
+
+  args = parser.parse_args()
+  if not args.basearg:
+    args.basearg = args.extraarg
+
+  return args
+
+
+def nano_path(args, branch):
+  return args.writedir + '/nanobench_' + branch
+
+
+def compile_branch(args, branch):
+  print "Compiling branch %s" % args.branch
+
+  os.chdir(args.skiadir)
+  commands = [
+    ['git', 'checkout', branch],
+    ['ninja', '-C', args.ninjadir, 'nanobench'],
+    ['cp', args.ninjadir + '/nanobench', nano_path(args, branch)]
+  ]
+  for command in commands:
+    subprocess.check_call(command, cwd=args.skiadir)
+
+
+def compile_nanobench(args):
+  compile_branch(args, args.branch)
+  compile_branch(args, args.baseline)
+
+
+def main():
+  args = parse_args()
+
+  # copy in case that it will be gone after git branch switching
+  orig_ab_name = CURRENT_DIR + "/" + AB_SCRIPT
+  temp_ab_name = args.writedir + "/" + AB_SCRIPT
+  subprocess.check_call(['cp', orig_ab_name, temp_ab_name])
+
+  if not args.no_compile:
+    compile_nanobench(args)
+
+  command = [
+    'python',
+    temp_ab_name,
+    args.skiadir,
+    args.writedir,
+    args.branch + ("_A" if args.branch == args.baseline else ""),
+    args.baseline + ("_B" if args.branch == args.baseline else ""),
+    nano_path(args, args.branch),
+    nano_path(args, args.baseline),
+    args.extraarg,
+    args.basearg,
+    str(args.reps),
+    "true" if args.skipbase else "false",
+    args.config,
+    str(args.threads if args.config in ["8888", "565"] else 1),
+    "true" if args.noinit else "false"
+  ]
+
+  p = subprocess.Popen(command, cwd=args.skiadir)
+  try:
+    p.wait()
+  except KeyboardInterrupt:
+    try:
+      p.terminate()
+    except OSError as e:
+      print e
+
+
+if __name__ == "__main__":
+  main()