diff options
-rw-r--r-- | infra/bots/jobs.json | 2 | ||||
-rw-r--r-- | infra/bots/recipe_modules/builder_name_schema/builder_name_schema.json | 5 | ||||
-rw-r--r-- | infra/bots/recipe_modules/vars/examples/full.expected/Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release.json (renamed from infra/bots/recipe_modules/vars/examples/full.expected/Calmbench-Debian9-Clang-x86_64-Release.json) | 0 | ||||
-rw-r--r-- | infra/bots/recipe_modules/vars/examples/full.py | 2 | ||||
-rw-r--r-- | infra/bots/recipes/calmbench.expected/Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release.json (renamed from infra/bots/recipes/calmbench.expected/Calmbench-Debian9-Clang-x86_64-Release.json) | 19 | ||||
-rw-r--r-- | infra/bots/recipes/calmbench.py | 14 | ||||
-rw-r--r-- | infra/bots/recipes/upload_calmbench_results.expected/normal_bot.json | 4 | ||||
-rw-r--r-- | infra/bots/recipes/upload_calmbench_results.expected/trybot.json | 4 | ||||
-rw-r--r-- | infra/bots/recipes/upload_calmbench_results.py | 2 | ||||
-rw-r--r-- | infra/bots/tasks.json | 14 | ||||
-rw-r--r-- | tools/calmbench/ab.py | 36 | ||||
-rw-r--r-- | tools/calmbench/calmbench.py | 12 |
12 files changed, 89 insertions, 25 deletions
diff --git a/infra/bots/jobs.json b/infra/bots/jobs.json index fca02967e3..de1721932b 100644 --- a/infra/bots/jobs.json +++ b/infra/bots/jobs.json @@ -89,7 +89,7 @@ "Build-Win-MSVC-x86_64-Release", "Build-Win-MSVC-x86_64-Release-ANGLE", "Build-Win-MSVC-x86_64-Release-Vulkan", - "Calmbench-Debian9-Clang-x86_64-Release", + "Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release", "Housekeeper-Nightly-RecreateSKPs_Canary", "Housekeeper-Nightly-UpdateMetaConfig", "Housekeeper-PerCommit", diff --git a/infra/bots/recipe_modules/builder_name_schema/builder_name_schema.json b/infra/bots/recipe_modules/builder_name_schema/builder_name_schema.json index 7db2eb1db6..cc58acbc3a 100644 --- a/infra/bots/recipe_modules/builder_name_schema/builder_name_schema.json +++ b/infra/bots/recipe_modules/builder_name_schema/builder_name_schema.json @@ -52,7 +52,10 @@ "Calmbench": [ "os", "compiler", - "target_arch", + "model", + "cpu_or_gpu", + "cpu_or_gpu_value", + "arch", "configuration" ] }, diff --git a/infra/bots/recipe_modules/vars/examples/full.expected/Calmbench-Debian9-Clang-x86_64-Release.json b/infra/bots/recipe_modules/vars/examples/full.expected/Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release.json index 4594f9e6b9..4594f9e6b9 100644 --- a/infra/bots/recipe_modules/vars/examples/full.expected/Calmbench-Debian9-Clang-x86_64-Release.json +++ b/infra/bots/recipe_modules/vars/examples/full.expected/Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release.json diff --git a/infra/bots/recipe_modules/vars/examples/full.py b/infra/bots/recipe_modules/vars/examples/full.py index da1149c8a6..aab1e401fb 100644 --- a/infra/bots/recipe_modules/vars/examples/full.py +++ b/infra/bots/recipe_modules/vars/examples/full.py @@ -31,7 +31,7 @@ TEST_BUILDERS = [ 'Perf-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-ASAN', 'Perf-Ubuntu14-GCC-GCE-CPU-AVX2-x86_64-Release-All-CT_BENCH_1k_SKPs', 'Upload-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-Coverage', - 'Calmbench-Debian9-Clang-x86_64-Release' + 'Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release' ] diff --git a/infra/bots/recipes/calmbench.expected/Calmbench-Debian9-Clang-x86_64-Release.json b/infra/bots/recipes/calmbench.expected/Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release.json index 52bcbe5008..d9915eaeb5 100644 --- a/infra/bots/recipes/calmbench.expected/Calmbench-Debian9-Clang-x86_64-Release.json +++ b/infra/bots/recipes/calmbench.expected/Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release.json @@ -174,7 +174,7 @@ "[CUSTOM_/_B_WORK]/skia/bin/gn", "gen", "[START_DIR]/out/Release", - "--args=cc=\"[START_DIR]/clang_linux/bin/clang\" cxx=\"[START_DIR]/clang_linux/bin/clang++\" extra_cflags=[\"-B[START_DIR]/clang_linux/bin\"] extra_ldflags=[\"-B[START_DIR]/clang_linux/bin\", \"-fuse-ld=lld\"] is_debug=false target_cpu=\"x86_64\"" + "--args=cc=\"[START_DIR]/clang_linux/bin/clang\" cxx=\"[START_DIR]/clang_linux/bin/clang++\" extra_cflags=[\"-B[START_DIR]/clang_linux/bin\"] extra_ldflags=[\"-B[START_DIR]/clang_linux/bin\", \"-fuse-ld=lld\"] is_debug=false" ], "cwd": "[CUSTOM_/_B_WORK]/skia", "env": { @@ -211,7 +211,22 @@ "--svgs [START_DIR]/svg --skps [START_DIR]/skp", "--writedir", "[CUSTOM_[SWARM_OUT_DIR]]", - "--concise" + "--concise", + "--githash", + "9046e2e693bb92a76e972b694580e5d17ad10748", + "--key", + "arch", + "x86_64", + "compiler", + "Clang", + "cpu_or_gpu", + "CPU", + "cpu_or_gpu_value", + "AVX2", + "model", + "GCE", + "os", + "Debian9" ], "cwd": "[CUSTOM_/_B_WORK]/skia", "env": { diff --git a/infra/bots/recipes/calmbench.py b/infra/bots/recipes/calmbench.py index cceceb6f53..f61d5c7d6a 100644 --- a/infra/bots/recipes/calmbench.py +++ b/infra/bots/recipes/calmbench.py @@ -31,18 +31,26 @@ def RunSteps(api): with api.context(cwd=api.vars.skia_dir): extra_arg = '--svgs %s --skps %s' % (api.flavor.device_dirs.svg_dir, api.flavor.device_dirs.skp_dir) - command = [ + command = [ # TODO liyuqian: handle GPU config in the future 'python', 'tools/calmbench/calmbench.py', 'modified', '--ninjadir', api.vars.skia_out.join("Release"), '--extraarg', extra_arg, '--writedir', api.vars.swarming_out_dir, - '--concise' + '--concise', + '--githash', api.vars.got_revision, ] + + keys_blacklist = ['configuration', 'role', 'test_filter'] + command.append('--key') + for k in sorted(api.vars.builder_cfg.keys()): + if not k in keys_blacklist: + command.extend([k, api.vars.builder_cfg[k]]) + api.run(api.step, 'Run calmbench', cmd=command) api.run.check_failure() def GenTests(api): - builder = "Calmbench-Debian9-Clang-x86_64-Release" + builder = "Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release" test = ( api.test(builder) + api.properties(buildername=builder, diff --git a/infra/bots/recipes/upload_calmbench_results.expected/normal_bot.json b/infra/bots/recipes/upload_calmbench_results.expected/normal_bot.json index 400f56e15c..d757cff52b 100644 --- a/infra/bots/recipes/upload_calmbench_results.expected/normal_bot.json +++ b/infra/bots/recipes/upload_calmbench_results.expected/normal_bot.json @@ -142,7 +142,7 @@ "-z", "json", "[START_DIR]/bench_modified_master.json", - "gs://skia-calmbench/calmbench-v1/2012/05/14/12/Calmbench-Debian9-Clang-x86_64-Release/bench_modified_master_9046e2e693bb92a76e972b694580e5d17ad10748_1337000001.json" + "gs://skia-calmbench/calmbench-v1/2012/05/14/12/Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release/bench_modified_master_9046e2e693bb92a76e972b694580e5d17ad10748_1337000001.json" ], "infra_step": true, "name": "upload json" @@ -154,7 +154,7 @@ "-z", "csv", "[START_DIR]/bench_modified_master.csv", - "gs://skia-calmbench/calmbench-v1/2012/05/14/12/Calmbench-Debian9-Clang-x86_64-Release/bench_modified_master_9046e2e693bb92a76e972b694580e5d17ad10748_1337000001.csv" + "gs://skia-calmbench/calmbench-v1/2012/05/14/12/Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release/bench_modified_master_9046e2e693bb92a76e972b694580e5d17ad10748_1337000001.csv" ], "infra_step": true, "name": "upload csv" diff --git a/infra/bots/recipes/upload_calmbench_results.expected/trybot.json b/infra/bots/recipes/upload_calmbench_results.expected/trybot.json index 12ee743d38..a20079d996 100644 --- a/infra/bots/recipes/upload_calmbench_results.expected/trybot.json +++ b/infra/bots/recipes/upload_calmbench_results.expected/trybot.json @@ -146,7 +146,7 @@ "-z", "json", "[START_DIR]/bench_modified_master.json", - "gs://skia-calmbench/trybot/calmbench-v1/2012/05/14/12/Calmbench-Debian9-Clang-x86_64-Release/456789/12/bench_modified_master_9046e2e693bb92a76e972b694580e5d17ad10748_1337000001.json" + "gs://skia-calmbench/trybot/calmbench-v1/2012/05/14/12/Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release/456789/12/bench_modified_master_9046e2e693bb92a76e972b694580e5d17ad10748_1337000001.json" ], "infra_step": true, "name": "upload json" @@ -158,7 +158,7 @@ "-z", "csv", "[START_DIR]/bench_modified_master.csv", - "gs://skia-calmbench/trybot/calmbench-v1/2012/05/14/12/Calmbench-Debian9-Clang-x86_64-Release/456789/12/bench_modified_master_9046e2e693bb92a76e972b694580e5d17ad10748_1337000001.csv" + "gs://skia-calmbench/trybot/calmbench-v1/2012/05/14/12/Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release/456789/12/bench_modified_master_9046e2e693bb92a76e972b694580e5d17ad10748_1337000001.csv" ], "infra_step": true, "name": "upload csv" diff --git a/infra/bots/recipes/upload_calmbench_results.py b/infra/bots/recipes/upload_calmbench_results.py index 2815b162be..cab067461b 100644 --- a/infra/bots/recipes/upload_calmbench_results.py +++ b/infra/bots/recipes/upload_calmbench_results.py @@ -73,7 +73,7 @@ def RunSteps(api): def GenTests(api): - builder = 'Calmbench-Debian9-Clang-x86_64-Release' + builder = 'Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release' yield ( api.test('normal_bot') + api.properties(buildername=builder, diff --git a/infra/bots/tasks.json b/infra/bots/tasks.json index fea6aea8f0..c8b761f604 100644 --- a/infra/bots/tasks.json +++ b/infra/bots/tasks.json @@ -545,10 +545,10 @@ "Build-Win-MSVC-x86_64-Release-Vulkan" ] }, - "Calmbench-Debian9-Clang-x86_64-Release": { + "Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release": { "priority": 0.8, "tasks": [ - "Upload-Calmbench-Debian9-Clang-x86_64-Release" + "Upload-Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release" ] }, "Housekeeper-Nightly-RecreateSKPs_Canary": { @@ -5519,7 +5519,7 @@ "isolate": "compile_skia.isolate", "priority": 0.8 }, - "Calmbench-Debian9-Clang-x86_64-Release": { + "Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release": { "cipd_packages": [ { "name": "skia/bots/clang_linux", @@ -5542,7 +5542,7 @@ "../../..", "calmbench", "repository=<(REPO)", - "buildername=Calmbench-Debian9-Clang-x86_64-Release", + "buildername=Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release", "swarm_out_dir=${ISOLATED_OUTDIR}", "revision=<(REVISION)", "patch_repo=<(PATCH_REPO)", @@ -22939,9 +22939,9 @@ "max_attempts": 1, "priority": 0.8 }, - "Upload-Calmbench-Debian9-Clang-x86_64-Release": { + "Upload-Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release": { "dependencies": [ - "Calmbench-Debian9-Clang-x86_64-Release" + "Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release" ], "dimensions": [ "cpu:x86-64-Haswell_GCE", @@ -22954,7 +22954,7 @@ "../../..", "upload_calmbench_results", "repository=<(REPO)", - "buildername=Calmbench-Debian9-Clang-x86_64-Release", + "buildername=Calmbench-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release", "swarm_out_dir=${ISOLATED_OUTDIR}", "revision=<(REVISION)", "patch_repo=<(PATCH_REPO)", diff --git a/tools/calmbench/ab.py b/tools/calmbench/ab.py index e417414820..2a3944d577 100644 --- a/tools/calmbench/ab.py +++ b/tools/calmbench/ab.py @@ -78,6 +78,11 @@ def parse_args(): help="If set, no verbose thread info will be printed.") parser.set_defaults(concise=False) + # Additional args for bots + BHELP = "bot specific options" + parser.add_argument('--githash', type=str, default=[], nargs='+', help=BHELP) + parser.add_argument('--keys', type=str, default=[], nargs='+', help=BHELP) + args = parser.parse_args() args.skip_b = args.skip_b == "true" args.noinit = args.noinit == "true" @@ -252,6 +257,13 @@ def format_r(r): return ('%6.2f' % percentage(r)) + "%" +def normalize_r(r): + if r > 1.0: + return r - 1.0 + else: + return 1.0 - 1/r + + def test(): args = parse_args() @@ -296,10 +308,26 @@ def test(): (format_r(r), suspect) with open("%s/bench_%s_%s.json" % (args.outdir, args.a, args.b), 'w') as f: - f.write(json.dumps(map( - lambda bench: {bench: regression(bench)}, - suspects - ))) + results = {} + for bench in timesA: + r = regression(bench) if bench in suspects else 1.0 + results[bench] = { + args.config: { + "signed_regression": normalize_r(r), + "lower_quantile_ms": get_lower_upper(timesA[bench])[0] * 1e-6, + "upper_quantile_ms": get_lower_upper(timesA[bench])[1] * 1e-6 + } + } + + output = {"results": results} + if args.githash: + output["gitHash"] = args.githash + if args.keys: + keys = {} + for i in range(len(args.keys) / 2): + keys[args.keys[i * 2]] = args.keys[i * 2 + 1] + output["key"] = keys + f.write(json.dumps(output, indent=4)) print ("\033[36mJSON results available in %s\033[0m" % f.name) with open("%s/bench_%s_%s.csv" % (args.outdir, args.a, args.b), 'w') as out: diff --git a/tools/calmbench/calmbench.py b/tools/calmbench/calmbench.py index 50208cc3fd..b786b5f346 100644 --- a/tools/calmbench/calmbench.py +++ b/tools/calmbench/calmbench.py @@ -88,7 +88,7 @@ def parse_args(): ['--baseline', str, 'master', baseline_help], ['--basearg', str, '', basearg_help], ['--reps', int, 2, reps_help], - ['--threads', int, default_threads, threads_help] + ['--threads', int, default_threads, threads_help], ] for d in definitions: @@ -108,6 +108,11 @@ def parse_args(): parser.set_defaults(noinit=False); parser.set_defaults(concise=False); + # Additional args for bots + BHELP = "bot specific options" + parser.add_argument('--githash', type=str, help=BHELP) + parser.add_argument('--keys', type=str, default=[], nargs='+', help=BHELP) + args = parser.parse_args() if not args.basearg: args.basearg = args.extraarg @@ -189,6 +194,11 @@ def main(): "true" if args.noinit else "false" ] + if args.githash: + command += ['--githash', args.githash] + if args.keys: + command += (['--keys'] + args.keys) + if args.concise: command.append("--concise") |