Add flakiness data to PR comment

2 files changed, 20 insertions, 11 deletions

diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff/bm_diff.py
index c389d03adb..97fb6b9310 100755
--- a/tools/profiling/microbenchmarks/bm_diff/bm_diff.py
+++ b/tools/profiling/microbenchmarks/bm_diff/bm_diff.py
@@ -129,18 +129,23 @@ class Benchmark:
     def row(self, flds):
         return [self.final[f] if f in self.final else '' for f in flds]
 
-def _read_json(filename):
+def _read_json(filename, badfiles):
+    stripped = ".".join(filename.split(".")[:-2])
     try:
-
         with open(filename) as f:
             return json.loads(f.read())
     except ValueError, e:
+        if stripped in badfiles:
+            badfiles[stripped] += 1
+        else:
+            badfiles[stripped] = 1
         return None
 
 
 def diff(bms, loops, track, old, new):
     benchmarks = collections.defaultdict(Benchmark)
 
+    badfiles = {}
     for bm in bms:
         for loop in range(0, loops):
             for line in subprocess.check_output(
@@ -149,13 +154,13 @@ def diff(bms, loops, track, old, new):
                 stripped_line = line.strip().replace("/", "_").replace(
                     "<", "_").replace(">", "_")
                 js_new_ctr = _read_json('%s.%s.counters.%s.%d.json' %
-                                        (bm, stripped_line, new, loop))
+                                        (bm, stripped_line, new, loop), badfiles)
                 js_new_opt = _read_json('%s.%s.opt.%s.%d.json' %
-                                        (bm, stripped_line, new, loop))
+                                        (bm, stripped_line, new, loop), badfiles)
                 js_old_ctr = _read_json('%s.%s.counters.%s.%d.json' %
-                                        (bm, stripped_line, old, loop))
+                                        (bm, stripped_line, old, loop), badfiles)
                 js_old_opt = _read_json('%s.%s.opt.%s.%d.json' %
-                                        (bm, stripped_line, old, loop))
+                                        (bm, stripped_line, old, loop), badfiles)
 
                 if js_new_ctr:
                     for row in bm_json.expand_json(js_new_ctr, js_new_opt):
@@ -181,12 +186,16 @@ def diff(bms, loops, track, old, new):
     for name in sorted(benchmarks.keys()):
         if benchmarks[name].skip(): continue
         rows.append([name] + benchmarks[name].row(fields))
+    note += 'flakiness data = %s' % str(badfiles)
     if rows:
-        return tabulate.tabulate(rows, headers=headers, floatfmt='+.2f')
+        return tabulate.tabulate(rows, headers=headers, floatfmt='+.2f'), note
     else:
-        return None
+        return None, note
 
 
 if __name__ == '__main__':
     args = _args()
-    print diff(args.benchmarks, args.loops, args.track, args.old, args.new)
+    diff, note = diff(args.benchmarks, args.loops, args.track, args.old, args.new)
+    print note
+    print ""
+    print diff
diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_main.py b/tools/profiling/microbenchmarks/bm_diff/bm_main.py
index 5be9aca411..f7ef700de1 100755
--- a/tools/profiling/microbenchmarks/bm_diff/bm_main.py
+++ b/tools/profiling/microbenchmarks/bm_diff/bm_main.py
@@ -133,13 +133,13 @@ def main(args):
     bm_run.run('new', args.benchmarks, args.jobs, args.loops, args.repetitions)
     bm_run.run(old, args.benchmarks, args.jobs, args.loops, args.repetitions)
 
-    diff = bm_diff.diff(args.benchmarks, args.loops, args.track, old, 'new')
+    diff, note = bm_diff.diff(args.benchmarks, args.loops, args.track, old, 'new')
     if diff:
         text = 'Performance differences noted:\n' + diff
     else:
         text = 'No significant performance differences'
     print text
-    comment_on_pr.comment_on_pr('```\n%s\n```' % text)
+    comment_on_pr.comment_on_pr('```\n%s\n\n%s\n```' % (note, text))
 
 
 if __name__ == '__main__':