Symbolize any stacktraces in DM and nanobench

As a soft rollout, this only affects the Linux NUCS
(on Ubuntu16).

BUG=skia:6206

Change-Id: Ic314e85159cd3c0d9e55bbdce412a8d61adebb33
Reviewed-on: https://skia-review.googlesource.com/8276
Commit-Queue: Kevin Lubick <kjlubick@google.com>
Reviewed-by: Ravi Mistry <rmistry@google.com>

4 files changed, 108 insertions, 3 deletions

diff --git a/infra/bots/recipe_modules/core/resources/symbolize_stack_trace.py b/infra/bots/recipe_modules/core/resources/symbolize_stack_trace.py
new file mode 100755
index 0000000000..ccd67c7665
--- /dev/null
+++ b/infra/bots/recipe_modules/core/resources/symbolize_stack_trace.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+# pylint: disable=line-too-long
+
+import os
+import re
+import subprocess
+import sys
+
+# Run a command and symbolize anything that looks like a stacktrace in the
+# stdout/stderr. This will return with the same error code as the command.
+
+# First parameter is the current working directory, which will be stripped
+# out of stacktraces. The rest of the parameters will be fed to
+# subprocess.check_output() and should be the command and arguments that
+# will be fed in.  If any environment variables are set when running this
+# script, they will be automatically used by the call to
+# subprocess.check_output().
+
+# This wrapper function is needed to make sure stdout and stderr stay properly
+# interleaved, to assist in debugging. There are no clean ways to achieve
+# this with recipes. For example, running the dm step with parameters like
+# stdout=api.raw_io.output(), stderr=api.raw_io.output() ended up with
+# stderr and stdout being separate files, which eliminated the interwoven logs.
+# Aside from specifying stdout/stderr, there are no ways to capture or reason
+# about the logs of previous steps without using a wrapper like this.
+
+def main(basedir, cmd):
+  logs = ""
+  retcode = 0
+  try:
+    logs = subprocess.check_output(cmd, stderr=subprocess.STDOUT).strip()
+  except subprocess.CalledProcessError as e:
+    retcode = e.returncode
+    logs = e.output
+
+  # Stacktraces generally look like:
+  # /lib/x86_64-linux-gnu/libc.so.6(abort+0x16a) [0x7fa90e8d0c62]
+  # /b/s/w/irISUIyA/linux_vulkan_intel_driver_debug/./libvulkan_intel.so(+0x1f4d0a) [0x7fa909eead0a]
+  # /b/s/w/irISUIyA/out/Debug/dm() [0x17c3c5f]
+  # The stack_line regex splits those into three parts. Experimentation has
+  # shown that the address in () works best for external libraries, but our code
+  # doesn't have that. So, we capture both addresses and prefer using the first
+  # over the second, unless the first is blank or invalid. Relative offsets
+  # like abort+0x16a are ignored.
+  stack_line = r'^(?P<path>.+)\(\+?(?P<addr>.*)\) \[(?P<addr2>.+)\]'
+  # After performing addr2line, the result can be something obnoxious like:
+  # foo(bar) at /b/s/w/a39kd/Skia/out/Clang/../../src/gpu/Frobulator.cpp:13
+  # The extra_path strips off the not-useful prefix and leaves just the
+  # important src/gpu/Frobulator.cpp:13 bit.
+  extra_path = r'/.*\.\./'
+  for line in logs.splitlines():
+    line = line.strip()
+
+    m = re.search(stack_line, line)
+    if m:
+      path = m.group('path')
+      addr = m.group('addr')
+      addr2 = m.group('addr2')
+      if os.path.exists(path):
+        if not addr or not addr.startswith('0x'):
+          addr = addr2
+        sym = subprocess.check_output(['addr2line', '-Cfpe', path, addr])
+        sym = sym.strip()
+        # If addr2line doesn't return anything useful, we don't replace the
+        # original address, so the human can see it.
+        if sym and not sym.startswith('?'):
+          if path.startswith(basedir):
+            path = path[len(basedir)+1:]
+          sym = re.sub(extra_path, '', sym)
+          line = path + ' ' + sym
+
+    print line
+
+  sys.exit(retcode)
+
+
+if __name__ == '__main__':
+  if len(sys.argv) < 3:
+    print >> sys.stderr, 'USAGE: %s working_dir cmd_and_args...' % sys.argv[0]
+    sys.exit(1)
+  main(sys.argv[1], sys.argv[2:])
diff --git a/infra/bots/recipe_modules/flavor/gn_flavor.py b/infra/bots/recipe_modules/flavor/gn_flavor.py
index 27ec3f9aae..e7d8653fc3 100644
--- a/infra/bots/recipe_modules/flavor/gn_flavor.py
+++ b/infra/bots/recipe_modules/flavor/gn_flavor.py
@@ -161,4 +161,17 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils):
       # Find the MSAN-built libc++.
       env['LD_LIBRARY_PATH'] = clang_linux + '/msan'
 
-    self._run(name, cmd, env=env)
+    to_symbolize = ['dm', 'nanobench']
+    if name in to_symbolize and 'Ubuntu16' == self.m.vars.builder_cfg['os']:
+      # Convert path objects or placeholders into strings such that they can
+      # be passed to symbolize_stack_trace.py
+      args = [self.m.vars.slave_dir] + [str(x) for x in cmd]
+      self._py('symbolized %s' % name,
+               self.m.vars.infrabots_dir.join('recipe_modules', 'core',
+               'resources', 'symbolize_stack_trace.py'),
+               args=args,
+               env=env,
+               infra_step=False)
+
+    else:
+      self._run(name, cmd, env=env)
diff --git a/infra/bots/recipe_modules/perf/example.expected/Perf-Ubuntu16-Clang-NUC-GPU-IntelIris540-x86_64-Debug-Vulkan.json b/infra/bots/recipe_modules/perf/example.expected/Perf-Ubuntu16-Clang-NUC-GPU-IntelIris540-x86_64-Debug-Vulkan.json
index 64f742b6ca..53c5acfb9c 100644
--- a/infra/bots/recipe_modules/perf/example.expected/Perf-Ubuntu16-Clang-NUC-GPU-IntelIris540-x86_64-Debug-Vulkan.json
+++ b/infra/bots/recipe_modules/perf/example.expected/Perf-Ubuntu16-Clang-NUC-GPU-IntelIris540-x86_64-Debug-Vulkan.json
@@ -61,6 +61,10 @@
   },
   {
     "cmd": [
+      "python",
+      "-u",
+      "[START_DIR]/skia/infra/bots/recipe_modules/core/resources/symbolize_stack_trace.py",
+      "[START_DIR]",
       "[START_DIR]/out/Debug/nanobench",
       "--undefok",
       "-i",
@@ -112,7 +116,7 @@
       "SKIA_OUT": "[START_DIR]/out",
       "VK_ICD_FILENAMES": "[START_DIR]/linux_vulkan_intel_driver_debug/intel_icd.x86_64.json"
     },
-    "name": "nanobench"
+    "name": "symbolized nanobench"
   },
   {
     "name": "$result",
diff --git a/infra/bots/recipe_modules/sktest/example.expected/Test-Ubuntu16-Clang-NUC-GPU-IntelIris540-x86_64-Debug-Vulkan.json b/infra/bots/recipe_modules/sktest/example.expected/Test-Ubuntu16-Clang-NUC-GPU-IntelIris540-x86_64-Debug-Vulkan.json
index 4fe7ed4dd9..416540a346 100644
--- a/infra/bots/recipe_modules/sktest/example.expected/Test-Ubuntu16-Clang-NUC-GPU-IntelIris540-x86_64-Debug-Vulkan.json
+++ b/infra/bots/recipe_modules/sktest/example.expected/Test-Ubuntu16-Clang-NUC-GPU-IntelIris540-x86_64-Debug-Vulkan.json
@@ -175,6 +175,10 @@
   },
   {
     "cmd": [
+      "python",
+      "-u",
+      "[START_DIR]/skia/infra/bots/recipe_modules/core/resources/symbolize_stack_trace.py",
+      "[START_DIR]",
       "[START_DIR]/out/Debug/dm",
       "--undefok",
       "--resourcePath",
@@ -331,7 +335,7 @@
       "SKIA_OUT": "[START_DIR]/out",
       "VK_ICD_FILENAMES": "[START_DIR]/linux_vulkan_intel_driver_debug/intel_icd.x86_64.json"
     },
-    "name": "dm"
+    "name": "symbolized dm"
   },
   {
     "name": "$result",