Add CIPD asset for Valgrind

Bug: skia:6668
Change-Id: I324602b1381f701ec780c02b26775c5e9e09cb84
Reviewed-on: https://skia-review.googlesource.com/17834
Commit-Queue: Eric Boren <borenet@google.com>
Reviewed-by: Ravi Mistry <rmistry@google.com>
Reviewed-by: Mike Klein <mtklein@chromium.org>

21 files changed, 252 insertions, 21 deletions

diff --git a/infra/bots/assets/scripts/common.py b/infra/bots/assets/scripts/common.py
index 4920c9b4fb..caa0ad899c 100755
--- a/infra/bots/assets/scripts/common.py
+++ b/infra/bots/assets/scripts/common.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 #
-# Copyright 2016 Google Inc.
+# Copyright 2017 Google Inc.
 #
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
diff --git a/infra/bots/assets/scripts/create.py b/infra/bots/assets/scripts/create.py
index 4f176085fb..7f67e12694 100755
--- a/infra/bots/assets/scripts/create.py
+++ b/infra/bots/assets/scripts/create.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 #
-# Copyright 2016 Google Inc.
+# Copyright 2017 Google Inc.
 #
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
diff --git a/infra/bots/assets/scripts/create_and_upload.py b/infra/bots/assets/scripts/create_and_upload.py
index 1356447477..de56a80fa8 100755
--- a/infra/bots/assets/scripts/create_and_upload.py
+++ b/infra/bots/assets/scripts/create_and_upload.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 #
-# Copyright 2016 Google Inc.
+# Copyright 2017 Google Inc.
 #
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
diff --git a/infra/bots/assets/scripts/download.py b/infra/bots/assets/scripts/download.py
index 96cc87d43f..ca999e0378 100755
--- a/infra/bots/assets/scripts/download.py
+++ b/infra/bots/assets/scripts/download.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 #
-# Copyright 2016 Google Inc.
+# Copyright 2017 Google Inc.
 #
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
diff --git a/infra/bots/assets/scripts/upload.py b/infra/bots/assets/scripts/upload.py
index ba7fc8b6a1..bdfbda783e 100755
--- a/infra/bots/assets/scripts/upload.py
+++ b/infra/bots/assets/scripts/upload.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 #
-# Copyright 2016 Google Inc.
+# Copyright 2017 Google Inc.
 #
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
diff --git a/infra/bots/assets/valgrind/VERSION b/infra/bots/assets/valgrind/VERSION
new file mode 100644
index 0000000000..62f9457511
--- /dev/null
+++ b/infra/bots/assets/valgrind/VERSION
@@ -0,0 +1 @@
+6
\ No newline at end of file
diff --git a/infra/bots/assets/valgrind/common.py b/infra/bots/assets/valgrind/common.py
new file mode 100755
index 0000000000..caa0ad899c
--- /dev/null
+++ b/infra/bots/assets/valgrind/common.py
@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+#
+# Copyright 2017 Google Inc.
+#
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+"""Common vars used by scripts in this directory."""
+
+
+import os
+import sys
+
+FILE_DIR = os.path.dirname(os.path.abspath(__file__))
+INFRA_BOTS_DIR = os.path.realpath(os.path.join(FILE_DIR, os.pardir, os.pardir))
+
+sys.path.insert(0, INFRA_BOTS_DIR)
+from assets import assets
+
+ASSET_NAME = os.path.basename(FILE_DIR)
+
+
+def run(cmd):
+  """Run a command, eg. "upload" or "download". """
+  assets.main([cmd, ASSET_NAME] + sys.argv[1:])
diff --git a/infra/bots/assets/valgrind/create.py b/infra/bots/assets/valgrind/create.py
new file mode 100755
index 0000000000..618c2354e6
--- /dev/null
+++ b/infra/bots/assets/valgrind/create.py
@@ -0,0 +1,104 @@
+#!/usr/bin/env python
+#
+# Copyright 2017 Google Inc.
+#
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+"""Create the asset."""
+
+
+import argparse
+import common
+import grp
+import os
+import pwd
+import shutil
+import subprocess
+import sys
+import tempfile
+import urllib2
+import utils
+
+
+VALGRIND = 'valgrind-3.12.0'
+TARBALL = '%s.tar.bz2' % VALGRIND
+DOWNLOAD_URL = 'http://valgrind.org/downloads/%s' % TARBALL
+TEMP_DIR = os.path.join(tempfile.gettempdir(), 'skia-%s' % VALGRIND)
+INSTALL_DIR = os.path.join(TEMP_DIR, 'valgrind_install')
+
+
+def download_tarball():
+  with utils.chdir(TEMP_DIR):
+    if os.path.isfile(TARBALL):
+      return
+    with open(TARBALL, 'wb') as f:
+      f.write(urllib2.urlopen(DOWNLOAD_URL).read())
+
+
+def unzip_tarball():
+  with utils.chdir(TEMP_DIR):
+    if os.path.isdir(VALGRIND):
+      return
+    subprocess.check_call(['tar', 'xvjf', TARBALL])
+
+
+def create_install_dir():
+  if os.path.isdir(INSTALL_DIR):
+    return
+  os.makedirs(INSTALL_DIR)
+
+
+def build_valgrind():
+  if os.path.isfile(os.path.join(INSTALL_DIR, 'bin', 'valgrind')):
+    return
+  with utils.chdir(os.path.join(TEMP_DIR, VALGRIND)):
+    subprocess.check_call(['./configure', '--prefix=%s' % INSTALL_DIR])
+    subprocess.check_call(['make'])
+    subprocess.check_call(['make', 'install'])
+
+
+def copy_files(target_dir):
+  with utils.chdir(os.path.join(TEMP_DIR, VALGRIND)):
+    os.mkdir(os.path.join(target_dir, 'bin'))
+    shutil.copy(os.path.join(INSTALL_DIR, 'bin', 'valgrind'),
+                os.path.join(target_dir, 'bin', 'valgrind'))
+    os.mkdir(os.path.join(target_dir, 'lib'))
+    os.mkdir(os.path.join(target_dir, 'lib', 'valgrind'))
+    for lib in ['memcheck-amd64-linux']:
+      shutil.copy(os.path.join(INSTALL_DIR, 'lib', 'valgrind', lib),
+                  os.path.join(target_dir, 'lib', 'valgrind', lib))
+    for lib in ['core', 'memcheck']:
+      libname = 'vgpreload_%s-amd64-linux.so' % lib
+      shutil.copy(os.path.join(INSTALL_DIR, 'lib', 'valgrind', libname),
+                  os.path.join(target_dir, 'lib', 'valgrind', libname))
+
+    shutil.copy('default.supp',
+                os.path.join(target_dir, 'lib', 'valgrind', 'default.supp'))
+
+
+def create_asset(target_dir):
+  """Create the asset."""
+  if os.name == 'nt':
+    print 'This script does not run on Windows.'
+    sys.exit(1)
+
+  create_install_dir()
+  if not os.path.isdir(TEMP_DIR):
+    os.makedirs(TEMP_DIR)
+  download_tarball()
+  unzip_tarball()
+  build_valgrind()
+  copy_files(target_dir)
+
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--target_dir', '-t', required=True)
+  args = parser.parse_args()
+  create_asset(args.target_dir)
+
+
+if __name__ == '__main__':
+  main()
diff --git a/infra/bots/assets/valgrind/create_and_upload.py b/infra/bots/assets/valgrind/create_and_upload.py
new file mode 100755
index 0000000000..de56a80fa8
--- /dev/null
+++ b/infra/bots/assets/valgrind/create_and_upload.py
@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+#
+# Copyright 2017 Google Inc.
+#
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+"""Create the asset and upload it."""
+
+
+import argparse
+import common
+import os
+import subprocess
+import sys
+import utils
+
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--gsutil')
+  args = parser.parse_args()
+
+  with utils.tmp_dir():
+    cwd = os.getcwd()
+    create_script = os.path.join(common.FILE_DIR, 'create.py')
+    upload_script = os.path.join(common.FILE_DIR, 'upload.py')
+
+    try:
+      subprocess.check_call(['python', create_script, '-t', cwd])
+      cmd = ['python', upload_script, '-t', cwd]
+      if args.gsutil:
+        cmd.extend(['--gsutil', args.gsutil])
+      subprocess.check_call(cmd)
+    except subprocess.CalledProcessError:
+      # Trap exceptions to avoid printing two stacktraces.
+      sys.exit(1)
+
+
+if __name__ == '__main__':
+  main()
diff --git a/infra/bots/assets/valgrind/download.py b/infra/bots/assets/valgrind/download.py
new file mode 100755
index 0000000000..ca999e0378
--- /dev/null
+++ b/infra/bots/assets/valgrind/download.py
@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+#
+# Copyright 2017 Google Inc.
+#
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+"""Download the current version of the asset."""
+
+
+import common
+
+
+if __name__ == '__main__':
+  common.run('download')
diff --git a/infra/bots/assets/valgrind/upload.py b/infra/bots/assets/valgrind/upload.py
new file mode 100755
index 0000000000..bdfbda783e
--- /dev/null
+++ b/infra/bots/assets/valgrind/upload.py
@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+#
+# Copyright 2017 Google Inc.
+#
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+"""Upload a new version of the asset."""
+
+
+import common
+
+
+if __name__ == '__main__':
+  common.run('upload')
diff --git a/infra/bots/gen_tasks.go b/infra/bots/gen_tasks.go
index 01711f03a9..786acc347d 100644
--- a/infra/bots/gen_tasks.go
+++ b/infra/bots/gen_tasks.go
@@ -566,6 +566,7 @@ func test(b *specs.TasksCfgBuilder, name string, parts map[string]string, compil
 		s.ExecutionTimeout = 9 * time.Hour
 		s.Expiration = 48 * time.Hour
 		s.IoTimeout = time.Hour
+		s.CipdPackages = append(s.CipdPackages, b.MustGetCipdPackageFromAsset("valgrind"))
 	} else if strings.Contains(parts["extra_config"], "MSAN") {
 		s.ExecutionTimeout = 9 * time.Hour
 	}
diff --git a/infra/bots/recipe_modules/flavor/__init__.py b/infra/bots/recipe_modules/flavor/__init__.py
index beb97ef824..ac84d20cf3 100644
--- a/infra/bots/recipe_modules/flavor/__init__.py
+++ b/infra/bots/recipe_modules/flavor/__init__.py
@@ -5,6 +5,7 @@
 DEPS = [
   'builder_name_schema',
   'depot_tools/bot_update',
+  'env',
   'file',
   'recipe_engine/context',
   'recipe_engine/path',
diff --git a/infra/bots/recipe_modules/flavor/example.expected/Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind_AbandonGpuContext.json b/infra/bots/recipe_modules/flavor/example.expected/Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind_AbandonGpuContext.json
index 235461f8a5..e28c2077d9 100644
--- a/infra/bots/recipe_modules/flavor/example.expected/Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind_AbandonGpuContext.json
+++ b/infra/bots/recipe_modules/flavor/example.expected/Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind_AbandonGpuContext.json
@@ -188,7 +188,7 @@
   },
   {
     "cmd": [
-      "valgrind",
+      "[START_DIR]/valgrind/bin/valgrind",
       "--gen-suppressions=all",
       "--leak-check=full",
       "--track-origins=yes",
@@ -202,7 +202,8 @@
       "BUILDTYPE": "Release",
       "CHROME_HEADLESS": "1",
       "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]",
-      "SKIA_OUT": "[START_DIR]/out"
+      "SKIA_OUT": "[START_DIR]/out",
+      "VALGRIND_LIB": "[START_DIR]/valgrind/lib/valgrind"
     },
     "name": "dm"
   },
diff --git a/infra/bots/recipe_modules/flavor/valgrind_flavor.py b/infra/bots/recipe_modules/flavor/valgrind_flavor.py
index 0ebc161118..79235b0c9c 100644
--- a/infra/bots/recipe_modules/flavor/valgrind_flavor.py
+++ b/infra/bots/recipe_modules/flavor/valgrind_flavor.py
@@ -14,14 +14,17 @@ class ValgrindFlavorUtils(gn_flavor.GNFlavorUtils):
     super(ValgrindFlavorUtils, self).__init__(m)
     self._suppressions_file = self.m.vars.skia_dir.join(
         'tools', 'valgrind.supp')
+    self._valgrind_cipd_dir = self.m.vars.slave_dir.join('valgrind')
+    self._valgrind_fake_dir = self._valgrind_cipd_dir
+    self._valgrind = self._valgrind_fake_dir.join('bin', 'valgrind')
+    self._lib_dir = self._valgrind_fake_dir.join('lib', 'valgrind')
 
   def step(self, name, cmd, **kwargs):
-    new_cmd = ['valgrind', '--gen-suppressions=all', '--leak-check=full',
+    new_cmd = [self._valgrind, '--gen-suppressions=all', '--leak-check=full',
                '--track-origins=yes', '--error-exitcode=1', '--num-callers=40',
                '--suppressions=%s' % self._suppressions_file]
     path_to_app = self.out_dir.join(cmd[0])
     new_cmd.append(path_to_app)
     new_cmd.extend(cmd[1:])
-    return self.m.run(self.m.step, name, cmd=new_cmd,
-                            **kwargs)
-
+    with self.m.env({'VALGRIND_LIB': self._lib_dir}):
+      return self.m.run(self.m.step, name, cmd=new_cmd, **kwargs)
diff --git a/infra/bots/recipes/perf.expected/Perf-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind.json b/infra/bots/recipes/perf.expected/Perf-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind.json
index d1eafae27a..478575e3d2 100644
--- a/infra/bots/recipes/perf.expected/Perf-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind.json
+++ b/infra/bots/recipes/perf.expected/Perf-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind.json
@@ -95,7 +95,7 @@
   },
   {
     "cmd": [
-      "valgrind",
+      "[START_DIR]/valgrind/bin/valgrind",
       "--gen-suppressions=all",
       "--leak-check=full",
       "--track-origins=yes",
@@ -159,7 +159,8 @@
       "BUILDTYPE": "Release",
       "CHROME_HEADLESS": "1",
       "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]",
-      "SKIA_OUT": "[START_DIR]/out"
+      "SKIA_OUT": "[START_DIR]/out",
+      "VALGRIND_LIB": "[START_DIR]/valgrind/lib/valgrind"
     },
     "name": "nanobench"
   },
diff --git a/infra/bots/recipes/perf.expected/Perf-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind_AbandonGpuContext.json b/infra/bots/recipes/perf.expected/Perf-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind_AbandonGpuContext.json
index 2ac9b3ca9b..c94949cb7e 100644
--- a/infra/bots/recipes/perf.expected/Perf-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind_AbandonGpuContext.json
+++ b/infra/bots/recipes/perf.expected/Perf-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind_AbandonGpuContext.json
@@ -95,7 +95,7 @@
   },
   {
     "cmd": [
-      "valgrind",
+      "[START_DIR]/valgrind/bin/valgrind",
       "--gen-suppressions=all",
       "--leak-check=full",
       "--track-origins=yes",
@@ -161,7 +161,8 @@
       "BUILDTYPE": "Release",
       "CHROME_HEADLESS": "1",
       "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]",
-      "SKIA_OUT": "[START_DIR]/out"
+      "SKIA_OUT": "[START_DIR]/out",
+      "VALGRIND_LIB": "[START_DIR]/valgrind/lib/valgrind"
     },
     "name": "nanobench"
   },
diff --git a/infra/bots/recipes/test.expected/Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind.json b/infra/bots/recipes/test.expected/Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind.json
index e3d67b9da4..f0be3c91b1 100644
--- a/infra/bots/recipes/test.expected/Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind.json
+++ b/infra/bots/recipes/test.expected/Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind.json
@@ -95,7 +95,7 @@
   },
   {
     "cmd": [
-      "valgrind",
+      "[START_DIR]/valgrind/bin/valgrind",
       "--gen-suppressions=all",
       "--leak-check=full",
       "--track-origins=yes",
@@ -481,7 +481,8 @@
       "BUILDTYPE": "Release",
       "CHROME_HEADLESS": "1",
       "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]",
-      "SKIA_OUT": "[START_DIR]/out"
+      "SKIA_OUT": "[START_DIR]/out",
+      "VALGRIND_LIB": "[START_DIR]/valgrind/lib/valgrind"
     },
     "name": "dm"
   },
diff --git a/infra/bots/recipes/test.expected/Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind_AbandonGpuContext.json b/infra/bots/recipes/test.expected/Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind_AbandonGpuContext.json
index 3b76ccb79d..ad1ecb9c35 100644
--- a/infra/bots/recipes/test.expected/Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind_AbandonGpuContext.json
+++ b/infra/bots/recipes/test.expected/Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind_AbandonGpuContext.json
@@ -95,7 +95,7 @@
   },
   {
     "cmd": [
-      "valgrind",
+      "[START_DIR]/valgrind/bin/valgrind",
       "--gen-suppressions=all",
       "--leak-check=full",
       "--track-origins=yes",
@@ -482,7 +482,8 @@
       "BUILDTYPE": "Release",
       "CHROME_HEADLESS": "1",
       "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]",
-      "SKIA_OUT": "[START_DIR]/out"
+      "SKIA_OUT": "[START_DIR]/out",
+      "VALGRIND_LIB": "[START_DIR]/valgrind/lib/valgrind"
     },
     "name": "dm"
   },
diff --git a/infra/bots/recipes/test.expected/Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind_PreAbandonGpuContext.json b/infra/bots/recipes/test.expected/Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind_PreAbandonGpuContext.json
index f92e717125..993b379b57 100644
--- a/infra/bots/recipes/test.expected/Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind_PreAbandonGpuContext.json
+++ b/infra/bots/recipes/test.expected/Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind_PreAbandonGpuContext.json
@@ -95,7 +95,7 @@
   },
   {
     "cmd": [
-      "valgrind",
+      "[START_DIR]/valgrind/bin/valgrind",
       "--gen-suppressions=all",
       "--leak-check=full",
       "--track-origins=yes",
@@ -483,7 +483,8 @@
       "BUILDTYPE": "Release",
       "CHROME_HEADLESS": "1",
       "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]",
-      "SKIA_OUT": "[START_DIR]/out"
+      "SKIA_OUT": "[START_DIR]/out",
+      "VALGRIND_LIB": "[START_DIR]/valgrind/lib/valgrind"
     },
     "name": "dm"
   },
diff --git a/infra/bots/tasks.json b/infra/bots/tasks.json
index 59656aea93..bfc8fa58c0 100644
--- a/infra/bots/tasks.json
+++ b/infra/bots/tasks.json
@@ -13483,6 +13483,11 @@
           "name": "skia/bots/svg",
           "path": "svg",
           "version": "version:5"
+        },
+        {
+          "name": "skia/bots/valgrind",
+          "path": "valgrind",
+          "version": "version:6"
         }
       ],
       "dependencies": [
@@ -13530,6 +13535,11 @@
           "name": "skia/bots/svg",
           "path": "svg",
           "version": "version:5"
+        },
+        {
+          "name": "skia/bots/valgrind",
+          "path": "valgrind",
+          "version": "version:6"
         }
       ],
       "dependencies": [
@@ -13577,6 +13587,11 @@
           "name": "skia/bots/svg",
           "path": "svg",
           "version": "version:5"
+        },
+        {
+          "name": "skia/bots/valgrind",
+          "path": "valgrind",
+          "version": "version:6"
         }
       ],
       "dependencies": [