aboutsummaryrefslogtreecommitdiffhomepage
path: root/infra/base-images/base-sanitizer-libs-builder
diff options
context:
space:
mode:
authorGravatar Max Moroz <mmoroz@chromium.org>2020-07-23 16:36:58 -0700
committerGravatar GitHub <noreply@github.com>2020-07-23 16:36:58 -0700
commita9d0062a554b1598fbbb71ccd1c38d46c3c80c45 (patch)
treef68cc4eca757573207b1ace82a945ccb205f9ee9 /infra/base-images/base-sanitizer-libs-builder
parent6892a59b5889400bdb45247f9e4fb48e6a0694b3 (diff)
[infra] Rename base-msan-builder image to base-sanitizer-libs-builder (#3388). (#4187)
* [infra] Rename base-msan-builder image to base-sanitizer-builder (#3388). * rename to base-sanitizer-libs-builder
Diffstat (limited to 'infra/base-images/base-sanitizer-libs-builder')
-rw-r--r--infra/base-images/base-sanitizer-libs-builder/Dockerfile28
-rwxr-xr-xinfra/base-images/base-sanitizer-libs-builder/compiler_wrapper.py175
-rw-r--r--infra/base-images/base-sanitizer-libs-builder/compiler_wrapper_test.py42
-rwxr-xr-xinfra/base-images/base-sanitizer-libs-builder/msan_build.py456
-rw-r--r--infra/base-images/base-sanitizer-libs-builder/packages/__init__.py0
-rw-r--r--infra/base-images/base-sanitizer-libs-builder/packages/boost1_58.py29
-rw-r--r--infra/base-images/base-sanitizer-libs-builder/packages/gnutls28.py37
-rw-r--r--infra/base-images/base-sanitizer-libs-builder/packages/libgcrypt20.py37
-rw-r--r--infra/base-images/base-sanitizer-libs-builder/packages/mesa.py28
-rw-r--r--infra/base-images/base-sanitizer-libs-builder/packages/nettle.py41
-rw-r--r--infra/base-images/base-sanitizer-libs-builder/packages/openssl.py42
-rw-r--r--infra/base-images/base-sanitizer-libs-builder/packages/package.py82
-rw-r--r--infra/base-images/base-sanitizer-libs-builder/packages/pixman.py42
-rw-r--r--infra/base-images/base-sanitizer-libs-builder/packages/pixman_blacklist.txt1
-rw-r--r--infra/base-images/base-sanitizer-libs-builder/packages/pulseaudio.py42
-rw-r--r--infra/base-images/base-sanitizer-libs-builder/packages/pulseaudio_fix_android.patch39
-rw-r--r--infra/base-images/base-sanitizer-libs-builder/packages/sqlite3.py32
-rw-r--r--infra/base-images/base-sanitizer-libs-builder/packages/systemd.py42
-rw-r--r--infra/base-images/base-sanitizer-libs-builder/packages/tar.py28
-rwxr-xr-xinfra/base-images/base-sanitizer-libs-builder/patch_build.py143
-rw-r--r--infra/base-images/base-sanitizer-libs-builder/wrapper_utils.py47
21 files changed, 1413 insertions, 0 deletions
diff --git a/infra/base-images/base-sanitizer-libs-builder/Dockerfile b/infra/base-images/base-sanitizer-libs-builder/Dockerfile
new file mode 100644
index 00000000..315730b6
--- /dev/null
+++ b/infra/base-images/base-sanitizer-libs-builder/Dockerfile
@@ -0,0 +1,28 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+FROM gcr.io/oss-fuzz-base/base-clang
+RUN sed -i -r 's/#\s*deb-src/deb-src/g' /etc/apt/sources.list
+RUN apt-get update && apt-get install -y python dpkg-dev patchelf python-apt zip
+
+# Take all libraries from lib/msan
+RUN cp -R /usr/msan/lib/* /usr/lib/
+
+COPY compiler_wrapper.py msan_build.py patch_build.py wrapper_utils.py /usr/local/bin/
+COPY packages /usr/local/bin/packages
+
+RUN mkdir /msan
+WORKDIR /msan
diff --git a/infra/base-images/base-sanitizer-libs-builder/compiler_wrapper.py b/infra/base-images/base-sanitizer-libs-builder/compiler_wrapper.py
new file mode 100755
index 00000000..04aa4207
--- /dev/null
+++ b/infra/base-images/base-sanitizer-libs-builder/compiler_wrapper.py
@@ -0,0 +1,175 @@
+#!/usr/bin/env python
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+from __future__ import print_function
+import os
+import subprocess
+import sys
+
+import msan_build
+
+GCC_ONLY_ARGS = [
+ '-aux-info',
+]
+
+
+def InvokedAsGcc():
+ """Return whether or not we're pretending to be GCC."""
+ return sys.argv[0].endswith('gcc') or sys.argv[0].endswith('g++')
+
+
+def Is32Bit(args):
+ """Return whether or not we're 32-bit."""
+ M32_BIT_ARGS = [
+ '-m32',
+ '-mx32',
+ ]
+
+ return any(arg in M32_BIT_ARGS for arg in args)
+
+
+def FilterWlArg(arg):
+ """Remove -z,defs and equivalents from a single -Wl option."""
+ parts = arg.split(',')[1:]
+
+ filtered = []
+ for part in parts:
+ if part == 'defs':
+ removed = filtered.pop()
+ assert removed == '-z'
+ continue
+
+ if part == '--no-undefined':
+ continue
+
+ filtered.append(part)
+
+ if filtered:
+ return '-Wl,' + ','.join(filtered)
+
+ # Filtered entire argument.
+ return None
+
+
+def _RemoveLastMatching(l, find):
+ for i in xrange(len(l) - 1, -1, -1):
+ if l[i] == find:
+ del l[i]
+ return
+
+ raise IndexError('Not found')
+
+
+def RemoveZDefs(args):
+ """Remove unsupported -Wl,-z,defs linker option."""
+ filtered = []
+
+ for arg in args:
+ if arg == '-Wl,defs':
+ _RemoveLastMatching(filtered, '-Wl,-z')
+ continue
+
+ if arg == '-Wl,--no-undefined':
+ continue
+
+ if arg.startswith('-Wl,'):
+ arg = FilterWlArg(arg)
+ if not arg:
+ continue
+
+ filtered.append(arg)
+
+ return filtered
+
+
+def GetCompilerArgs(args, is_cxx):
+ """Generate compiler args."""
+ compiler_args = args[1:]
+
+ if Is32Bit(args):
+ # 32 bit builds not supported.
+ compiler_args.extend([
+ '-fno-sanitize=memory',
+ '-fno-sanitize-memory-track-origins',
+ ])
+
+ return compiler_args
+
+ compiler_args = RemoveZDefs(compiler_args)
+ compiler_args.extend([
+ # FORTIFY_SOURCE is not supported by sanitizers.
+ '-U_FORTIFY_SOURCE',
+ '-Wp,-U_FORTIFY_SOURCE',
+ # Reduce binary size.
+ '-gline-tables-only',
+ # Disable all warnings.
+ '-w',
+ # LTO isn't supported.
+ '-fno-lto',
+ ])
+
+ if InvokedAsGcc():
+ compiler_args.extend([
+ # For better compatibility with flags passed via -Wa,...
+ '-fno-integrated-as',
+ ])
+
+ if '-fsanitize=memory' not in args:
+ # If MSan flags weren't added for some reason, add them here.
+ compiler_args.extend(msan_build.GetInjectedFlags())
+
+ if is_cxx:
+ compiler_args.append('-stdlib=libc++')
+
+ return compiler_args
+
+
+def FindRealClang():
+ """Return path to real clang."""
+ return os.environ['REAL_CLANG_PATH']
+
+
+def FallbackToGcc(args):
+ """Check whether if we should fall back to GCC."""
+ if not InvokedAsGcc():
+ return False
+
+ return any(arg in GCC_ONLY_ARGS for arg in args[1:])
+
+
+def main(args):
+ if FallbackToGcc(args):
+ sys.exit(subprocess.call(['/usr/bin/' + os.path.basename(args[0])] +
+ args[1:]))
+
+ is_cxx = args[0].endswith('++')
+ real_clang = FindRealClang()
+
+ if is_cxx:
+ real_clang += '++'
+
+ args = [real_clang] + GetCompilerArgs(args, is_cxx)
+ debug_log_path = os.getenv('WRAPPER_DEBUG_LOG_PATH')
+ if debug_log_path:
+ with open(debug_log_path, 'a') as f:
+ f.write(str(args) + '\n')
+
+ sys.exit(subprocess.call(args))
+
+
+if __name__ == '__main__':
+ main(sys.argv)
diff --git a/infra/base-images/base-sanitizer-libs-builder/compiler_wrapper_test.py b/infra/base-images/base-sanitizer-libs-builder/compiler_wrapper_test.py
new file mode 100644
index 00000000..a05592d3
--- /dev/null
+++ b/infra/base-images/base-sanitizer-libs-builder/compiler_wrapper_test.py
@@ -0,0 +1,42 @@
+"""Tests for compiler_wrapper."""
+
+from __future__ import print_function
+
+import unittest
+
+import compiler_wrapper
+
+
+class CompilerWrapperTest(unittest.TestCase):
+
+ def testFilterZDefs(self):
+ self.assertListEqual(
+ ['arg'],
+ compiler_wrapper.RemoveZDefs(['arg', '-Wl,-z,defs']))
+
+ self.assertListEqual(
+ ['arg'],
+ compiler_wrapper.RemoveZDefs(['arg', '-Wl,-z,--no-undefined']))
+
+ self.assertListEqual(
+ ['arg', '-Wl,-z,relro'],
+ compiler_wrapper.RemoveZDefs(['arg', '-Wl,-z,relro']))
+
+ self.assertListEqual(
+ ['arg', '-Wl,-soname,lib.so.1,-z,relro'],
+ compiler_wrapper.RemoveZDefs(['arg', '-Wl,-soname,lib.so.1,-z,defs,-z,relro']))
+
+ self.assertListEqual(
+ ['arg', '-Wl,-z,relro'],
+ compiler_wrapper.RemoveZDefs(['arg', '-Wl,-z,relro,-z,defs']))
+
+ self.assertListEqual(
+ ['arg'],
+ compiler_wrapper.RemoveZDefs(['arg', '-Wl,-z', '-Wl,defs']))
+
+ self.assertListEqual(
+ ['arg', 'arg2'],
+ compiler_wrapper.RemoveZDefs(['arg', '-Wl,-z', 'arg2', '-Wl,--no-undefined']))
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/infra/base-images/base-sanitizer-libs-builder/msan_build.py b/infra/base-images/base-sanitizer-libs-builder/msan_build.py
new file mode 100755
index 00000000..928b1a59
--- /dev/null
+++ b/infra/base-images/base-sanitizer-libs-builder/msan_build.py
@@ -0,0 +1,456 @@
+#!/usr/bin/env python
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+from __future__ import print_function
+import argparse
+import imp
+import os
+import multiprocessing
+import resource
+import shutil
+import subprocess
+import tempfile
+
+import apt
+from apt import debfile
+
+from packages import package
+import wrapper_utils
+
+SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
+PACKAGES_DIR = os.path.join(SCRIPT_DIR, 'packages')
+
+TRACK_ORIGINS_ARG = '-fsanitize-memory-track-origins='
+
+INJECTED_ARGS = [
+ '-fsanitize=memory',
+ '-fsanitize-recover=memory',
+ '-fPIC',
+ '-fno-omit-frame-pointer',
+]
+
+
+class MSanBuildException(Exception):
+ """Base exception."""
+
+
+def GetTrackOriginsFlag():
+ """Get the track origins flag."""
+ if os.getenv('MSAN_NO_TRACK_ORIGINS'):
+ return TRACK_ORIGINS_ARG + '0'
+
+ return TRACK_ORIGINS_ARG + '2'
+
+
+def GetInjectedFlags():
+ return INJECTED_ARGS + [GetTrackOriginsFlag()]
+
+
+def SetUpEnvironment(work_dir):
+ """Set up build environment."""
+ env = {}
+ env['REAL_CLANG_PATH'] = subprocess.check_output(['which', 'clang']).strip()
+ print('Real clang at', env['REAL_CLANG_PATH'])
+ compiler_wrapper_path = os.path.join(SCRIPT_DIR, 'compiler_wrapper.py')
+
+ # Symlink binaries into TMP/bin
+ bin_dir = os.path.join(work_dir, 'bin')
+ os.mkdir(bin_dir)
+
+ dpkg_host_architecture = wrapper_utils.DpkgHostArchitecture()
+ wrapper_utils.CreateSymlinks(
+ compiler_wrapper_path, bin_dir, [
+ 'clang',
+ 'clang++',
+ # Not all build rules respect $CC/$CXX, so make additional symlinks.
+ 'gcc',
+ 'g++',
+ 'cc',
+ 'c++',
+ dpkg_host_architecture + '-gcc',
+ dpkg_host_architecture + '-g++',
+ ])
+
+ env['CC'] = os.path.join(bin_dir, 'clang')
+ env['CXX'] = os.path.join(bin_dir, 'clang++')
+
+ MSAN_OPTIONS = ' '.join(GetInjectedFlags())
+
+ # We don't use nostrip because some build rules incorrectly break when it is
+ # passed. Instead we install our own no-op strip binaries.
+ env['DEB_BUILD_OPTIONS'] = ('nocheck parallel=%d' %
+ multiprocessing.cpu_count())
+ env['DEB_CFLAGS_APPEND'] = MSAN_OPTIONS
+ env['DEB_CXXFLAGS_APPEND'] = MSAN_OPTIONS + ' -stdlib=libc++'
+ env['DEB_CPPFLAGS_APPEND'] = MSAN_OPTIONS
+ env['DEB_LDFLAGS_APPEND'] = MSAN_OPTIONS
+ env['DPKG_GENSYMBOLS_CHECK_LEVEL'] = '0'
+
+ # debian/rules can set DPKG_GENSYMBOLS_CHECK_LEVEL explicitly, so override it.
+ gen_symbols_wrapper = (
+ '#!/bin/sh\n'
+ 'export DPKG_GENSYMBOLS_CHECK_LEVEL=0\n'
+ '/usr/bin/dpkg-gensymbols "$@"\n')
+
+ wrapper_utils.InstallWrapper(bin_dir, 'dpkg-gensymbols',
+ gen_symbols_wrapper)
+
+ # Install no-op strip binaries.
+ no_op_strip = ('#!/bin/sh\n'
+ 'exit 0\n')
+ wrapper_utils.InstallWrapper(
+ bin_dir, 'strip', no_op_strip,
+ [dpkg_host_architecture + '-strip'])
+
+ env['PATH'] = bin_dir + ':' + os.environ['PATH']
+
+ # nocheck doesn't disable override_dh_auto_test. So we have this hack to try
+ # to disable "make check" or "make test" invocations.
+ make_wrapper = (
+ '#!/bin/bash\n'
+ 'if [ "$1" = "test" ] || [ "$1" = "check" ]; then\n'
+ ' exit 0\n'
+ 'fi\n'
+ '/usr/bin/make "$@"\n')
+ wrapper_utils.InstallWrapper(bin_dir, 'make',
+ make_wrapper)
+
+ # Prevent entire build from failing because of bugs/uninstrumented in tools
+ # that are part of the build.
+ msan_log_dir = os.path.join(work_dir, 'msan')
+ os.mkdir(msan_log_dir)
+ msan_log_path = os.path.join(msan_log_dir, 'log')
+ env['MSAN_OPTIONS'] = (
+ 'halt_on_error=0:exitcode=0:report_umrs=0:log_path=' + msan_log_path)
+
+ # Increase maximum stack size to prevent tests from failing.
+ limit = 128 * 1024 * 1024
+ resource.setrlimit(resource.RLIMIT_STACK, (limit, limit))
+ return env
+
+
+def FindPackageDebs(package_name, work_directory):
+ """Find package debs."""
+ deb_paths = []
+ cache = apt.Cache()
+
+ for filename in os.listdir(work_directory):
+ file_path = os.path.join(work_directory, filename)
+ if not file_path.endswith('.deb'):
+ continue
+
+ # Matching package name.
+ deb = debfile.DebPackage(file_path)
+ if deb.pkgname == package_name:
+ deb_paths.append(file_path)
+ continue
+
+ # Also include -dev packages that depend on the runtime package.
+ pkg = cache[deb.pkgname]
+ if pkg.section != 'libdevel' and pkg.section != 'universe/libdevel':
+ continue
+
+ # But ignore -dbg packages.
+ if deb.pkgname.endswith('-dbg'):
+ continue
+
+ for dependency in deb.depends:
+ if any(dep[0] == package_name for dep in dependency):
+ deb_paths.append(file_path)
+ break
+
+ return deb_paths
+
+
+def ExtractLibraries(deb_paths, work_directory, output_directory):
+ """Extract libraries from .deb packages."""
+ extract_directory = os.path.join(work_directory, 'extracted')
+ if os.path.exists(extract_directory):
+ shutil.rmtree(extract_directory, ignore_errors=True)
+
+ os.mkdir(extract_directory)
+
+ for deb_path in deb_paths:
+ subprocess.check_call(['dpkg-deb', '-x', deb_path, extract_directory])
+
+ extracted = []
+ for root, _, filenames in os.walk(extract_directory):
+ if 'libx32' in root or 'lib32' in root:
+ continue
+
+ for filename in filenames:
+ if (not filename.endswith('.so') and '.so.' not in filename and
+ not filename.endswith('.a') and '.a' not in filename):
+ continue
+
+ file_path = os.path.join(root, filename)
+ rel_file_path = os.path.relpath(file_path, extract_directory)
+ rel_directory = os.path.dirname(rel_file_path)
+
+ target_dir = os.path.join(output_directory, rel_directory)
+ if not os.path.exists(target_dir):
+ os.makedirs(target_dir)
+
+ target_file_path = os.path.join(output_directory, rel_file_path)
+ extracted.append(target_file_path)
+
+ if os.path.lexists(target_file_path):
+ os.remove(target_file_path)
+
+ if os.path.islink(file_path):
+ link_path = os.readlink(file_path)
+ if os.path.isabs(link_path):
+ # Make absolute links relative.
+ link_path = os.path.relpath(
+ link_path, os.path.join('/', rel_directory))
+
+ os.symlink(link_path, target_file_path)
+ else:
+ shutil.copy2(file_path, target_file_path)
+
+ return extracted
+
+
+def GetPackage(package_name):
+ apt_cache = apt.Cache()
+ version = apt_cache[package_name].candidate
+ source_name = version.source_name
+ local_source_name = source_name.replace('.', '_')
+
+ custom_package_path = os.path.join(PACKAGES_DIR, local_source_name) + '.py'
+ if not os.path.exists(custom_package_path):
+ print('Using default package build steps.')
+ return package.Package(source_name, version)
+
+ print('Using custom package build steps.')
+ module = imp.load_source('packages.' + local_source_name, custom_package_path)
+ return module.Package(version)
+
+
+def PatchRpath(path, output_directory):
+ """Patch rpath to be relative to $ORIGIN."""
+ try:
+ rpaths = subprocess.check_output(
+ ['patchelf', '--print-rpath', path]).strip()
+ except subprocess.CalledProcessError:
+ return
+
+ if not rpaths:
+ return
+
+ processed_rpath = []
+ rel_directory = os.path.join(
+ '/', os.path.dirname(os.path.relpath(path, output_directory)))
+
+ for rpath in rpaths.split(':'):
+ if '$ORIGIN' in rpath:
+ # Already relative.
+ processed_rpath.append(rpath)
+ continue
+
+ processed_rpath.append(os.path.join(
+ '$ORIGIN',
+ os.path.relpath(rpath, rel_directory)))
+
+ processed_rpath = ':'.join(processed_rpath)
+ print('Patching rpath for', path, 'to', processed_rpath)
+ subprocess.check_call(
+ ['patchelf', '--force-rpath', '--set-rpath',
+ processed_rpath, path])
+
+
+def _CollectDependencies(apt_cache, pkg, cache, dependencies):
+ """Collect dependencies that need to be built."""
+ C_OR_CXX_DEPS = [
+ 'libc++1',
+ 'libc6',
+ 'libc++abi1',
+ 'libgcc1',
+ 'libstdc++6',
+ ]
+
+ BLACKLISTED_PACKAGES = [
+ 'libcapnp-0.5.3', # fails to compile on newer clang.
+ 'libllvm5.0',
+ 'libmircore1',
+ 'libmircommon7',
+ 'libmirclient9',
+ 'libmirprotobuf3',
+ 'multiarch-support',
+ ]
+
+ if pkg.name in BLACKLISTED_PACKAGES:
+ return False
+
+ if pkg.section != 'libs' and pkg.section != 'universe/libs':
+ return False
+
+ if pkg.name in C_OR_CXX_DEPS:
+ return True
+
+ is_c_or_cxx = False
+ for dependency in pkg.candidate.dependencies:
+ dependency = dependency[0]
+
+ if dependency.name in cache:
+ is_c_or_cxx |= cache[dependency.name]
+ else:
+ is_c_or_cxx |= _CollectDependencies(apt_cache, apt_cache[dependency.name],
+ cache, dependencies)
+ if is_c_or_cxx:
+ dependencies.append(pkg.name)
+
+ cache[pkg.name] = is_c_or_cxx
+ return is_c_or_cxx
+
+
+def GetBuildList(package_name):
+ """Get list of packages that need to be built including dependencies."""
+ apt_cache = apt.Cache()
+ pkg = apt_cache[package_name]
+
+ dependencies = []
+ _CollectDependencies(apt_cache, pkg, {}, dependencies)
+ return dependencies
+
+
+class MSanBuilder(object):
+ """MSan builder."""
+
+ def __init__(self, debug=False, log_path=None, work_dir=None, no_track_origins=False):
+ self.debug = debug
+ self.log_path = log_path
+ self.work_dir = work_dir
+ self.no_track_origins = no_track_origins
+ self.env = None
+
+ def __enter__(self):
+ if not self.work_dir:
+ self.work_dir = tempfile.mkdtemp(dir=self.work_dir)
+
+ if os.path.exists(self.work_dir):
+ shutil.rmtree(self.work_dir, ignore_errors=True)
+
+ os.makedirs(self.work_dir)
+ self.env = SetUpEnvironment(self.work_dir)
+
+ if self.debug and self.log_path:
+ self.env['WRAPPER_DEBUG_LOG_PATH'] = self.log_path
+
+ if self.no_track_origins:
+ self.env['MSAN_NO_TRACK_ORIGINS'] = '1'
+
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ if not self.debug:
+ shutil.rmtree(self.work_dir, ignore_errors=True)
+
+ def Build(self, package_name, output_directory, create_subdirs=False):
+ """Build the package and write results into the output directory."""
+ deb_paths = FindPackageDebs(package_name, self.work_dir)
+ if deb_paths:
+ print('Source package already built for', package_name)
+ else:
+ pkg = GetPackage(package_name)
+
+ pkg.InstallBuildDeps()
+ source_directory = pkg.DownloadSource(self.work_dir)
+ print('Source downloaded to', source_directory)
+
+ # custom bin directory for custom build scripts to write wrappers.
+ custom_bin_dir = os.path.join(self.work_dir, package_name + '_bin')
+ os.mkdir(custom_bin_dir)
+ env = self.env.copy()
+ env['PATH'] = custom_bin_dir + ':' + env['PATH']
+
+ pkg.Build(source_directory, env, custom_bin_dir)
+ shutil.rmtree(custom_bin_dir, ignore_errors=True)
+
+ deb_paths = FindPackageDebs(package_name, self.work_dir)
+
+ if not deb_paths:
+ raise MSanBuildException('Failed to find .deb packages.')
+
+ print('Extracting', ' '.join(deb_paths))
+
+ if create_subdirs:
+ extract_directory = os.path.join(output_directory, package_name)
+ else:
+ extract_directory = output_directory
+
+ extracted_paths = ExtractLibraries(deb_paths, self.work_dir,
+ extract_directory)
+ for extracted_path in extracted_paths:
+ if not os.path.islink(extracted_path):
+ PatchRpath(extracted_path, extract_directory)
+
+
+def main():
+ parser = argparse.ArgumentParser('msan_build.py', description='MSan builder.')
+ parser.add_argument('package_names', nargs='+', help='Name of the packages.')
+ parser.add_argument('output_dir', help='Output directory.')
+ parser.add_argument('--create-subdirs', action='store_true',
+ help=('Create subdirectories in the output '
+ 'directory for each package.'))
+ parser.add_argument('--work-dir', help='Work directory.')
+ parser.add_argument('--no-build-deps', action='store_true',
+ help='Don\'t build dependencies.')
+ parser.add_argument('--debug', action='store_true', help='Enable debug mode.')
+ parser.add_argument('--log-path', help='Log path for debugging.')
+ parser.add_argument('--no-track-origins',
+ action='store_true',
+ help='Build with -fsanitize-memory-track-origins=0.')
+ args = parser.parse_args()
+
+ if args.no_track_origins:
+ os.environ['MSAN_NO_TRACK_ORIGINS'] = '1'
+
+ if not os.path.exists(args.output_dir):
+ os.makedirs(args.output_dir)
+
+ if args.no_build_deps:
+ package_names = args.package_names
+ else:
+ all_packages = set()
+ package_names = []
+
+ # Get list of packages to build, including all dependencies.
+ for package_name in args.package_names:
+ for dep in GetBuildList(package_name):
+ if dep in all_packages:
+ continue
+
+ if args.create_subdirs:
+ os.mkdir(os.path.join(args.output_dir, dep))
+
+ all_packages.add(dep)
+ package_names.append(dep)
+
+ print('Going to build:')
+ for package_name in package_names:
+ print('\t', package_name)
+
+ with MSanBuilder(debug=args.debug, log_path=args.log_path,
+ work_dir=args.work_dir,
+ no_track_origins=args.no_track_origins) as builder:
+ for package_name in package_names:
+ builder.Build(package_name, args.output_dir, args.create_subdirs)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/infra/base-images/base-sanitizer-libs-builder/packages/__init__.py b/infra/base-images/base-sanitizer-libs-builder/packages/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/infra/base-images/base-sanitizer-libs-builder/packages/__init__.py
diff --git a/infra/base-images/base-sanitizer-libs-builder/packages/boost1_58.py b/infra/base-images/base-sanitizer-libs-builder/packages/boost1_58.py
new file mode 100644
index 00000000..8071b7ec
--- /dev/null
+++ b/infra/base-images/base-sanitizer-libs-builder/packages/boost1_58.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+import package
+
+
+class Package(package.Package):
+ """boost1.58 package."""
+
+ def __init__(self, apt_version):
+ super(Package, self).__init__('boost1.58', apt_version)
+
+ def PreBuild(self, source_directory, env, custom_bin_dir):
+ # Otherwise py_nonblocking.cpp fails to build.
+ env['DEB_CXXFLAGS_APPEND'] += ' -std=c++98'
diff --git a/infra/base-images/base-sanitizer-libs-builder/packages/gnutls28.py b/infra/base-images/base-sanitizer-libs-builder/packages/gnutls28.py
new file mode 100644
index 00000000..f8407a66
--- /dev/null
+++ b/infra/base-images/base-sanitizer-libs-builder/packages/gnutls28.py
@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+import os
+import shutil
+
+import package
+import wrapper_utils
+
+
+class Package(package.Package):
+ """gnutls28 package."""
+
+ def __init__(self, apt_version):
+ super(Package, self).__init__('gnutls28', apt_version)
+
+ def PreBuild(self, source_directory, env, custom_bin_dir):
+ configure_wrapper = (
+ '#!/bin/bash\n'
+ '/usr/bin/dh_auto_configure "$@" --disable-hardware-acceleration')
+
+ wrapper_utils.InstallWrapper(
+ custom_bin_dir, 'dh_auto_configure', configure_wrapper)
diff --git a/infra/base-images/base-sanitizer-libs-builder/packages/libgcrypt20.py b/infra/base-images/base-sanitizer-libs-builder/packages/libgcrypt20.py
new file mode 100644
index 00000000..9d200af6
--- /dev/null
+++ b/infra/base-images/base-sanitizer-libs-builder/packages/libgcrypt20.py
@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+import os
+import shutil
+
+import package
+import wrapper_utils
+
+
+class Package(package.Package):
+ """libgcrypt20 package."""
+
+ def __init__(self, apt_version):
+ super(Package, self).__init__('libgcrypt20', apt_version)
+
+ def PreBuild(self, source_directory, env, custom_bin_dir):
+ configure_wrapper = (
+ '#!/bin/bash\n'
+ '/usr/bin/dh_auto_configure "$@" --disable-asm')
+
+ wrapper_utils.InstallWrapper(
+ custom_bin_dir, 'dh_auto_configure', configure_wrapper)
diff --git a/infra/base-images/base-sanitizer-libs-builder/packages/mesa.py b/infra/base-images/base-sanitizer-libs-builder/packages/mesa.py
new file mode 100644
index 00000000..ec2e9d21
--- /dev/null
+++ b/infra/base-images/base-sanitizer-libs-builder/packages/mesa.py
@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+import package
+
+
+class Package(package.Package):
+ """mesa package."""
+
+ def __init__(self, apt_version):
+ super(Package, self).__init__('mesa', apt_version)
+
+ def PreBuild(self, source_directory, env, custom_bin_dir):
+ env['DEB_CXXFLAGS_APPEND'] += ' -std=c++11'
diff --git a/infra/base-images/base-sanitizer-libs-builder/packages/nettle.py b/infra/base-images/base-sanitizer-libs-builder/packages/nettle.py
new file mode 100644
index 00000000..e1b0e2f8
--- /dev/null
+++ b/infra/base-images/base-sanitizer-libs-builder/packages/nettle.py
@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+import os
+import shutil
+
+import package
+
+
+def AddNoAsmArg(config_path):
+ """Add --disable-assembler to config scripts."""
+ shutil.move(config_path, config_path + '.real')
+ with open(config_path, 'w') as f:
+ f.write(
+ '#!/bin/sh\n'
+ '%s.real --disable-assembler "$@"\n' % config_path)
+ os.chmod(config_path, 0755)
+
+
+class Package(package.Package):
+ """nettle package."""
+
+ def __init__(self, apt_version):
+ super(Package, self).__init__('nettle', apt_version)
+
+ def PreBuild(self, source_directory, env, custom_bin_dir):
+ AddNoAsmArg(os.path.join(source_directory, 'configure'))
diff --git a/infra/base-images/base-sanitizer-libs-builder/packages/openssl.py b/infra/base-images/base-sanitizer-libs-builder/packages/openssl.py
new file mode 100644
index 00000000..e24ccc58
--- /dev/null
+++ b/infra/base-images/base-sanitizer-libs-builder/packages/openssl.py
@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+import os
+import shutil
+
+import package
+
+
+def AddNoAsmArg(config_path):
+ """Add --no-asm to config scripts."""
+ shutil.move(config_path, config_path + '.real')
+ with open(config_path, 'w') as f:
+ f.write(
+ '#!/bin/sh\n'
+ '%s.real no-asm "$@"\n' % config_path)
+ os.chmod(config_path, 0755)
+
+
+class Package(package.Package):
+ """openssl package."""
+
+ def __init__(self, apt_version):
+ super(Package, self).__init__('openssl', apt_version)
+
+ def PreBuild(self, source_directory, env, custom_bin_dir):
+ AddNoAsmArg(os.path.join(source_directory, 'Configure'))
+ AddNoAsmArg(os.path.join(source_directory, 'config'))
diff --git a/infra/base-images/base-sanitizer-libs-builder/packages/package.py b/infra/base-images/base-sanitizer-libs-builder/packages/package.py
new file mode 100644
index 00000000..059c2358
--- /dev/null
+++ b/infra/base-images/base-sanitizer-libs-builder/packages/package.py
@@ -0,0 +1,82 @@
+#!/usr/bin/env python
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+import os
+import subprocess
+
+import apt
+
+SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
+
+
+def ApplyPatch(source_directory, patch_name):
+ """Apply custom patch."""
+ subprocess.check_call(['patch', '-p1', '-i',
+ os.path.join(SCRIPT_DIR, patch_name)],
+ cwd=source_directory)
+
+
+class PackageException(Exception):
+ """Base package exception."""
+
+
+class Package(object):
+ """Base package."""
+
+ def __init__(self, name, apt_version):
+ self.name = name
+ self.apt_version = apt_version
+
+ def PreBuild(self, source_directory, env, custom_bin_dir):
+ return
+
+ def PostBuild(self, source_directory, env, custom_bin_dir):
+ return
+
+ def PreDownload(self, download_directory):
+ return
+
+ def PostDownload(self, source_directory):
+ return
+
+ def InstallBuildDeps(self):
+ """Install build dependencies for a package."""
+ subprocess.check_call(['apt-get', 'update'])
+ subprocess.check_call(['apt-get', 'build-dep', '-y', self.name])
+
+ # Reload package after update.
+ self.apt_version = (
+ apt.Cache()[self.apt_version.package.name].candidate)
+
+ def DownloadSource(self, download_directory):
+ """Download the source for a package."""
+ self.PreDownload(download_directory)
+
+ source_directory = self.apt_version.fetch_source(download_directory)
+
+ self.PostDownload(source_directory)
+ return source_directory
+
+ def Build(self, source_directory, env, custom_bin_dir):
+ """Build .deb packages."""
+ self.PreBuild(source_directory, env, custom_bin_dir)
+ subprocess.check_call(
+ ['dpkg-buildpackage', '-us', '-uc', '-B'],
+ cwd=source_directory, env=env)
+ self.PostBuild(source_directory, env, custom_bin_dir)
+
+
diff --git a/infra/base-images/base-sanitizer-libs-builder/packages/pixman.py b/infra/base-images/base-sanitizer-libs-builder/packages/pixman.py
new file mode 100644
index 00000000..d63b1468
--- /dev/null
+++ b/infra/base-images/base-sanitizer-libs-builder/packages/pixman.py
@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+import os
+import shutil
+
+import package
+
+
+class Package(package.Package):
+ """pixman package."""
+
+ def __init__(self, apt_version):
+ super(Package, self).__init__('pixman', apt_version)
+
+ def PostDownload(self, source_directory):
+ # Incorrect checking of GCC vector extension availability.
+ os.system(
+ 'sed s/support_for_gcc_vector_extensions=yes/'
+ 'support_for_gcc_vector_extensions=no/ -i %s/configure.ac' %
+ source_directory)
+
+ def PreBuild(self, source_directory, env, custom_bin_dir):
+ blacklist_flag = ' -fsanitize-blacklist=' + os.path.join(
+ os.path.dirname(os.path.abspath(__file__)),
+ 'pixman_blacklist.txt')
+ env['DEB_CXXFLAGS_APPEND'] += blacklist_flag
+ env['DEB_CFLAGS_APPEND'] += blacklist_flag
diff --git a/infra/base-images/base-sanitizer-libs-builder/packages/pixman_blacklist.txt b/infra/base-images/base-sanitizer-libs-builder/packages/pixman_blacklist.txt
new file mode 100644
index 00000000..69cf159d
--- /dev/null
+++ b/infra/base-images/base-sanitizer-libs-builder/packages/pixman_blacklist.txt
@@ -0,0 +1 @@
+src:*/pixman-sse2.c
diff --git a/infra/base-images/base-sanitizer-libs-builder/packages/pulseaudio.py b/infra/base-images/base-sanitizer-libs-builder/packages/pulseaudio.py
new file mode 100644
index 00000000..853b9e72
--- /dev/null
+++ b/infra/base-images/base-sanitizer-libs-builder/packages/pulseaudio.py
@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+from __future__ import print_function
+import glob
+import os
+import subprocess
+
+import package
+
+
+class Package(package.Package):
+ """PulseAudio package."""
+
+ def __init__(self, apt_version):
+ super(Package, self).__init__('pulseaudio', apt_version)
+
+ def PostDownload(self, source_directory):
+ """Remove blacklisted patches."""
+ # Fix *droid* patches.
+ bad_patch_path = os.path.join(
+ source_directory, 'debian', 'patches',
+ '0600-droid-sync-with-upstream-for-Android-5-support-and-b.patch')
+ if not os.path.exists(bad_patch_path):
+ return
+
+ print('Applying custom patches.')
+ package.ApplyPatch(source_directory, 'pulseaudio_fix_android.patch')
diff --git a/infra/base-images/base-sanitizer-libs-builder/packages/pulseaudio_fix_android.patch b/infra/base-images/base-sanitizer-libs-builder/packages/pulseaudio_fix_android.patch
new file mode 100644
index 00000000..e86f7982
--- /dev/null
+++ b/infra/base-images/base-sanitizer-libs-builder/packages/pulseaudio_fix_android.patch
@@ -0,0 +1,39 @@
+--- pulseaudio-8.0/src/modules/droid/module-droid-card.c 2017-11-27 22:09:42.533589970 +0000
++++ pulseaudio-8.0.fixed/src/modules/droid/module-droid-card.c 2017-11-27 22:28:23.847250467 +0000
+@@ -66,10 +66,11 @@
+ #include "droid-extcon.h"
+ #endif
+
+-#if ANDROID_VERSION_MAJOR == 4 && ANDROID_VERSION_MINOR == 2
++#if ANDROID_VERSION_MAJOR == 4 && ANDROID_VERSION_MINOR == 4
+ #include "module-droid-card-19-symdef.h"
+ #elif ANDROID_VERSION_MAJOR == 5 && ANDROID_VERSION_MINOR == 1
+ #include "module-droid-card-22-symdef.h"
++#else
+ #endif
+
+ PA_MODULE_AUTHOR("Juho Hämäläinen");
+diff -ru pulseaudio-8.0/src/modules/droid/module-droid-sink.c pulseaudio-8.0.fixed/src/modules/droid/module-droid-sink.c
+--- pulseaudio-8.0/src/modules/droid/module-droid-sink.c 2017-11-27 22:09:42.533589970 +0000
++++ pulseaudio-8.0.fixed/src/modules/droid/module-droid-sink.c 2017-11-27 22:29:53.776348900 +0000
+@@ -40,7 +40,7 @@
+ #include "droid-util.h"
+ #include "droid-sink.h"
+
+-#if ANDROID_VERSION_MAJOR == 4 && ANDROID_VERSION_MINOR == 2
++#if ANDROID_VERSION_MAJOR == 4 && ANDROID_VERSION_MINOR == 4
+ #include "module-droid-sink-19-symdef.h"
+ #elif ANDROID_VERSION_MAJOR == 5 && ANDROID_VERSION_MINOR == 1
+ #include "module-droid-sink-22-symdef.h"
+diff -ru pulseaudio-8.0/src/modules/droid/module-droid-source.c pulseaudio-8.0.fixed/src/modules/droid/module-droid-source.c
+--- pulseaudio-8.0/src/modules/droid/module-droid-source.c 2017-11-27 22:09:42.533589970 +0000
++++ pulseaudio-8.0.fixed/src/modules/droid/module-droid-source.c 2017-11-27 22:30:03.920472828 +0000
+@@ -40,7 +40,7 @@
+ #include "droid-util.h"
+ #include "droid-source.h"
+
+-#if ANDROID_VERSION_MAJOR == 4 && ANDROID_VERSION_MINOR == 2
++#if ANDROID_VERSION_MAJOR == 4 && ANDROID_VERSION_MINOR == 4
+ #include "module-droid-source-19-symdef.h"
+ #elif ANDROID_VERSION_MAJOR == 5 && ANDROID_VERSION_MINOR == 1
+ #include "module-droid-source-22-symdef.h"
diff --git a/infra/base-images/base-sanitizer-libs-builder/packages/sqlite3.py b/infra/base-images/base-sanitizer-libs-builder/packages/sqlite3.py
new file mode 100644
index 00000000..3e1a1070
--- /dev/null
+++ b/infra/base-images/base-sanitizer-libs-builder/packages/sqlite3.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+import os
+
+import package
+
+
+class Package(package.Package):
+ """sqlite3 package."""
+
+ def __init__(self, apt_version):
+ super(Package, self).__init__('sqlite3', apt_version)
+
+ def PreBuild(self, source_directory, env, custom_bin_dir):
+ os.system(
+ 'sed -i "s/package ifneeded sqlite3//" %s/debian/rules' %
+ source_directory)
diff --git a/infra/base-images/base-sanitizer-libs-builder/packages/systemd.py b/infra/base-images/base-sanitizer-libs-builder/packages/systemd.py
new file mode 100644
index 00000000..5cb6d60b
--- /dev/null
+++ b/infra/base-images/base-sanitizer-libs-builder/packages/systemd.py
@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+from __future__ import print_function
+import glob
+import os
+import subprocess
+
+import package
+import wrapper_utils
+
+
+class Package(package.Package):
+ """systemd package."""
+
+ def __init__(self, apt_version):
+ super(Package, self).__init__('systemd', apt_version)
+
+ def PreBuild(self, source_directory, env, custom_bin_dir):
+ # Hide msan symbols from nm. the systemd build system uses this to find
+ # undefined symbols and errors out if it does.
+ nm_wrapper = (
+ '#!/bin/bash\n'
+ '/usr/bin/nm "$@" | grep -E -v "U (__msan|memset)"\n'
+ 'exit ${PIPESTATUS[0]}\n')
+
+ wrapper_utils.InstallWrapper(custom_bin_dir, 'nm', nm_wrapper,
+ [wrapper_utils.DpkgHostArchitecture() + '-nm'])
diff --git a/infra/base-images/base-sanitizer-libs-builder/packages/tar.py b/infra/base-images/base-sanitizer-libs-builder/packages/tar.py
new file mode 100644
index 00000000..74abd5c7
--- /dev/null
+++ b/infra/base-images/base-sanitizer-libs-builder/packages/tar.py
@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+import package
+
+
+class Package(package.Package):
+ """tar package."""
+
+ def __init__(self, apt_version):
+ super(Package, self).__init__('tar', apt_version)
+
+ def PreBuild(self, source_directory, env, custom_bin_dir):
+ env['FORCE_UNSAFE_CONFIGURE'] = '1'
diff --git a/infra/base-images/base-sanitizer-libs-builder/patch_build.py b/infra/base-images/base-sanitizer-libs-builder/patch_build.py
new file mode 100755
index 00000000..cb1f4b1d
--- /dev/null
+++ b/infra/base-images/base-sanitizer-libs-builder/patch_build.py
@@ -0,0 +1,143 @@
+#!/usr/bin/env python
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+from __future__ import print_function
+import argparse
+import os
+import re
+import shutil
+import subprocess
+import sys
+
+INSTRUMENTED_LIBRARIES_DIRNAME = 'instrumented_libraries'
+MSAN_LIBS_PATH = os.getenv('MSAN_LIBS_PATH', '/msan')
+
+
+def IsElf(file_path):
+ """Whether if the file is an elf file."""
+ with open(file_path) as f:
+ return f.read(4) == '\x7fELF'
+
+
+def Ldd(binary_path):
+ """Run ldd on a file."""
+ try:
+ output = subprocess.check_output(['ldd', binary_path], stderr=subprocess.STDOUT)
+ except subprocess.CalledProcessError:
+ print('Failed to call ldd on', binary_path, file=sys.stderr)
+ return []
+
+ libs = []
+
+ OUTPUT_PATTERN = re.compile(r'\s*([^\s]+)\s*=>\s*([^\s]+)')
+ for line in output.splitlines():
+ match = OUTPUT_PATTERN.match(line)
+ if not match:
+ continue
+
+ libs.append((match.group(1), match.group(2)))
+
+ return libs
+
+
+def FindLib(path):
+ """Find instrumented version of lib."""
+ candidate_path = os.path.join(MSAN_LIBS_PATH, path[1:])
+ if os.path.exists(candidate_path):
+ return candidate_path
+
+ for lib_dir in os.listdir(MSAN_LIBS_PATH):
+ candidate_path = os.path.join(MSAN_LIBS_PATH, lib_dir, path[1:])
+ if os.path.exists(candidate_path):
+ return candidate_path
+
+ return None
+
+
+def PatchBinary(binary_path, instrumented_dir):
+ """Patch binary to link to instrumented libs."""
+ extra_rpaths = set()
+
+ for name, path in Ldd(binary_path):
+ if not os.path.isabs(path):
+ continue
+
+ instrumented_path = FindLib(path)
+ if not instrumented_path:
+ print('WARNING: Instrumented library not found for', path,
+ file=sys.stderr)
+ continue
+
+ target_path = os.path.join(instrumented_dir, path[1:])
+ if not os.path.exists(target_path):
+ print('Copying instrumented lib to', target_path)
+ target_dir = os.path.dirname(target_path)
+ if not os.path.exists(target_dir):
+ os.makedirs(target_dir)
+ shutil.copy2(instrumented_path, target_path)
+
+ extra_rpaths.add(
+ os.path.join('$ORIGIN', INSTRUMENTED_LIBRARIES_DIRNAME,
+ os.path.dirname(path[1:])))
+
+ if not extra_rpaths:
+ return
+
+ existing_rpaths = subprocess.check_output(
+ ['patchelf', '--print-rpath', binary_path]).strip()
+ processed_rpaths = ':'.join(extra_rpaths)
+ if existing_rpaths:
+ processed_rpaths += ':' + existing_rpaths
+ print('Patching rpath for', binary_path, 'from', existing_rpaths, 'to',
+ processed_rpaths)
+
+ subprocess.check_call(
+ ['patchelf', '--force-rpath', '--set-rpath',
+ processed_rpaths, binary_path])
+
+
+def PatchBuild(output_directory):
+ """Patch build to use msan libs."""
+ instrumented_dir = os.path.join(output_directory,
+ INSTRUMENTED_LIBRARIES_DIRNAME)
+ if not os.path.exists(instrumented_dir):
+ os.mkdir(instrumented_dir)
+
+ for root_dir, _, filenames in os.walk(output_directory):
+ for filename in filenames:
+ file_path = os.path.join(root_dir, filename)
+
+ if os.path.islink(file_path):
+ continue
+
+ if not IsElf(file_path):
+ continue
+
+ PatchBinary(file_path, instrumented_dir)
+
+
+def main():
+ parser = argparse.ArgumentParser('patch_build.py', description='MSan build patcher.')
+ parser.add_argument('output_dir', help='Output directory.')
+
+ args = parser.parse_args()
+
+ PatchBuild(os.path.abspath(args.output_dir))
+
+
+if __name__ == '__main__':
+ main()
diff --git a/infra/base-images/base-sanitizer-libs-builder/wrapper_utils.py b/infra/base-images/base-sanitizer-libs-builder/wrapper_utils.py
new file mode 100644
index 00000000..0cbf1677
--- /dev/null
+++ b/infra/base-images/base-sanitizer-libs-builder/wrapper_utils.py
@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+from __future__ import print_function
+
+import contextlib
+import os
+import subprocess
+
+
+def DpkgHostArchitecture():
+ """Return the host architecture."""
+ return subprocess.check_output(
+ ['dpkg-architecture', '-qDEB_HOST_GNU_TYPE']).strip()
+
+
+def InstallWrapper(bin_dir, name, contents, extra_names=None):
+ """Install a custom wrapper script into |bin_dir|."""
+ path = os.path.join(bin_dir, name)
+ with open(path, 'w') as f:
+ f.write(contents)
+
+ os.chmod(path, 0755)
+
+ if extra_names:
+ CreateSymlinks(path, bin_dir, extra_names)
+
+
+def CreateSymlinks(original_path, bin_dir, extra_names):
+ """Create symlinks."""
+ for extra_name in extra_names:
+ extra_path = os.path.join(bin_dir, extra_name)
+ os.symlink(original_path, extra_path)