diff options
Diffstat (limited to 'infra/bots/recipe_modules/flavor')
| -rw-r--r-- | infra/bots/recipe_modules/flavor/__init__.py | 17 | ||||
| -rw-r--r-- | infra/bots/recipe_modules/flavor/api.py | 219 | ||||
| -rw-r--r-- | infra/bots/recipe_modules/flavor/default_flavor.py | 152 | ||||
| -rw-r--r-- | infra/bots/recipe_modules/flavor/gn_android_flavor.py | 179 | ||||
| -rw-r--r-- | infra/bots/recipe_modules/flavor/gn_flavor.py | 154 | ||||
| -rw-r--r-- | infra/bots/recipe_modules/flavor/ios_flavor.py | 168 | ||||
| -rw-r--r-- | infra/bots/recipe_modules/flavor/pdfium_flavor.py | 64 | ||||
| -rw-r--r-- | infra/bots/recipe_modules/flavor/valgrind_flavor.py | 27 |
8 files changed, 980 insertions, 0 deletions
diff --git a/infra/bots/recipe_modules/flavor/__init__.py b/infra/bots/recipe_modules/flavor/__init__.py new file mode 100644 index 0000000000..b532b41f34 --- /dev/null +++ b/infra/bots/recipe_modules/flavor/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +DEPS = [ + 'build/adb', + 'build/file', + 'builder_name_schema', + 'recipe_engine/path', + 'recipe_engine/platform', + 'recipe_engine/properties', + 'recipe_engine/python', + 'recipe_engine/raw_io', + 'recipe_engine/step', + 'run', + 'vars', +] diff --git a/infra/bots/recipe_modules/flavor/api.py b/infra/bots/recipe_modules/flavor/api.py new file mode 100644 index 0000000000..1032f64207 --- /dev/null +++ b/infra/bots/recipe_modules/flavor/api.py @@ -0,0 +1,219 @@ +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +# pylint: disable=W0201 + + +from recipe_engine import recipe_api + +from . import default_flavor +from . import gn_android_flavor +from . import gn_flavor +from . import ios_flavor +from . import pdfium_flavor +from . import valgrind_flavor + + +TEST_EXPECTED_SKP_VERSION = '42' +TEST_EXPECTED_SVG_VERSION = '42' +TEST_EXPECTED_SK_IMAGE_VERSION = '42' + +VERSION_FILE_SK_IMAGE = 'SK_IMAGE_VERSION' +VERSION_FILE_SKP = 'SKP_VERSION' +VERSION_FILE_SVG = 'SVG_VERSION' + +VERSION_NONE = -1 + +def is_android(builder_cfg): + return 'Android' in builder_cfg.get('extra_config', '') + + +def is_ios(builder_cfg): + return ('iOS' == builder_cfg.get('extra_config', '') or + builder_cfg.get('os') == 'iOS') + + +def is_pdfium(builder_cfg): + return 'PDFium' in builder_cfg.get('extra_config', '') + + +def is_valgrind(builder_cfg): + return 'Valgrind' in builder_cfg.get('extra_config', '') + + +class SkiaFlavorApi(recipe_api.RecipeApi): + def get_flavor(self, builder_cfg): + """Return a flavor utils object specific to the given builder.""" + if is_android(builder_cfg): + return gn_android_flavor.GNAndroidFlavorUtils(self.m) + elif is_ios(builder_cfg): + return ios_flavor.iOSFlavorUtils(self.m) + elif is_pdfium(builder_cfg): + return pdfium_flavor.PDFiumFlavorUtils(self.m) + elif is_valgrind(builder_cfg): + return valgrind_flavor.ValgrindFlavorUtils(self.m) + else: + return gn_flavor.GNFlavorUtils(self.m) + + def setup(self): + self._f = self.get_flavor(self.m.vars.builder_cfg) + + def step(self, name, cmd, **kwargs): + return self._f.step(name, cmd, **kwargs) + + def compile(self, target, **kwargs): + return self._f.compile(target, **kwargs) + + def copy_extra_build_products(self, swarming_out_dir): + return self._f.copy_extra_build_products(swarming_out_dir) + + @property + def out_dir(self): + return self._f.out_dir + + def device_path_join(self, *args): + return self._f.device_path_join(*args) + + def copy_directory_contents_to_device(self, host_dir, device_dir): + return self._f.copy_directory_contents_to_device(host_dir, device_dir) + + def copy_directory_contents_to_host(self, device_dir, host_dir): + return self._f.copy_directory_contents_to_host(device_dir, host_dir) + + def copy_file_to_device(self, host_path, device_path): + return self._f.copy_file_to_device(host_path, device_path) + + def create_clean_host_dir(self, path): + return self._f.create_clean_host_dir(path) + + def create_clean_device_dir(self, path): + return self._f.create_clean_device_dir(path) + + def read_file_on_device(self, path): + return self._f.read_file_on_device(path) + + def remove_file_on_device(self, path): + return self._f.remove_file_on_device(path) + + def install_everything(self): + self.install(skps=True, images=True, svgs=True, resources=True) + + def install(self, skps=False, images=False, svgs=False, resources=False): + self._f.install() + self.device_dirs = self._f.device_dirs + + # TODO(borenet): Only copy files which have changed. + if resources: + self.copy_directory_contents_to_device( + self.m.vars.resource_dir, + self.device_dirs.resource_dir) + + if skps: + self._copy_skps() + if images: + self._copy_images() + if svgs: + self._copy_svgs() + + def cleanup_steps(self): + return self._f.cleanup_steps() + + def _copy_dir(self, host_version, version_file, tmp_dir, + host_path, device_path, test_expected_version, + test_actual_version): + actual_version_file = self.m.path.join(tmp_dir, version_file) + # Copy to device. + device_version_file = self.device_path_join( + self.device_dirs.tmp_dir, version_file) + if str(actual_version_file) != str(device_version_file): + try: + device_version = self.read_file_on_device(device_version_file) + except self.m.step.StepFailure: + device_version = VERSION_NONE + if device_version != host_version: + self.remove_file_on_device(device_version_file) + self.create_clean_device_dir(device_path) + self.copy_directory_contents_to_device( + host_path, device_path) + + # Copy the new version file. + self.copy_file_to_device(actual_version_file, device_version_file) + + def _copy_images(self): + """Download and copy test images if needed.""" + version_file = self.m.vars.infrabots_dir.join( + 'assets', 'skimage', 'VERSION') + test_data = self.m.properties.get( + 'test_downloaded_sk_image_version', TEST_EXPECTED_SK_IMAGE_VERSION) + version = self.m.run.readfile( + version_file, + name='Get downloaded skimage VERSION', + test_data=test_data).rstrip() + self.m.run.writefile( + self.m.path.join(self.m.vars.tmp_dir, VERSION_FILE_SK_IMAGE), + version) + self._copy_dir( + version, + VERSION_FILE_SK_IMAGE, + self.m.vars.tmp_dir, + self.m.vars.images_dir, + self.device_dirs.images_dir, + test_expected_version=self.m.properties.get( + 'test_downloaded_sk_image_version', + TEST_EXPECTED_SK_IMAGE_VERSION), + test_actual_version=self.m.properties.get( + 'test_downloaded_sk_image_version', + TEST_EXPECTED_SK_IMAGE_VERSION)) + return version + + def _copy_skps(self): + """Download and copy the SKPs if needed.""" + version_file = self.m.vars.infrabots_dir.join( + 'assets', 'skp', 'VERSION') + test_data = self.m.properties.get( + 'test_downloaded_skp_version', TEST_EXPECTED_SKP_VERSION) + version = self.m.run.readfile( + version_file, + name='Get downloaded SKP VERSION', + test_data=test_data).rstrip() + self.m.run.writefile( + self.m.path.join(self.m.vars.tmp_dir, VERSION_FILE_SKP), + version) + self._copy_dir( + version, + VERSION_FILE_SKP, + self.m.vars.tmp_dir, + self.m.vars.local_skp_dir, + self.device_dirs.skp_dir, + test_expected_version=self.m.properties.get( + 'test_downloaded_skp_version', TEST_EXPECTED_SKP_VERSION), + test_actual_version=self.m.properties.get( + 'test_downloaded_skp_version', TEST_EXPECTED_SKP_VERSION)) + return version + + def _copy_svgs(self): + """Download and copy the SVGs if needed.""" + version_file = self.m.vars.infrabots_dir.join( + 'assets', 'svg', 'VERSION') + test_data = self.m.properties.get( + 'test_downloaded_svg_version', TEST_EXPECTED_SVG_VERSION) + version = self.m.run.readfile( + version_file, + name='Get downloaded SVG VERSION', + test_data=test_data).rstrip() + self.m.run.writefile( + self.m.path.join(self.m.vars.tmp_dir, VERSION_FILE_SVG), + version) + self._copy_dir( + version, + VERSION_FILE_SVG, + self.m.vars.tmp_dir, + self.m.vars.local_svg_dir, + self.device_dirs.svg_dir, + test_expected_version=self.m.properties.get( + 'test_downloaded_svg_version', TEST_EXPECTED_SVG_VERSION), + test_actual_version=self.m.properties.get( + 'test_downloaded_svg_version', TEST_EXPECTED_SVG_VERSION)) + return version diff --git a/infra/bots/recipe_modules/flavor/default_flavor.py b/infra/bots/recipe_modules/flavor/default_flavor.py new file mode 100644 index 0000000000..dacb0fd056 --- /dev/null +++ b/infra/bots/recipe_modules/flavor/default_flavor.py @@ -0,0 +1,152 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +# pylint: disable=W0201 + + +"""Default flavor utils class, used for desktop builders.""" + + +import json + + +WIN_TOOLCHAIN_DIR = 't' + + +class DeviceDirs(object): + def __init__(self, + dm_dir, + perf_data_dir, + resource_dir, + images_dir, + skp_dir, + svg_dir, + tmp_dir): + self._dm_dir = dm_dir + self._perf_data_dir = perf_data_dir + self._resource_dir = resource_dir + self._images_dir = images_dir + self._skp_dir = skp_dir + self._svg_dir = svg_dir + self._tmp_dir = tmp_dir + + @property + def dm_dir(self): + """Where DM writes.""" + return self._dm_dir + + @property + def perf_data_dir(self): + return self._perf_data_dir + + @property + def resource_dir(self): + return self._resource_dir + + @property + def images_dir(self): + return self._images_dir + + @property + def skp_dir(self): + """Holds SKP files that are consumed by RenderSKPs and BenchPictures.""" + return self._skp_dir + + @property + def svg_dir(self): + return self._svg_dir + + @property + def tmp_dir(self): + return self._tmp_dir + + +class DefaultFlavorUtils(object): + """Utilities to be used by build steps. + + The methods in this class define how certain high-level functions should + work. Each build step flavor should correspond to a subclass of + DefaultFlavorUtils which may override any of these functions as appropriate + for that flavor. + + For example, the AndroidFlavorUtils will override the functions for + copying files between the host and Android device, as well as the + 'step' function, so that commands may be run through ADB. + """ + def __init__(self, m): + self.m = m + self._chrome_path = None + self._win_toolchain_dir = self.m.vars.slave_dir.join(WIN_TOOLCHAIN_DIR) + win_toolchain_asset_path = self.m.vars.infrabots_dir.join( + 'assets', 'win_toolchain', 'VERSION') + if not self.m.path.exists(win_toolchain_asset_path): + self._win_toolchain_dir = self.m.vars.slave_dir + + def copy_extra_build_products(self, swarming_out_dir): + pass + + @property + def out_dir(self): + """Flavor-specific out directory.""" + return self.m.vars.skia_out.join(self.m.vars.configuration) + + def device_path_join(self, *args): + """Like os.path.join(), but for paths on a connected device.""" + return self.m.path.join(*args) + + def copy_directory_contents_to_device(self, host_dir, device_dir): + """Like shutil.copytree(), but for copying to a connected device.""" + # For "normal" builders who don't have an attached device, we expect + # host_dir and device_dir to be the same. + if str(host_dir) != str(device_dir): + raise ValueError('For builders who do not have attached devices, copying ' + 'from host to device is undefined and only allowed if ' + 'host_path and device_path are the same (%s vs %s).' % ( + str(host_dir), str(device_dir))) # pragma: no cover + + def copy_directory_contents_to_host(self, device_dir, host_dir): + """Like shutil.copytree(), but for copying from a connected device.""" + # For "normal" builders who don't have an attached device, we expect + # host_dir and device_dir to be the same. + if str(host_dir) != str(device_dir): + raise ValueError('For builders who do not have attached devices, copying ' + 'from device to host is undefined and only allowed if ' + 'host_path and device_path are the same (%s vs %s).' % ( + str(host_dir), str(device_dir))) # pragma: no cover + + def copy_file_to_device(self, host_path, device_path): + """Like shutil.copyfile, but for copying to a connected device.""" + # For "normal" builders who don't have an attached device, we expect + # host_dir and device_dir to be the same. + if str(host_path) != str(device_path): # pragma: no cover + raise ValueError('For builders who do not have attached devices, copying ' + 'from host to device is undefined and only allowed if ' + 'host_path and device_path are the same (%s vs %s).' % ( + str(host_path), str(device_path))) + + def create_clean_device_dir(self, path): + """Like shutil.rmtree() + os.makedirs(), but on a connected device.""" + self.create_clean_host_dir(path) + + def create_clean_host_dir(self, path): + """Convenience function for creating a clean directory.""" + self.m.run.rmtree(path) + self.m.file.makedirs( + self.m.path.basename(path), path, infra_step=True) + + def install(self): + """Run device-specific installation steps.""" + self.device_dirs = DeviceDirs( + dm_dir=self.m.vars.dm_dir, + perf_data_dir=self.m.vars.perf_data_dir, + resource_dir=self.m.vars.resource_dir, + images_dir=self.m.vars.images_dir, + skp_dir=self.m.vars.local_skp_dir, + svg_dir=self.m.vars.local_svg_dir, + tmp_dir=self.m.vars.tmp_dir) + + def cleanup_steps(self): + """Run any device-specific cleanup steps.""" + pass diff --git a/infra/bots/recipe_modules/flavor/gn_android_flavor.py b/infra/bots/recipe_modules/flavor/gn_android_flavor.py new file mode 100644 index 0000000000..bf1510b143 --- /dev/null +++ b/infra/bots/recipe_modules/flavor/gn_android_flavor.py @@ -0,0 +1,179 @@ +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import default_flavor +import subprocess + + +"""GN Android flavor utils, used for building Skia for Android with GN.""" +class GNAndroidFlavorUtils(default_flavor.DefaultFlavorUtils): + def __init__(self, m): + super(GNAndroidFlavorUtils, self).__init__(m) + self._ever_ran_adb = False + + self.device_dirs = default_flavor.DeviceDirs( + dm_dir = self.m.vars.android_data_dir + 'dm_out', + perf_data_dir = self.m.vars.android_data_dir + 'perf', + resource_dir = self.m.vars.android_data_dir + 'resources', + images_dir = self.m.vars.android_data_dir + 'images', + skp_dir = self.m.vars.android_data_dir + 'skps', + svg_dir = self.m.vars.android_data_dir + 'svgs', + tmp_dir = self.m.vars.android_data_dir) + + def _strip_environment(self): + self.m.vars.default_env = {k: v for (k,v) + in self.m.vars.default_env.iteritems() + if k in ['PATH']} + + def _run(self, title, *cmd, **kwargs): + self._strip_environment() + return self.m.run(self.m.step, title, cmd=list(cmd), + cwd=self.m.vars.skia_dir, **kwargs) + + def _py(self, title, script, infra_step=True): + self._strip_environment() + return self.m.run(self.m.python, title, script=script, + cwd=self.m.vars.skia_dir, env=None, infra_step=infra_step) + + def _adb(self, title, *cmd, **kwargs): + self._ever_ran_adb = True + # The only non-infra adb steps (dm / nanobench) happen to not use _adb(). + if 'infra_step' not in kwargs: + kwargs['infra_step'] = True + return self._run(title, 'adb', *cmd, **kwargs) + + def compile(self, unused_target, **kwargs): + compiler = self.m.vars.builder_cfg.get('compiler') + configuration = self.m.vars.builder_cfg.get('configuration') + extra_config = self.m.vars.builder_cfg.get('extra_config', '') + os = self.m.vars.builder_cfg.get('os') + target_arch = self.m.vars.builder_cfg.get('target_arch') + + assert compiler == 'Clang' # At this rate we might not ever support GCC. + + extra_cflags = [] + if configuration == 'Debug': + extra_cflags.append('-O1') + + ndk_asset = 'android_ndk_linux' + if 'Mac' in os: + ndk_asset = 'android_ndk_darwin' + elif 'Win' in os: + ndk_asset = 'n' + + quote = lambda x: '"%s"' % x + args = { + 'ndk': quote(self.m.vars.slave_dir.join(ndk_asset)), + 'target_cpu': quote(target_arch), + } + + if configuration != 'Debug': + args['is_debug'] = 'false' + if 'Vulkan' in extra_config: + args['ndk_api'] = 24 + args['skia_enable_vulkan_debug_layers'] = 'false' + if 'FrameworkDefs' in extra_config: + args['skia_enable_android_framework_defines'] = 'true' + if extra_cflags: + args['extra_cflags'] = repr(extra_cflags).replace("'", '"') + + gn_args = ' '.join('%s=%s' % (k,v) for (k,v) in sorted(args.iteritems())) + + gn = 'gn.exe' if 'Win' in os else 'gn' + ninja = 'ninja.exe' if 'Win' in os else 'ninja' + gn = self.m.vars.skia_dir.join('bin', gn) + + self._py('fetch-gn', self.m.vars.skia_dir.join('bin', 'fetch-gn')) + self._run('gn gen', gn, 'gen', self.out_dir, '--args=' + gn_args) + self._run('ninja', ninja, '-C', self.out_dir) + + def install(self): + self._adb('mkdir ' + self.device_dirs.resource_dir, + 'shell', 'mkdir', '-p', self.device_dirs.resource_dir) + + def cleanup_steps(self): + if self._ever_ran_adb: + self.m.python.inline('dump log', """ + import os + import subprocess + import sys + out = sys.argv[1] + log = subprocess.check_output(['adb', 'logcat', '-d']) + for line in log.split('\\n'): + tokens = line.split() + if len(tokens) == 11 and tokens[-7] == 'F' and tokens[-3] == 'pc': + addr, path = tokens[-2:] + local = os.path.join(out, os.path.basename(path)) + if os.path.exists(local): + sym = subprocess.check_output(['addr2line', '-Cfpe', local, addr]) + line = line.replace(addr, addr + ' ' + sym.strip()) + print line + """, + args=[self.m.vars.skia_out.join(self.m.vars.configuration)], + infra_step=True) + self._adb('kill adb server', 'kill-server') + + def step(self, name, cmd, env=None, **kwargs): + app = self.m.vars.skia_out.join(self.m.vars.configuration, cmd[0]) + self._adb('push %s' % cmd[0], + 'push', app, self.m.vars.android_bin_dir) + + sh = '%s.sh' % cmd[0] + self.m.run.writefile(self.m.vars.tmp_dir.join(sh), + 'set -x; %s%s; echo $? >%src' % + (self.m.vars.android_bin_dir, subprocess.list2cmdline(map(str, cmd)), + self.m.vars.android_bin_dir)) + self._adb('push %s' % sh, + 'push', self.m.vars.tmp_dir.join(sh), self.m.vars.android_bin_dir) + + self._adb('clear log', 'logcat', '-c') + self.m.python.inline('%s' % cmd[0], """ + import subprocess + import sys + bin_dir = sys.argv[1] + sh = sys.argv[2] + subprocess.check_call(['adb', 'shell', 'sh', bin_dir + sh]) + try: + sys.exit(int(subprocess.check_output(['adb', 'shell', 'cat', + bin_dir + 'rc']))) + except ValueError: + print "Couldn't read the return code. Probably killed for OOM." + sys.exit(1) + """, args=[self.m.vars.android_bin_dir, sh]) + + def copy_file_to_device(self, host, device): + self._adb('push %s %s' % (host, device), 'push', host, device) + + def copy_directory_contents_to_device(self, host, device): + # Copy the tree, avoiding hidden directories and resolving symlinks. + self.m.python.inline('push %s/* %s' % (host, device), """ + import os + import subprocess + import sys + host = sys.argv[1] + device = sys.argv[2] + for d, _, fs in os.walk(host): + p = os.path.relpath(d, host) + if p != '.' and p.startswith('.'): + continue + for f in fs: + print os.path.join(p,f) + subprocess.check_call(['adb', 'push', + os.path.realpath(os.path.join(host, p, f)), + os.path.join(device, p, f)]) + """, args=[host, device], cwd=self.m.vars.skia_dir, infra_step=True) + + def copy_directory_contents_to_host(self, device, host): + self._adb('pull %s %s' % (device, host), 'pull', device, host) + + def read_file_on_device(self, path): + return self._adb('read %s' % path, + 'shell', 'cat', path, stdout=self.m.raw_io.output()).stdout + + def remove_file_on_device(self, path): + self._adb('rm %s' % path, 'shell', 'rm', '-f', path) + + def create_clean_device_dir(self, path): + self._adb('rm %s' % path, 'shell', 'rm', '-rf', path) + self._adb('mkdir %s' % path, 'shell', 'mkdir', '-p', path) diff --git a/infra/bots/recipe_modules/flavor/gn_flavor.py b/infra/bots/recipe_modules/flavor/gn_flavor.py new file mode 100644 index 0000000000..f68e3b231f --- /dev/null +++ b/infra/bots/recipe_modules/flavor/gn_flavor.py @@ -0,0 +1,154 @@ +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import default_flavor + +"""GN flavor utils, used for building Skia with GN.""" +class GNFlavorUtils(default_flavor.DefaultFlavorUtils): + def _strip_environment(self): + self.m.vars.default_env = {k: v for (k,v) + in self.m.vars.default_env.iteritems() + if k in ['PATH']} + + def _run(self, title, cmd, env=None, infra_step=False): + self._strip_environment() + self.m.run(self.m.step, title, cmd=cmd, + env=env, cwd=self.m.vars.skia_dir, infra_step=infra_step) + + def _py(self, title, script, env=None, infra_step=True): + self._strip_environment() + self.m.run(self.m.python, title, script=script, + env=env, cwd=self.m.vars.skia_dir, infra_step=infra_step) + + def build_command_buffer(self): + self.m.run(self.m.python, 'build command_buffer', + script=self.m.vars.skia_dir.join('tools', 'build_command_buffer.py'), + args=[ + '--chrome-dir', self.m.vars.checkout_root, + '--output-dir', self.m.vars.skia_out.join(self.m.vars.configuration), + '--no-sync', '--make-output-dir']) + + def compile(self, unused_target, **kwargs): + """Build Skia with GN.""" + compiler = self.m.vars.builder_cfg.get('compiler', '') + configuration = self.m.vars.builder_cfg.get('configuration', '') + extra_config = self.m.vars.builder_cfg.get('extra_config', '') + os = self.m.vars.builder_cfg.get('os', '') + target_arch = self.m.vars.builder_cfg.get('target_arch', '') + + clang_linux = str(self.m.vars.slave_dir.join('clang_linux')) + linux_vulkan_sdk = str(self.m.vars.slave_dir.join('linux_vulkan_sdk')) + win_toolchain = str(self.m.vars.slave_dir.join( + 't', 'depot_tools', 'win_toolchain', 'vs_files', + 'd3cb0e37bdd120ad0ac4650b674b09e81be45616')) + win_vulkan_sdk = str(self.m.vars.slave_dir.join('win_vulkan_sdk')) + + cc, cxx = None, None + extra_cflags = [] + extra_ldflags = [] + + if compiler == 'Clang' and os == 'Ubuntu': + cc = clang_linux + '/bin/clang' + cxx = clang_linux + '/bin/clang++' + extra_ldflags.append('-fuse-ld=lld') + elif compiler == 'Clang': + cc, cxx = 'clang', 'clang++' + elif compiler == 'GCC': + cc, cxx = 'gcc', 'g++' + + if compiler != 'MSVC' and configuration == 'Debug': + extra_cflags.append('-O1') + + if extra_config == 'Exceptions': + extra_cflags.append('/EHsc') + if extra_config == 'Fast': + extra_cflags.extend(['-march=native', '-fomit-frame-pointer', '-O3', + '-ffp-contract=off']) + if extra_config.startswith('SK'): + extra_cflags.append('-D' + extra_config) + if extra_config == 'MSAN': + extra_ldflags.append('-L' + clang_linux + '/msan') + + args = {} + + if configuration != 'Debug': + args['is_debug'] = 'false' + if extra_config == 'ANGLE': + args['skia_use_angle'] = 'true' + if extra_config == 'CommandBuffer': + self.m.run.run_once(self.build_command_buffer) + if extra_config == 'GDI': + args['skia_use_gdi'] = 'true' + if extra_config == 'MSAN': + args['skia_use_fontconfig'] = 'false' + if extra_config == 'Mesa': + args['skia_use_mesa'] = 'true' + if extra_config == 'NoGPU': + args['skia_enable_gpu'] = 'false' + if extra_config == 'Vulkan': + if os == 'Ubuntu': + args['skia_vulkan_sdk'] = '"%s"' % linux_vulkan_sdk + if 'Win' in os: + args['skia_vulkan_sdk'] = '"%s"' % win_vulkan_sdk + + for (k,v) in { + 'cc': cc, + 'cxx': cxx, + 'sanitize': extra_config if 'SAN' in extra_config else '', + 'target_cpu': target_arch, + 'target_os': 'ios' if 'iOS' in extra_config else '', + 'windk': win_toolchain if 'Win' in os else '', + }.iteritems(): + if v: + args[k] = '"%s"' % v + if extra_cflags: + args['extra_cflags'] = repr(extra_cflags).replace("'", '"') + if extra_ldflags: + args['extra_ldflags'] = repr(extra_ldflags).replace("'", '"') + + gn_args = ' '.join('%s=%s' % (k,v) for (k,v) in sorted(args.iteritems())) + + gn = 'gn.exe' if 'Win' in os else 'gn' + ninja = 'ninja.exe' if 'Win' in os else 'ninja' + gn = self.m.vars.skia_dir.join('bin', gn) + + self._py('fetch-gn', self.m.vars.skia_dir.join('bin', 'fetch-gn')) + self._run('gn gen', [gn, 'gen', self.out_dir, '--args=' + gn_args]) + self._run('ninja', [ninja, '-C', self.out_dir]) + + def copy_extra_build_products(self, swarming_out_dir): + configuration = self.m.vars.builder_cfg.get('configuration', '') + extra_config = self.m.vars.builder_cfg.get('extra_config', '') + os = self.m.vars.builder_cfg.get('os', '') + + win_vulkan_sdk = str(self.m.vars.slave_dir.join('win_vulkan_sdk')) + if 'Win' in os and extra_config == 'Vulkan': + self.m.run.copy_build_products( + win_vulkan_sdk, + swarming_out_dir.join('out', configuration + '_x64')) + + def step(self, name, cmd, env=None, **kwargs): + app = self.m.vars.skia_out.join(self.m.vars.configuration, cmd[0]) + cmd = [app] + cmd[1:] + env = {} + + clang_linux = str(self.m.vars.slave_dir.join('clang_linux')) + extra_config = self.m.vars.builder_cfg.get('extra_config', '') + + if 'SAN' in extra_config: + # Sanitized binaries may want to run clang_linux/bin/llvm-symbolizer. + self.m.vars.default_env['PATH'] = '%%(PATH)s:%s' % clang_linux + '/bin' + elif 'Ubuntu' == self.m.vars.builder_cfg.get('os', ''): + cmd = ['catchsegv'] + cmd + + if 'ASAN' == extra_config: + env[ 'ASAN_OPTIONS'] = 'symbolize=1 detect_leaks=1' + env[ 'LSAN_OPTIONS'] = 'symbolize=1 print_suppressions=1' + env['UBSAN_OPTIONS'] = 'symbolize=1 print_stacktrace=1' + + if 'MSAN' == extra_config: + # Find the MSAN-built libc++. + env['LD_LIBRARY_PATH'] = clang_linux + '/msan' + + self._run(name, cmd, env=env) diff --git a/infra/bots/recipe_modules/flavor/ios_flavor.py b/infra/bots/recipe_modules/flavor/ios_flavor.py new file mode 100644 index 0000000000..5d33d1c4df --- /dev/null +++ b/infra/bots/recipe_modules/flavor/ios_flavor.py @@ -0,0 +1,168 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +# pylint: disable=W0201 + + +import copy +import default_flavor + + +"""iOS flavor utils, used for building for and running tests on iOS.""" + + +class iOSFlavorUtils(default_flavor.DefaultFlavorUtils): + def __init__(self, m): + super(iOSFlavorUtils, self).__init__(m) + self.default_env = {} + self.default_env['XCODEBUILD'] = ( + self.m.vars.slave_dir.join('xcodebuild')) + self.ios_bin = self.m.vars.skia_dir.join( + 'platform_tools', 'ios', 'bin') + + def step(self, name, cmd, **kwargs): + args = [self.ios_bin.join('ios_run_skia')] + env = {} + env.update(kwargs.pop('env', {})) + env.update(self.default_env) + # Convert 'dm' and 'nanobench' from positional arguments + # to flags, which is what iOSShell expects to select which + # one is being run. + cmd = ["--" + c if c in ['dm', 'nanobench'] else c + for c in cmd] + return self.m.run(self.m.step, name=name, cmd=args + cmd, + env=env, **kwargs) + + def compile(self, target, **kwargs): + """Build the given target.""" + cmd = [self.ios_bin.join('ios_ninja')] + self.m.run(self.m.step, 'build iOSShell', cmd=cmd, + cwd=self.m.path['checkout'], **kwargs) + + def device_path_join(self, *args): + """Like os.path.join(), but for paths on a connected iOS device.""" + return '/'.join(args) + + def _remove_device_dir(self, path): + """Remove the directory on the device.""" + return self.m.run( + self.m.step, + 'rmdir %s' % path, + cmd=[self.ios_bin.join('ios_rm'), path], + env=self.default_env, + infra_step=True, + ) + + def _create_device_dir(self, path): + """Create the directory on the device.""" + return self.m.run( + self.m.step, + 'mkdir %s' % path, + cmd=[self.ios_bin.join('ios_mkdir'), path], + env=self.default_env, + infra_step=True, + ) + + def copy_directory_contents_to_device(self, host_dir, device_dir): + """Like shutil.copytree(), but for copying to a connected device.""" + return self.m.run( + self.m.step, + name='push %s to %s' % (self.m.path.basename(host_dir), + self.m.path.basename(device_dir)), + cmd=[self.ios_bin.join('ios_push_if_needed'), + host_dir, device_dir], + env=self.default_env, + infra_step=True, + ) + + def copy_directory_contents_to_host(self, device_dir, host_dir): + """Like shutil.copytree(), but for copying from a connected device.""" + self.m.run( + self.m.step, + name='pull %s' % self.m.path.basename(device_dir), + cmd=[self.ios_bin.join('ios_pull_if_needed'), + device_dir, host_dir], + env=self.default_env, + infra_step=True, + ) + + def copy_file_to_device(self, host_path, device_path): + """Like shutil.copyfile, but for copying to a connected device.""" + self.m.run( + self.m.step, + name='push %s' % host_path, + cmd=[self.ios_bin.join('ios_push_file'), host_path, device_path], + env=self.default_env, + infra_step=True, + ) + + def copy_extra_build_products(self, swarming_out_dir): + xcode_dir = self.m.path.join( + 'xcodebuild', '%s-iphoneos' % self.m.vars.configuration) + self.m.run.copy_build_products( + self.m.vars.skia_dir.join(xcode_dir), + swarming_out_dir.join(xcode_dir)) + + def create_clean_device_dir(self, path): + """Like shutil.rmtree() + os.makedirs(), but on a connected device.""" + self._remove_device_dir(path) + self._create_device_dir(path) + + def install(self): + """Run device-specific installation steps.""" + prefix = self.device_path_join('skiabot', 'skia_') + self.device_dirs = default_flavor.DeviceDirs( + dm_dir=prefix + 'dm', + perf_data_dir=prefix + 'perf', + resource_dir=prefix + 'resources', + images_dir=prefix + 'images', + skp_dir=prefix + 'skp/skps', + svg_dir=prefix + 'svg/svgs', + tmp_dir=prefix + 'tmp_dir') + + self.m.run( + self.m.step, + name='install iOSShell', + cmd=[self.ios_bin.join('ios_install')], + env=self.default_env, + infra_step=True) + + def cleanup_steps(self): + """Run any device-specific cleanup steps.""" + if self.m.vars.role in (self.m.builder_name_schema.BUILDER_ROLE_TEST, + self.m.builder_name_schema.BUILDER_ROLE_PERF): + self.m.run( + self.m.step, + name='reboot', + cmd=[self.ios_bin.join('ios_restart')], + env=self.default_env, + infra_step=True) + self.m.run( + self.m.step, + name='wait for reboot', + cmd=['sleep', '20'], + env=self.default_env, + infra_step=True) + + def read_file_on_device(self, path): + """Read the given file.""" + ret = self.m.run( + self.m.step, + name='read %s' % self.m.path.basename(path), + cmd=[self.ios_bin.join('ios_cat_file'), path], + env=self.default_env, + stdout=self.m.raw_io.output(), + infra_step=True) + return ret.stdout.rstrip() if ret.stdout else ret.stdout + + def remove_file_on_device(self, path): + """Remove the file on the device.""" + return self.m.run( + self.m.step, + 'rm %s' % path, + cmd=[self.ios_bin.join('ios_rm'), path], + env=self.default_env, + infra_step=True, + ) diff --git a/infra/bots/recipe_modules/flavor/pdfium_flavor.py b/infra/bots/recipe_modules/flavor/pdfium_flavor.py new file mode 100644 index 0000000000..3482560841 --- /dev/null +++ b/infra/bots/recipe_modules/flavor/pdfium_flavor.py @@ -0,0 +1,64 @@ +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import re + +import default_flavor + + +"""PDFium flavor utils, used for building PDFium with Skia.""" + + +class PDFiumFlavorUtils(default_flavor.DefaultFlavorUtils): + + def compile(self, target, **kwargs): + """Build PDFium with Skia.""" + pdfium_dir = self.m.vars.checkout_root.join('pdfium') + + # Runhook to generate the gn binary in buildtools. + self.m.run( + self.m.step, + 'runhook', + cmd=['gclient', 'runhook', 'gn_linux64'], + cwd=pdfium_dir, + **kwargs) + + # Install the sysroot. + self.m.run( + self.m.step, + 'sysroot', + cmd=['python', 'build/linux/sysroot_scripts/install-sysroot.py', + '--arch=amd64'], + cwd=pdfium_dir) + + # Setup gn args. + gn_args = [ + 'pdf_is_standalone=true', + 'clang_use_chrome_plugins=false', + 'is_component_build=false', + 'is_debug=false', + ] + if 'SkiaPaths' in self.m.vars.builder_name: + gn_args.append('pdf_use_skia_paths=true') + else: + gn_args.append('pdf_use_skia=true') + + + env = kwargs.pop('env', {}) + env['CHROMIUM_BUILDTOOLS_PATH'] = str(pdfium_dir.join('buildtools')) + self.m.run( + self.m.step, + 'gn_gen', + cmd=['gn', 'gen', 'out/skia', '--args=%s' % ' '.join(gn_args)], + cwd=pdfium_dir, + env=env) + + # Build PDFium. + self.m.run( + self.m.step, + 'build_pdfium', + cmd=['ninja', '-C', 'out/skia', '-j100'], + cwd=pdfium_dir, + env=env, + **kwargs) diff --git a/infra/bots/recipe_modules/flavor/valgrind_flavor.py b/infra/bots/recipe_modules/flavor/valgrind_flavor.py new file mode 100644 index 0000000000..0ebc161118 --- /dev/null +++ b/infra/bots/recipe_modules/flavor/valgrind_flavor.py @@ -0,0 +1,27 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +import gn_flavor + + +"""Utils for running under Valgrind.""" + + +class ValgrindFlavorUtils(gn_flavor.GNFlavorUtils): + def __init__(self, m): + super(ValgrindFlavorUtils, self).__init__(m) + self._suppressions_file = self.m.vars.skia_dir.join( + 'tools', 'valgrind.supp') + + def step(self, name, cmd, **kwargs): + new_cmd = ['valgrind', '--gen-suppressions=all', '--leak-check=full', + '--track-origins=yes', '--error-exitcode=1', '--num-callers=40', + '--suppressions=%s' % self._suppressions_file] + path_to_app = self.out_dir.join(cmd[0]) + new_cmd.append(path_to_app) + new_cmd.extend(cmd[1:]) + return self.m.run(self.m.step, name, cmd=new_cmd, + **kwargs) + |