aboutsummaryrefslogtreecommitdiffhomepage
path: root/infra/bots/recipe_modules/flavor
diff options
context:
space:
mode:
Diffstat (limited to 'infra/bots/recipe_modules/flavor')
-rw-r--r--infra/bots/recipe_modules/flavor/__init__.py15
-rw-r--r--infra/bots/recipe_modules/flavor/android_flavor.py312
-rw-r--r--infra/bots/recipe_modules/flavor/api.py126
-rw-r--r--infra/bots/recipe_modules/flavor/cmake_flavor.py14
-rw-r--r--infra/bots/recipe_modules/flavor/coverage_flavor.py75
-rw-r--r--infra/bots/recipe_modules/flavor/default_flavor.py232
-rw-r--r--infra/bots/recipe_modules/flavor/gn_flavor.py53
-rw-r--r--infra/bots/recipe_modules/flavor/ios_flavor.py176
-rw-r--r--infra/bots/recipe_modules/flavor/pdfium_flavor.py64
-rw-r--r--infra/bots/recipe_modules/flavor/valgrind_flavor.py27
-rw-r--r--infra/bots/recipe_modules/flavor/xsan_flavor.py75
11 files changed, 1169 insertions, 0 deletions
diff --git a/infra/bots/recipe_modules/flavor/__init__.py b/infra/bots/recipe_modules/flavor/__init__.py
new file mode 100644
index 0000000000..ce433e047d
--- /dev/null
+++ b/infra/bots/recipe_modules/flavor/__init__.py
@@ -0,0 +1,15 @@
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+DEPS = [
+ 'build/adb',
+ 'build/file',
+ 'recipe_engine/path',
+ 'recipe_engine/platform',
+ 'recipe_engine/python',
+ 'recipe_engine/raw_io',
+ 'recipe_engine/step',
+ 'run',
+ 'vars',
+]
diff --git a/infra/bots/recipe_modules/flavor/android_flavor.py b/infra/bots/recipe_modules/flavor/android_flavor.py
new file mode 100644
index 0000000000..35e0b35b0b
--- /dev/null
+++ b/infra/bots/recipe_modules/flavor/android_flavor.py
@@ -0,0 +1,312 @@
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+# pylint: disable=W0201
+
+
+import copy
+import default_flavor
+
+
+"""Android flavor utils, used for building for and running tests on Android."""
+
+
+class _ADBWrapper(object):
+ """Wrapper for the ADB recipe module.
+
+ The ADB recipe module looks for the ADB binary at a path we don't have checked
+ out on our bots. This wrapper ensures that we set a custom ADB path before
+ attempting to use the module.
+ """
+ def __init__(self, m, path_to_adb, serial_args, android_flavor):
+ self.m = m
+ self.m.adb.set_adb_path(path_to_adb)
+ self._has_root = False # This is set in install().
+ self._serial_args = serial_args
+ self._wait_count = 0
+ self._android_flavor = android_flavor
+
+ def wait_for_device(self):
+ """Run 'adb wait-for-device'."""
+ self._wait_count += 1
+ cmd = [
+ self._android_flavor.android_bin.join('adb_wait_for_device')
+ ] + self._serial_args
+ self.m.run(
+ self.m.step,
+ name='wait for device (%d)' % self._wait_count,
+ cmd=cmd,
+ env=self._android_flavor._default_env,
+ infra_step=True)
+
+ cmd = [
+ self._android_flavor.android_bin.join('adb_wait_for_charge'),
+ ] + self._serial_args
+ self.m.run(
+ self.m.step,
+ name='wait for charge (%d)' % self._wait_count,
+ cmd=cmd,
+ env=self._android_flavor._default_env,
+ infra_step=True)
+
+ def maybe_wait_for_device(self):
+ """Run 'adb wait-for-device' if it hasn't already been run."""
+ if self._wait_count == 0:
+ self.wait_for_device()
+
+ def __call__(self, *args, **kwargs):
+ self.maybe_wait_for_device()
+ return self.m.run(self.m.adb, *args, **kwargs)
+
+
+class AndroidFlavorUtils(default_flavor.DefaultFlavorUtils):
+ def __init__(self, m):
+ super(AndroidFlavorUtils, self).__init__(m)
+ self.device = self.m.vars.builder_spec['device_cfg']
+ self.android_bin = self.m.vars.skia_dir.join(
+ 'platform_tools', 'android', 'bin')
+ self._android_sdk_root = self.m.vars.slave_dir.join(
+ 'android_sdk', 'android-sdk')
+ self.serial = None
+ self.serial_args = []
+ try:
+ path_to_adb = self.m.step(
+ 'which adb',
+ cmd=['which', 'adb'],
+ stdout=self.m.raw_io.output(),
+ infra_step=True).stdout.rstrip()
+ except self.m.step.StepFailure:
+ path_to_adb = self.m.path.join(self._android_sdk_root,
+ 'platform-tools', 'adb')
+ self._adb = _ADBWrapper(
+ self.m, path_to_adb, self.serial_args, self)
+ self._default_env = {'ANDROID_SDK_ROOT': self._android_sdk_root,
+ 'ANDROID_HOME': self._android_sdk_root,
+ 'SKIA_ANDROID_VERBOSE_SETUP': 1}
+
+ def step(self, name, cmd, env=None, **kwargs):
+ self._adb.maybe_wait_for_device()
+ args = [
+ self.android_bin.join('android_run_skia'),
+ '--verbose',
+ '--logcat',
+ '-d', self.device,
+ ] + self.serial_args + [
+ '-t', self.m.vars.configuration,
+ ]
+ env = dict(env or {})
+ env.update(self._default_env)
+
+ return self.m.run(self.m.step, name=name, cmd=args + cmd,
+ env=env, **kwargs)
+
+ def compile(self, target):
+ """Build the given target."""
+ env = dict(self._default_env)
+ ccache = self.m.run.ccache()
+ if ccache:
+ env['ANDROID_MAKE_CCACHE'] = ccache
+
+ cmd = [self.android_bin.join('android_ninja'), target, '-d', self.device]
+ if 'Clang' in self.m.vars.builder_name:
+ cmd.append('--clang')
+ if 'GCC' in self.m.vars.builder_name:
+ cmd.append('--gcc')
+ if 'Vulkan' in self.m.vars.builder_name:
+ cmd.append('--vulkan')
+ self.m.run(self.m.step, 'build %s' % target, cmd=cmd,
+ env=env, cwd=self.m.path['checkout'])
+
+ def device_path_join(self, *args):
+ """Like os.path.join(), but for paths on a connected Android device."""
+ return '/'.join(args)
+
+ def device_path_exists(self, path):
+ """Like os.path.exists(), but for paths on a connected device."""
+ exists_str = 'FILE_EXISTS'
+ return exists_str in self._adb(
+ name='exists %s' % self.m.path.basename(path),
+ serial=self.serial,
+ cmd=['shell', 'if', '[', '-e', path, '];',
+ 'then', 'echo', exists_str + ';', 'fi'],
+ stdout=self.m.raw_io.output(),
+ infra_step=True
+ ).stdout
+
+ def _remove_device_dir(self, path):
+ """Remove the directory on the device."""
+ self._adb(name='rmdir %s' % self.m.path.basename(path),
+ serial=self.serial,
+ cmd=['shell', 'rm', '-r', path],
+ infra_step=True)
+ # Sometimes the removal fails silently. Verify that it worked.
+ if self.device_path_exists(path):
+ raise Exception('Failed to remove %s!' % path) # pragma: no cover
+
+ def _create_device_dir(self, path):
+ """Create the directory on the device."""
+ self._adb(name='mkdir %s' % self.m.path.basename(path),
+ serial=self.serial,
+ cmd=['shell', 'mkdir', '-p', path],
+ infra_step=True)
+
+ def copy_directory_contents_to_device(self, host_dir, device_dir):
+ """Like shutil.copytree(), but for copying to a connected device."""
+ self.m.run(
+ self.m.step,
+ name='push %s' % self.m.path.basename(host_dir),
+ cmd=[
+ self.android_bin.join('adb_push_if_needed'), '--verbose',
+ ] + self.serial_args + [
+ host_dir, device_dir,
+ ],
+ env=self._default_env,
+ infra_step=True)
+
+ def copy_directory_contents_to_host(self, device_dir, host_dir):
+ """Like shutil.copytree(), but for copying from a connected device."""
+ self.m.run(
+ self.m.step,
+ name='pull %s' % self.m.path.basename(device_dir),
+ cmd=[
+ self.android_bin.join('adb_pull_if_needed'), '--verbose',
+ ] + self.serial_args + [
+ device_dir, host_dir,
+ ],
+ env=self._default_env,
+ infra_step=True)
+
+ def copy_file_to_device(self, host_path, device_path):
+ """Like shutil.copyfile, but for copying to a connected device."""
+ self._adb(name='push %s' % self.m.path.basename(host_path),
+ serial=self.serial,
+ cmd=['push', host_path, device_path],
+ infra_step=True)
+
+ def create_clean_device_dir(self, path):
+ """Like shutil.rmtree() + os.makedirs(), but on a connected device."""
+ self._remove_device_dir(path)
+ self._create_device_dir(path)
+
+ def has_root(self):
+ """Determine if we have root access on this device."""
+ # Special case: GalaxyS3 hangs on `adb root`. Don't bother.
+ if 'GalaxyS3' in self.m.vars.builder_name:
+ return False
+
+ # Determine if we have root access.
+ has_root = False
+ try:
+ output = self._adb(name='adb root',
+ serial=self.serial,
+ cmd=['root'],
+ stdout=self.m.raw_io.output(),
+ infra_step=True).stdout.rstrip()
+ if ('restarting adbd as root' in output or
+ 'adbd is already running as root' in output):
+ has_root = True
+ except self.m.step.StepFailure: # pragma: nocover
+ pass
+ # Wait for the device to reconnect.
+ self.m.run(
+ self.m.step,
+ name='wait',
+ cmd=['sleep', '10'],
+ infra_step=True)
+ self._adb.wait_for_device()
+ return has_root
+
+ def install(self):
+ """Run device-specific installation steps."""
+ device_scratch_dir = self._adb(
+ name='get EXTERNAL_STORAGE dir',
+ serial=self.serial,
+ cmd=['shell', 'echo', '$EXTERNAL_STORAGE'],
+ stdout=self.m.raw_io.output(),
+ infra_step=True,
+ ).stdout.rstrip()
+ prefix = self.device_path_join(device_scratch_dir, 'skiabot', 'skia_')
+ self.device_dirs = default_flavor.DeviceDirs(
+ dm_dir=prefix + 'dm',
+ perf_data_dir=prefix + 'perf',
+ resource_dir=prefix + 'resources',
+ images_dir=prefix + 'images',
+ skp_dir=prefix + 'skp/skps',
+ tmp_dir=prefix + 'tmp_dir')
+
+ self._has_root = self.has_root()
+ self.m.run(self.m.step,
+ name='kill skia',
+ cmd=[
+ self.android_bin.join('android_kill_skia'),
+ '--verbose',
+ ] + self.serial_args,
+ env=self._default_env,
+ infra_step=True)
+ if self._has_root:
+ self._adb(name='stop shell',
+ serial=self.serial,
+ cmd=['shell', 'stop'],
+ infra_step=True)
+
+ # Print out battery stats.
+ self._adb(name='starting battery stats',
+ serial=self.serial,
+ cmd=['shell', 'dumpsys', 'batteryproperties'],
+ infra_step=True)
+
+ # Print out CPU scale info.
+ if self._has_root:
+ self._adb(name='cat scaling_governor',
+ serial=self.serial,
+ cmd=['shell', 'cat',
+ '/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor'],
+ infra_step=True)
+ self._adb(name='cat cpu_freq',
+ serial=self.serial,
+ cmd=['shell', 'cat',
+ '/sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq'],
+ infra_step=True)
+
+ def cleanup_steps(self):
+ """Run any device-specific cleanup steps."""
+ if self.m.vars.do_test_steps or self.m.vars.do_perf_steps:
+ self._adb(name='final battery stats',
+ serial=self.serial,
+ cmd=['shell', 'dumpsys', 'batteryproperties'],
+ infra_step=True)
+ self._adb(name='reboot',
+ serial=self.serial,
+ cmd=['reboot'],
+ infra_step=True)
+ self.m.run(
+ self.m.step,
+ name='wait for reboot',
+ cmd=['sleep', '10'],
+ infra_step=True)
+ self._adb.wait_for_device()
+ # The ADB binary conflicts with py-adb used by swarming. Kill it
+ # when finished to play nice.
+ self._adb(name='kill-server',
+ serial=self.serial,
+ cmd=['kill-server'],
+ infra_step=True)
+
+ def read_file_on_device(self, path, *args, **kwargs):
+ """Read the given file."""
+ return self._adb(name='read %s' % self.m.path.basename(path),
+ serial=self.serial,
+ cmd=['shell', 'cat', path],
+ stdout=self.m.raw_io.output(),
+ infra_step=True).stdout.rstrip()
+
+ def remove_file_on_device(self, path, *args, **kwargs):
+ """Delete the given file."""
+ return self._adb(name='rm %s' % self.m.path.basename(path),
+ serial=self.serial,
+ cmd=['shell', 'rm', '-f', path],
+ infra_step=True,
+ *args,
+ **kwargs)
diff --git a/infra/bots/recipe_modules/flavor/api.py b/infra/bots/recipe_modules/flavor/api.py
new file mode 100644
index 0000000000..fbfa9ba1cd
--- /dev/null
+++ b/infra/bots/recipe_modules/flavor/api.py
@@ -0,0 +1,126 @@
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+# pylint: disable=W0201
+
+
+from recipe_engine import recipe_api
+
+from . import android_flavor
+from . import cmake_flavor
+from . import coverage_flavor
+from . import default_flavor
+from . import gn_flavor
+from . import ios_flavor
+from . import pdfium_flavor
+from . import valgrind_flavor
+from . import xsan_flavor
+
+
+def is_android(builder_cfg):
+ """Determine whether the given builder is an Android builder."""
+ return ('Android' in builder_cfg.get('extra_config', '') or
+ builder_cfg.get('os') == 'Android')
+
+
+def is_cmake(builder_cfg):
+ return 'CMake' in builder_cfg.get('extra_config', '')
+
+
+def is_gn(builder_cfg):
+ return 'GN' == builder_cfg.get('extra_config', '')
+
+
+def is_ios(builder_cfg):
+ return ('iOS' in builder_cfg.get('extra_config', '') or
+ builder_cfg.get('os') == 'iOS')
+
+
+def is_pdfium(builder_cfg):
+ return 'PDFium' in builder_cfg.get('extra_config', '')
+
+
+def is_valgrind(builder_cfg):
+ return 'Valgrind' in builder_cfg.get('extra_config', '')
+
+
+def is_xsan(builder_cfg):
+ return ('ASAN' in builder_cfg.get('extra_config', '') or
+ 'MSAN' in builder_cfg.get('extra_config', '') or
+ 'TSAN' in builder_cfg.get('extra_config', ''))
+
+
+class SkiaFlavorApi(recipe_api.RecipeApi):
+ def get_flavor(self, builder_cfg):
+ """Return a flavor utils object specific to the given builder."""
+ if is_android(builder_cfg):
+ return android_flavor.AndroidFlavorUtils(self.m)
+ elif is_cmake(builder_cfg):
+ return cmake_flavor.CMakeFlavorUtils(self.m)
+ elif is_gn(builder_cfg):
+ return gn_flavor.GNFlavorUtils(self.m)
+ elif is_ios(builder_cfg):
+ return ios_flavor.iOSFlavorUtils(self.m)
+ elif is_pdfium(builder_cfg):
+ return pdfium_flavor.PDFiumFlavorUtils(self.m)
+ elif is_valgrind(builder_cfg):
+ return valgrind_flavor.ValgrindFlavorUtils(self.m)
+ elif is_xsan(builder_cfg):
+ return xsan_flavor.XSanFlavorUtils(self.m)
+ elif builder_cfg.get('configuration') == 'Coverage':
+ return coverage_flavor.CoverageFlavorUtils(self.m)
+ else:
+ return default_flavor.DefaultFlavorUtils(self.m)
+
+ def setup(self):
+ self._f = self.get_flavor(self.m.vars.builder_cfg)
+
+ def step(self, name, cmd, **kwargs):
+ return self._f.step(name, cmd, **kwargs)
+
+ def compile(self, target):
+ return self._f.compile(target)
+
+ def copy_extra_build_products(self, swarming_out_dir):
+ return self._f.copy_extra_build_products(swarming_out_dir)
+
+ @property
+ def out_dir(self):
+ return self._f.out_dir
+
+ def device_path_join(self, *args):
+ return self._f.device_path_join(*args)
+
+ def device_path_exists(self, path):
+ return self._f.device_path_exists(path) # pragma: no cover
+
+ def copy_directory_contents_to_device(self, host_dir, device_dir):
+ return self._f.copy_directory_contents_to_device(host_dir, device_dir)
+
+ def copy_directory_contents_to_host(self, device_dir, host_dir):
+ return self._f.copy_directory_contents_to_host(device_dir, host_dir)
+
+ def copy_file_to_device(self, host_path, device_path):
+ return self._f.copy_file_to_device(host_path, device_path)
+
+ def create_clean_host_dir(self, path):
+ return self._f.create_clean_host_dir(path)
+
+ def create_clean_device_dir(self, path):
+ return self._f.create_clean_device_dir(path)
+
+ def read_file_on_device(self, path):
+ return self._f.read_file_on_device(path)
+
+ def remove_file_on_device(self, path):
+ return self._f.remove_file_on_device(path)
+
+ def install(self):
+ rv = self._f.install()
+ self.device_dirs = self._f.device_dirs
+ return rv
+
+ def cleanup_steps(self):
+ return self._f.cleanup_steps()
diff --git a/infra/bots/recipe_modules/flavor/cmake_flavor.py b/infra/bots/recipe_modules/flavor/cmake_flavor.py
new file mode 100644
index 0000000000..8b254eceb7
--- /dev/null
+++ b/infra/bots/recipe_modules/flavor/cmake_flavor.py
@@ -0,0 +1,14 @@
+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import default_flavor
+
+"""CMake flavor utils, used for building Skia with CMake."""
+
+class CMakeFlavorUtils(default_flavor.DefaultFlavorUtils):
+ def compile(self, target):
+ """Build Skia with CMake. Ignores `target`."""
+ cmake_build = self.m.vars.skia_dir.join('cmake', 'cmake_build')
+ self.m.run(self.m.step, 'cmake_build', cmd=[cmake_build],
+ cwd=self.m.path['checkout'])
diff --git a/infra/bots/recipe_modules/flavor/coverage_flavor.py b/infra/bots/recipe_modules/flavor/coverage_flavor.py
new file mode 100644
index 0000000000..a3bb5828ce
--- /dev/null
+++ b/infra/bots/recipe_modules/flavor/coverage_flavor.py
@@ -0,0 +1,75 @@
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+import datetime
+import default_flavor
+import posixpath
+
+
+"""Utils for running coverage tests."""
+
+
+class CoverageFlavorUtils(default_flavor.DefaultFlavorUtils):
+
+ def step(self, name, cmd, **kwargs):
+ """Run the given step through coverage."""
+ compile_target = 'dm'
+ build_cmd = [self.m.vars.skia_dir.join('tools', 'llvm_coverage_build'),
+ compile_target]
+ self.m.run(self.m.step,
+ 'build %s' % compile_target,
+ cmd=build_cmd,
+ cwd=self.m.path['checkout'])
+
+ # Slice out the 'key' and 'properties' arguments to be reused.
+ key = []
+ properties = []
+ current = None
+ for i in xrange(0, len(cmd)):
+ if isinstance(cmd[i], basestring) and cmd[i] == '--key':
+ current = key
+ elif isinstance(cmd[i], basestring) and cmd[i] == '--properties':
+ current = properties
+ elif isinstance(cmd[i], basestring) and cmd[i].startswith('--'):
+ current = None
+ if current is not None:
+ current.append(cmd[i])
+
+ results_dir = self.m.vars.skia_out.join('coverage_results')
+ self.create_clean_host_dir(results_dir)
+
+ # Run DM under coverage.
+ report_file_basename = '%s.cov' % self.m.vars.got_revision
+ report_file = results_dir.join(report_file_basename)
+ args = [
+ 'python',
+ self.m.vars.skia_dir.join('tools', 'llvm_coverage_run.py'),
+ ] + cmd + ['--outResultsFile', report_file]
+ self.m.run(self.m.step, name=name, cmd=args,
+ cwd=self.m.path['checkout'], **kwargs)
+
+ # Generate nanobench-style JSON output from the coverage report.
+ nanobench_json = results_dir.join('nanobench_%s.json' % (
+ self.m.vars.got_revision))
+ line_by_line_basename = ('coverage_by_line_%s.json' % (
+ self.m.vars.got_revision))
+ line_by_line = results_dir.join(line_by_line_basename)
+ args = [
+ 'python',
+ self.m.vars.skia_dir.join('tools', 'parse_llvm_coverage.py'),
+ '--report', report_file, '--nanobench', nanobench_json,
+ '--linebyline', line_by_line]
+ args.extend(key)
+ args.extend(properties)
+ self.m.run(
+ self.m.step,
+ 'Generate Coverage Data',
+ cmd=args, cwd=self.m.path['checkout'])
+
+ # Copy files from results_dir into swarming_out_dir.
+ for r in self.m.file.listdir('results_dir', results_dir):
+ self.m.file.copy(
+ 'Copy to swarming out', results_dir.join(r),
+ self.m.vars.swarming_out_dir)
diff --git a/infra/bots/recipe_modules/flavor/default_flavor.py b/infra/bots/recipe_modules/flavor/default_flavor.py
new file mode 100644
index 0000000000..4cdbaaa671
--- /dev/null
+++ b/infra/bots/recipe_modules/flavor/default_flavor.py
@@ -0,0 +1,232 @@
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+# pylint: disable=W0201
+
+
+"""Default flavor utils class, used for desktop builders."""
+
+
+import json
+
+
+WIN_TOOLCHAIN_DIR = 't'
+
+
+class DeviceDirs(object):
+ def __init__(self,
+ dm_dir,
+ perf_data_dir,
+ resource_dir,
+ images_dir,
+ skp_dir,
+ tmp_dir):
+ self._dm_dir = dm_dir
+ self._perf_data_dir = perf_data_dir
+ self._resource_dir = resource_dir
+ self._images_dir = images_dir
+ self._skp_dir = skp_dir
+ self._tmp_dir = tmp_dir
+
+ @property
+ def dm_dir(self):
+ """Where DM writes."""
+ return self._dm_dir
+
+ @property
+ def perf_data_dir(self):
+ return self._perf_data_dir
+
+ @property
+ def resource_dir(self):
+ return self._resource_dir
+
+ @property
+ def images_dir(self):
+ return self._images_dir
+
+ @property
+ def skp_dir(self):
+ """Holds SKP files that are consumed by RenderSKPs and BenchPictures."""
+ return self._skp_dir
+
+ @property
+ def tmp_dir(self):
+ return self._tmp_dir
+
+
+class DefaultFlavorUtils(object):
+ """Utilities to be used by build steps.
+
+ The methods in this class define how certain high-level functions should
+ work. Each build step flavor should correspond to a subclass of
+ DefaultFlavorUtils which may override any of these functions as appropriate
+ for that flavor.
+
+ For example, the AndroidFlavorUtils will override the functions for
+ copying files between the host and Android device, as well as the
+ 'step' function, so that commands may be run through ADB.
+ """
+ def __init__(self, m):
+ self.m = m
+ self._chrome_path = None
+ self._win_toolchain_dir = self.m.vars.slave_dir.join(WIN_TOOLCHAIN_DIR)
+ win_toolchain_asset_path = self.m.vars.infrabots_dir.join(
+ 'assets', 'win_toolchain', 'VERSION')
+ if not self.m.path.exists(win_toolchain_asset_path):
+ self._win_toolchain_dir = self.m.vars.slave_dir
+
+
+ def step(self, name, cmd, **kwargs):
+ """Wrapper for the Step API; runs a step as appropriate for this flavor."""
+ path_to_app = self.m.vars.skia_out.join(
+ self.m.vars.configuration, cmd[0])
+ if (self.m.platform.is_linux and
+ 'x86_64' in self.m.vars.builder_name and
+ not 'TSAN' in self.m.vars.builder_name):
+ new_cmd = ['catchsegv', path_to_app]
+ else:
+ new_cmd = [path_to_app]
+ new_cmd.extend(cmd[1:])
+ return self.m.run(self.m.step,
+ name, cmd=new_cmd, **kwargs)
+
+ @property
+ def chrome_path(self):
+ """Path to a checkout of Chrome on this machine."""
+ return self._win_toolchain_dir.join('src')
+
+ def bootstrap_win_toolchain(self):
+ """Run bootstrapping script for the Windows toolchain."""
+ bootstrap_script = self.m.vars.infrabots_dir.join(
+ 'bootstrap_win_toolchain_json.py')
+ win_toolchain_json = self._win_toolchain_dir.join(
+ 'src', 'build', 'win_toolchain.json')
+ self.m.python(
+ 'bootstrap win toolchain',
+ script=bootstrap_script,
+ args=['--win_toolchain_json', win_toolchain_json,
+ '--depot_tools_parent_dir',
+ self._win_toolchain_dir])
+
+ def build_command_buffer(self):
+ """Build command_buffer."""
+ script = self.m.vars.skia_dir.join('tools', 'build_command_buffer.py')
+ self.m.run(
+ self.m.python, 'build command_buffer',
+ script=script,
+ args=['--chrome-dir', self.m.vars.checkout_root,
+ '--output-dir', self.out_dir,
+ '--chrome-build-type', self.m.vars.configuration,
+ '--no-sync'])
+
+ def compile(self, target):
+ """Build the given target."""
+ # The CHROME_PATH environment variable is needed for builders that use
+ # toolchains downloaded by Chrome.
+ env = {'CHROME_PATH': self.chrome_path}
+ if self.m.platform.is_win:
+ make_cmd = ['python', 'make.py']
+ self.m.run.run_once(self.bootstrap_win_toolchain)
+ if 'Vulkan' in self.m.vars.builder_name:
+ env['VK_SDK_PATH'] = self.m.vars.slave_dir.join('win_vulkan_sdk')
+ else:
+ make_cmd = ['make']
+ cmd = make_cmd + [target]
+ try:
+ self.m.run(self.m.step, 'build %s' % target, cmd=cmd,
+ env=env, cwd=self.m.path['checkout'])
+ except self.m.step.StepFailure:
+ if self.m.platform.is_win:
+ # The linker occasionally crashes on Windows. Try again.
+ self.m.run(self.m.step, 'build %s' % target, cmd=cmd,
+ env=env, cwd=self.m.path['checkout'])
+ else:
+ raise
+ if 'CommandBuffer' in self.m.vars.builder_name:
+ self.m.run.run_once(self.build_command_buffer)
+
+ def copy_extra_build_products(self, swarming_out_dir):
+ """Copy extra build products to specified directory.
+
+ Copy flavor-specific build products to swarming_out_dir for use in test and
+ perf steps."""
+ if ("Win" in self.m.vars.builder_name and
+ "Vulkan" in self.m.vars.builder_name):
+ # This copies vulkan-1.dll that has been bundled into win_vulkan_sdk
+ # since version 2 See skia/api BUILD_PRODUCTS_ISOLATE_WHITELIST
+ self.m.run.copy_build_products(
+ self.m.path['slave_build'].join('win_vulkan_sdk'),
+ swarming_out_dir)
+
+ @property
+ def out_dir(self):
+ """Flavor-specific out directory."""
+ return self.m.vars.skia_out.join(self.m.vars.configuration)
+
+ def device_path_join(self, *args):
+ """Like os.path.join(), but for paths on a connected device."""
+ return self.m.path.join(*args)
+
+ def device_path_exists(self, path): # pragma: no cover
+ """Like os.path.exists(), but for paths on a connected device."""
+ return self.m.path.exists(path, infra_step=True)
+
+ def copy_directory_contents_to_device(self, host_dir, device_dir):
+ """Like shutil.copytree(), but for copying to a connected device."""
+ # For "normal" builders who don't have an attached device, we expect
+ # host_dir and device_dir to be the same.
+ if str(host_dir) != str(device_dir):
+ raise ValueError('For builders who do not have attached devices, copying '
+ 'from host to device is undefined and only allowed if '
+ 'host_path and device_path are the same (%s vs %s).' % (
+ str(host_dir), str(device_dir))) # pragma: no cover
+
+ def copy_directory_contents_to_host(self, device_dir, host_dir):
+ """Like shutil.copytree(), but for copying from a connected device."""
+ # For "normal" builders who don't have an attached device, we expect
+ # host_dir and device_dir to be the same.
+ if str(host_dir) != str(device_dir):
+ raise ValueError('For builders who do not have attached devices, copying '
+ 'from device to host is undefined and only allowed if '
+ 'host_path and device_path are the same (%s vs %s).' % (
+ str(host_dir), str(device_dir))) # pragma: no cover
+
+ def copy_file_to_device(self, host_path, device_path):
+ """Like shutil.copyfile, but for copying to a connected device."""
+ # For "normal" builders who don't have an attached device, we expect
+ # host_dir and device_dir to be the same.
+ if str(host_path) != str(device_path): # pragma: no cover
+ raise ValueError('For builders who do not have attached devices, copying '
+ 'from host to device is undefined and only allowed if '
+ 'host_path and device_path are the same (%s vs %s).' % (
+ str(host_path), str(device_path)))
+
+ def create_clean_device_dir(self, path):
+ """Like shutil.rmtree() + os.makedirs(), but on a connected device."""
+ self.create_clean_host_dir(path)
+
+ def create_clean_host_dir(self, path):
+ """Convenience function for creating a clean directory."""
+ self.m.run.rmtree(path)
+ self.m.file.makedirs(
+ self.m.path.basename(path), path, infra_step=True)
+
+ def install(self):
+ """Run device-specific installation steps."""
+ self.device_dirs = DeviceDirs(
+ dm_dir=self.m.vars.dm_dir,
+ perf_data_dir=self.m.vars.perf_data_dir,
+ resource_dir=self.m.vars.resource_dir,
+ images_dir=self.m.vars.images_dir,
+ skp_dir=self.m.vars.local_skp_dir,
+ tmp_dir=self.m.vars.tmp_dir)
+
+ def cleanup_steps(self):
+ """Run any device-specific cleanup steps."""
+ pass
+
+ def __repr__(self):
+ return '<%s object>' % self.__class__.__name__ # pragma: no cover
diff --git a/infra/bots/recipe_modules/flavor/gn_flavor.py b/infra/bots/recipe_modules/flavor/gn_flavor.py
new file mode 100644
index 0000000000..ea5bbb6934
--- /dev/null
+++ b/infra/bots/recipe_modules/flavor/gn_flavor.py
@@ -0,0 +1,53 @@
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import default_flavor
+
+"""GN flavor utils, used for building Skia with GN."""
+class GNFlavorUtils(default_flavor.DefaultFlavorUtils):
+ def compile(self, target):
+ """Build Skia with GN."""
+ # Get the gn executable.
+ fetch_gn = self.m.vars.skia_dir.join('bin', 'fetch-gn')
+ self.m.run(self.m.step, 'fetch-gn',
+ cmd=[fetch_gn],
+ cwd=self.m.vars.skia_dir)
+
+ is_debug = 'is_debug=true'
+ if self.m.vars.configuration != 'Debug':
+ is_debug = 'is_debug=false'
+ gn_args = [is_debug]
+
+ is_clang = 'Clang' in self.m.vars.builder_name
+ is_gcc = 'GCC' in self.m.vars.builder_name
+
+ cc, cxx = 'cc', 'c++'
+ if is_clang:
+ cc, cxx = 'clang', 'clang++'
+ elif is_gcc:
+ cc, cxx = 'gcc', 'g++'
+
+ ccache = self.m.run.ccache()
+ if ccache:
+ cc, cxx = '%s %s' % (ccache, cc), '%s %s' % (ccache, cxx)
+ if is_clang:
+ # Stifle "argument unused during compilation: ..." warnings.
+ stifle = '-Qunused-arguments'
+ cc, cxx = '%s %s' % (cc, stifle), '%s %s' % (cxx, stifle)
+
+ gn_args += [ 'cc="%s"' % cc, 'cxx="%s"' % cxx ]
+
+ # Run gn gen.
+ gn_exe = 'gn'
+ if self.m.platform.is_win:
+ gn_exe = 'gn.exe'
+ gn_gen = [gn_exe, 'gen', self.out_dir, '--args=%s' % ' '.join(gn_args)]
+ self.m.run(self.m.step, 'gn_gen', cmd=gn_gen,
+ cwd=self.m.vars.skia_dir)
+
+ # Run ninja.
+ ninja_cmd = ['ninja', '-C', self.out_dir]
+ self.m.run(self.m.step, 'compile %s' % target,
+ cmd=ninja_cmd,
+ cwd=self.m.vars.skia_dir)
diff --git a/infra/bots/recipe_modules/flavor/ios_flavor.py b/infra/bots/recipe_modules/flavor/ios_flavor.py
new file mode 100644
index 0000000000..d0d60f4284
--- /dev/null
+++ b/infra/bots/recipe_modules/flavor/ios_flavor.py
@@ -0,0 +1,176 @@
+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+# pylint: disable=W0201
+
+
+import copy
+import default_flavor
+
+
+"""iOS flavor utils, used for building for and running tests on iOS."""
+
+
+class iOSFlavorUtils(default_flavor.DefaultFlavorUtils):
+ def __init__(self, m):
+ super(iOSFlavorUtils, self).__init__(m)
+ self.default_env = {}
+ self.default_env['XCODEBUILD'] = (
+ self.m.vars.slave_dir.join('xcodebuild'))
+ self.ios_bin = self.m.vars.skia_dir.join(
+ 'platform_tools', 'ios', 'bin')
+
+ def step(self, name, cmd, **kwargs):
+ args = [self.ios_bin.join('ios_run_skia')]
+ env = {}
+ env.update(kwargs.pop('env', {}))
+ env.update(self.default_env)
+ # Convert 'dm' and 'nanobench' from positional arguments
+ # to flags, which is what iOSShell expects to select which
+ # one is being run.
+ cmd = ["--" + c if c in ['dm', 'nanobench'] else c
+ for c in cmd]
+ return self.m.run(self.m.step, name=name, cmd=args + cmd,
+ env=env, **kwargs)
+
+ def compile(self, target):
+ """Build the given target."""
+ cmd = [self.ios_bin.join('ios_ninja')]
+ self.m.run(self.m.step, 'build iOSShell', cmd=cmd,
+ cwd=self.m.path['checkout'])
+
+ def device_path_join(self, *args):
+ """Like os.path.join(), but for paths on a connected iOS device."""
+ return '/'.join(args)
+
+ def device_path_exists(self, path):
+ """Like os.path.exists(), but for paths on a connected device."""
+ return self.m.run(
+ self.m.step,
+ 'exists %s' % path,
+ cmd=[self.ios_bin.join('ios_path_exists'), path],
+ env=self.default_env,
+ infra_step=True,
+ ) # pragma: no cover
+
+ def _remove_device_dir(self, path):
+ """Remove the directory on the device."""
+ return self.m.run(
+ self.m.step,
+ 'rmdir %s' % path,
+ cmd=[self.ios_bin.join('ios_rm'), path],
+ env=self.default_env,
+ infra_step=True,
+ )
+
+ def _create_device_dir(self, path):
+ """Create the directory on the device."""
+ return self.m.run(
+ self.m.step,
+ 'mkdir %s' % path,
+ cmd=[self.ios_bin.join('ios_mkdir'), path],
+ env=self.default_env,
+ infra_step=True,
+ )
+
+ def copy_directory_contents_to_device(self, host_dir, device_dir):
+ """Like shutil.copytree(), but for copying to a connected device."""
+ return self.m.run(
+ self.m.step,
+ name='push %s to %s' % (self.m.path.basename(host_dir),
+ self.m.path.basename(device_dir)),
+ cmd=[self.ios_bin.join('ios_push_if_needed'),
+ host_dir, device_dir],
+ env=self.default_env,
+ infra_step=True,
+ )
+
+ def copy_directory_contents_to_host(self, device_dir, host_dir):
+ """Like shutil.copytree(), but for copying from a connected device."""
+ self.m.run(
+ self.m.step,
+ name='pull %s' % self.m.path.basename(device_dir),
+ cmd=[self.ios_bin.join('ios_pull_if_needed'),
+ device_dir, host_dir],
+ env=self.default_env,
+ infra_step=True,
+ )
+
+ def copy_file_to_device(self, host_path, device_path):
+ """Like shutil.copyfile, but for copying to a connected device."""
+ self.m.run(
+ self.m.step,
+ name='push %s' % host_path,
+ cmd=[self.ios_bin.join('ios_push_file'), host_path, device_path],
+ env=self.default_env,
+ infra_step=True,
+ ) # pragma: no cover
+
+ def copy_extra_build_products(self, swarming_out_dir):
+ xcode_dir = self.m.path.join(
+ 'xcodebuild', '%s-iphoneos' % self.m.vars.configuration)
+ self.m.run.copy_build_products(
+ self.m.vars.skia_dir.join(xcode_dir),
+ swarming_out_dir.join(xcode_dir))
+
+ def create_clean_device_dir(self, path):
+ """Like shutil.rmtree() + os.makedirs(), but on a connected device."""
+ self._remove_device_dir(path)
+ self._create_device_dir(path)
+
+ def install(self):
+ """Run device-specific installation steps."""
+ prefix = self.device_path_join('skiabot', 'skia_')
+ self.device_dirs = default_flavor.DeviceDirs(
+ dm_dir=prefix + 'dm',
+ perf_data_dir=prefix + 'perf',
+ resource_dir=prefix + 'resources',
+ images_dir=prefix + 'images',
+ skp_dir=prefix + 'skp/skps',
+ tmp_dir=prefix + 'tmp_dir')
+
+ self.m.run(
+ self.m.step,
+ name='install iOSShell',
+ cmd=[self.ios_bin.join('ios_install')],
+ env=self.default_env,
+ infra_step=True)
+
+ def cleanup_steps(self):
+ """Run any device-specific cleanup steps."""
+ if self.m.vars.do_test_steps or self.m.vars.do_perf_steps:
+ self.m.run(
+ self.m.step,
+ name='reboot',
+ cmd=[self.ios_bin.join('ios_restart')],
+ env=self.default_env,
+ infra_step=True)
+ self.m.run(
+ self.m.step,
+ name='wait for reboot',
+ cmd=['sleep', '20'],
+ env=self.default_env,
+ infra_step=True)
+
+ def read_file_on_device(self, path):
+ """Read the given file."""
+ ret = self.m.run(
+ self.m.step,
+ name='read %s' % self.m.path.basename(path),
+ cmd=[self.ios_bin.join('ios_cat_file'), path],
+ env=self.default_env,
+ stdout=self.m.raw_io.output(),
+ infra_step=True)
+ return ret.stdout.rstrip() if ret.stdout else ret.stdout
+
+ def remove_file_on_device(self, path):
+ """Remove the file on the device."""
+ return self.m.run(
+ self.m.step,
+ 'rm %s' % path,
+ cmd=[self.ios_bin.join('ios_rm'), path],
+ env=self.default_env,
+ infra_step=True,
+ )
diff --git a/infra/bots/recipe_modules/flavor/pdfium_flavor.py b/infra/bots/recipe_modules/flavor/pdfium_flavor.py
new file mode 100644
index 0000000000..ce11aeac2e
--- /dev/null
+++ b/infra/bots/recipe_modules/flavor/pdfium_flavor.py
@@ -0,0 +1,64 @@
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import re
+
+import default_flavor
+
+
+"""PDFium flavor utils, used for building PDFium with Skia."""
+
+
+class PDFiumFlavorUtils(default_flavor.DefaultFlavorUtils):
+
+ def compile(self, target):
+ """Build PDFium with Skia."""
+ pdfium_dir = self.m.vars.checkout_root.join('pdfium')
+
+ # Runhook to generate the gn binary in buildtools.
+ self.m.run(
+ self.m.step,
+ 'runhook',
+ cmd=['gclient', 'runhook', 'gn_linux64'],
+ cwd=pdfium_dir)
+
+ # Setup gn args.
+ gn_args = ['pdf_use_skia=true', 'pdf_is_standalone=true',
+ 'clang_use_chrome_plugins=false']
+ self.m.run(
+ self.m.step,
+ 'gn_gen',
+ cmd=['gn', 'gen', 'out/skia', '--args=%s' % ' '.join(gn_args)],
+ cwd=pdfium_dir,
+ env={'CHROMIUM_BUILDTOOLS_PATH': str(pdfium_dir.join('buildtools'))})
+
+ # Modify DEPS file to contain the current Skia revision.
+ skia_revision = self.m.vars.got_revision
+ deps_file = pdfium_dir.join('DEPS')
+ test_data = "'skia_revision': 'abc'"
+
+ original_contents = self.m.file.read(
+ 'read PDFium DEPS', deps_file, test_data=test_data, infra_step=True)
+
+ deps_skia_regexp = re.compile(
+ r'(?<=["\']skia_revision["\']: ["\'])([a-fA-F0-9]+)(?=["\'])',
+ re.MULTILINE)
+ patched_contents = re.sub(deps_skia_regexp, str(skia_revision),
+ original_contents)
+ self.m.file.write('write PDFium DEPs', deps_file,
+ patched_contents, infra_step=True)
+
+ # gclient sync after updating DEPS.
+ self.m.run(
+ self.m.step,
+ 'sync_pdfium',
+ cmd=['gclient', 'sync'],
+ cwd=pdfium_dir)
+
+ # Build PDFium.
+ self.m.run(
+ self.m.step,
+ 'build_pdfium',
+ cmd=['ninja', '-C', 'out/skia', '-j100'],
+ cwd=pdfium_dir)
diff --git a/infra/bots/recipe_modules/flavor/valgrind_flavor.py b/infra/bots/recipe_modules/flavor/valgrind_flavor.py
new file mode 100644
index 0000000000..2dea3e7ab4
--- /dev/null
+++ b/infra/bots/recipe_modules/flavor/valgrind_flavor.py
@@ -0,0 +1,27 @@
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+import default_flavor
+
+
+"""Utils for running under Valgrind."""
+
+
+class ValgrindFlavorUtils(default_flavor.DefaultFlavorUtils):
+ def __init__(self, m):
+ super(ValgrindFlavorUtils, self).__init__(m)
+ self._suppressions_file = self.m.vars.skia_dir.join(
+ 'tools', 'valgrind.supp')
+
+ def step(self, name, cmd, **kwargs):
+ new_cmd = ['valgrind', '--gen-suppressions=all', '--leak-check=full',
+ '--track-origins=yes', '--error-exitcode=1', '--num-callers=40',
+ '--suppressions=%s' % self._suppressions_file]
+ path_to_app = self.out_dir.join(cmd[0])
+ new_cmd.append(path_to_app)
+ new_cmd.extend(cmd[1:])
+ return self.m.run(self.m.step, name, cmd=new_cmd,
+ **kwargs)
+
diff --git a/infra/bots/recipe_modules/flavor/xsan_flavor.py b/infra/bots/recipe_modules/flavor/xsan_flavor.py
new file mode 100644
index 0000000000..89169b77c1
--- /dev/null
+++ b/infra/bots/recipe_modules/flavor/xsan_flavor.py
@@ -0,0 +1,75 @@
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+"""Utils for running under *SAN"""
+
+
+import default_flavor
+
+
+class XSanFlavorUtils(default_flavor.DefaultFlavorUtils):
+ def __init__(self, m):
+ super(XSanFlavorUtils, self).__init__(m)
+ self._sanitizer = {
+ # We'd love to just pass 'address,undefined' and get all the checks, but
+ # we're not anywhere close to being able to do that. Instead we start
+ # with a set of checks that we know pass or nearly pass. See here for
+ # more information:
+ # http://clang.llvm.org/docs/UsersManual.html#controlling-code-generation
+ 'ASAN': ('address,bool,function,integer-divide-by-zero,nonnull-attribute,'
+ 'null,object-size,return,returns-nonnull-attribute,shift,'
+ 'signed-integer-overflow,unreachable,vla-bound,vptr'),
+ # MSAN and TSAN can't run together with ASAN, so they're their own bots.
+ 'MSAN': 'memory',
+ 'TSAN': 'thread',
+ }[self.m.vars.builder_cfg['extra_config'].replace('Swarming', '')]
+
+ def compile(self, target):
+ cmd = [self.m.vars.skia_dir.join('tools', 'xsan_build'),
+ self._sanitizer, target]
+ self.m.run(self.m.step, 'build %s' % target, cmd=cmd,
+ cwd=self.m.vars.skia_dir)
+
+ def copy_extra_build_products(self, swarming_out_dir):
+ # Include msan_out if MSAN.
+ if 'MSAN' in self.m.vars.builder_cfg['extra_config']:
+ msan_out = self.m.path.join(
+ 'third_party', 'externals', 'llvm', 'msan_out')
+ self.m.file.copytree(
+ 'copy msan_out',
+ self.m.vars.skia_dir.join(msan_out),
+ swarming_out_dir.join(msan_out),
+ symlinks=True)
+ # Include llvm_symbolizer from the Chromium DEPS so that suppressions work
+ # by symbol name.
+ # TODO(benjaminwagner): Figure out how to add this to Skia DEPS for
+ # target_os 'llvm'.
+ self.m.file.copytree(
+ 'copy llvm-build',
+ self.m.vars.checkout_root.join('src', 'third_party', 'llvm-build'),
+ swarming_out_dir.join('llvm-build'),
+ symlinks=True)
+
+ def step(self, name, cmd, env=None, **kwargs):
+ """Wrapper for the Step API; runs a step as appropriate for this flavor."""
+ skia_dir = self.m.vars.skia_dir
+ lsan_suppressions = skia_dir.join('tools', 'lsan.supp')
+ tsan_suppressions = skia_dir.join('tools', 'tsan.supp')
+ ubsan_suppressions = skia_dir.join('tools', 'ubsan.supp')
+ env = dict(env or {})
+ env['ASAN_OPTIONS'] = 'symbolize=1 detect_leaks=1'
+ env['LSAN_OPTIONS'] = ('symbolize=1 print_suppressions=1 suppressions=%s' %
+ lsan_suppressions)
+ env['TSAN_OPTIONS'] = 'suppressions=%s' % tsan_suppressions
+ env['UBSAN_OPTIONS'] = 'suppressions=%s' % ubsan_suppressions
+ self.m.vars.default_env['PATH'] = '%%(PATH)s:%s' % (
+ self.m.vars.slave_dir.join('llvm-build', 'Release+Asserts', 'bin'))
+ env['LD_LIBRARY_PATH'] = self.m.vars.slave_dir.join(
+ 'third_party', 'externals', 'llvm', 'msan_out', 'lib')
+
+ path_to_app = self.out_dir.join(cmd[0])
+ new_cmd = [path_to_app]
+ new_cmd.extend(cmd[1:])
+ return self.m.run(self.m.step, name, cmd=new_cmd, env=env, **kwargs)
generated by cgit v1.2.3 (git 2.39.1) at 2024-07-05 12:11:47 +0000