From 15bc5603e44fdcac7418f08842974a80f935a577 Mon Sep 17 00:00:00 2001 From: borenet Date: Wed, 30 Mar 2016 07:41:32 -0700 Subject: Fixes for Swarming recipes BUG=skia:4763 GOLD_TRYBOT_URL= https://gold.skia.org/search2?unt=true&query=source_type%3Dgm&master=false&issue=1827413002 NOTRY=true Review URL: https://codereview.chromium.org/1827413002 --- infra/bots/add_isolated_input.py | 33 --- infra/bots/common.py | 436 ----------------------------------- infra/bots/compile_skia.isolate | 5 - infra/bots/compile_skia.py | 27 --- infra/bots/flavor/__init__.py | 6 - infra/bots/flavor/android_devices.py | 100 -------- infra/bots/flavor/android_flavor.py | 247 -------------------- infra/bots/flavor/chromeos_flavor.py | 59 ----- infra/bots/flavor/cmake_flavor.py | 20 -- infra/bots/flavor/coverage_flavor.py | 114 --------- infra/bots/flavor/default_flavor.py | 190 --------------- infra/bots/flavor/ios_flavor.py | 114 --------- infra/bots/flavor/ssh_devices.py | 32 --- infra/bots/flavor/ssh_flavor.py | 123 ---------- infra/bots/flavor/valgrind_flavor.py | 31 --- infra/bots/flavor/xsan_flavor.py | 54 ----- infra/bots/images.isolate | 1 + infra/bots/perf_skia.isolate | 13 -- infra/bots/perf_skia.py | 35 --- infra/bots/skps.isolate | 1 + infra/bots/test_skia.isolate | 13 -- infra/bots/test_skia.py | 35 --- 22 files changed, 2 insertions(+), 1687 deletions(-) delete mode 100644 infra/bots/add_isolated_input.py delete mode 100644 infra/bots/compile_skia.py delete mode 100644 infra/bots/flavor/__init__.py delete mode 100644 infra/bots/flavor/android_devices.py delete mode 100644 infra/bots/flavor/android_flavor.py delete mode 100644 infra/bots/flavor/chromeos_flavor.py delete mode 100644 infra/bots/flavor/cmake_flavor.py delete mode 100644 infra/bots/flavor/coverage_flavor.py delete mode 100644 infra/bots/flavor/default_flavor.py delete mode 100644 infra/bots/flavor/ios_flavor.py delete mode 100644 infra/bots/flavor/ssh_devices.py delete mode 100644 infra/bots/flavor/ssh_flavor.py delete mode 100644 infra/bots/flavor/valgrind_flavor.py delete mode 100644 infra/bots/flavor/xsan_flavor.py delete mode 100644 infra/bots/perf_skia.py delete mode 100644 infra/bots/test_skia.py (limited to 'infra') diff --git a/infra/bots/add_isolated_input.py b/infra/bots/add_isolated_input.py deleted file mode 100644 index 70bde6d7ac..0000000000 --- a/infra/bots/add_isolated_input.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Google Inc. -# -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - - -import argparse -import json - - -"""Add the given hash to the includes section of the given isolated file.""" - - -def add_isolated_hash(isolated_file, hash_str): - with open(isolated_file) as f: - isolated = json.load(f) - isolated['includes'].append(hash_str) - with open(isolated_file, 'w') as f: - json.dump(isolated, f, sort_keys=True) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('--isolated_file', required=True) - parser.add_argument('--hash', required=True) - args = parser.parse_args() - add_isolated_hash(args.isolated_file, args.hash) - - -if __name__ == '__main__': - main() diff --git a/infra/bots/common.py b/infra/bots/common.py index 8d91f743bd..e4314dd78f 100644 --- a/infra/bots/common.py +++ b/infra/bots/common.py @@ -6,91 +6,20 @@ # found in the LICENSE file. -import contextlib -import glob -import math import os -import psutil import shutil -import socket import subprocess -import sys -import time -import urllib2 -from flavor import android_flavor -from flavor import chromeos_flavor -from flavor import cmake_flavor -from flavor import coverage_flavor -from flavor import default_flavor -from flavor import ios_flavor -from flavor import valgrind_flavor -from flavor import xsan_flavor - - -CONFIG_COVERAGE = 'Coverage' -CONFIG_DEBUG = 'Debug' -CONFIG_RELEASE = 'Release' -VALID_CONFIGS = (CONFIG_COVERAGE, CONFIG_DEBUG, CONFIG_RELEASE) - -BUILD_PRODUCTS_WHITELIST = [ - 'dm', - 'dm.exe', - 'nanobench', - 'nanobench.exe', - '*.so', - '*.dll', -] - -GM_ACTUAL_FILENAME = 'actual-results.json' -GM_EXPECTATIONS_FILENAME = 'expected-results.json' -GM_IGNORE_TESTS_FILENAME = 'ignored-tests.txt' - -GOLD_UNINTERESTING_HASHES_URL = 'https://gold.skia.org/_/hashes' GS_GM_BUCKET = 'chromium-skia-gm' -GS_SUMMARIES_BUCKET = 'chromium-skia-gm-summaries' GS_SUBDIR_TMPL_SK_IMAGE = 'skimage/v%s' GS_SUBDIR_TMPL_SKP = 'playback_%s/skps' -SKIA_REPO = 'https://skia.googlesource.com/skia.git' -INFRA_REPO = 'https://skia.googlesource.com/buildbot.git' - -SERVICE_ACCOUNT_FILE = 'service-account-skia.json' -SERVICE_ACCOUNT_INTERNAL_FILE = 'service-account-skia-internal.json' - VERSION_FILE_SK_IMAGE = 'SK_IMAGE_VERSION' VERSION_FILE_SKP = 'SKP_VERSION' -def is_android(bot_cfg): - """Determine whether the given bot is an Android bot.""" - return ('Android' in bot_cfg.get('extra_config', '') or - bot_cfg.get('os') == 'Android') - -def is_chromeos(bot_cfg): - return ('CrOS' in bot_cfg.get('extra_config', '') or - bot_cfg.get('os') == 'ChromeOS') - -def is_cmake(bot_cfg): - return 'CMake' in bot_cfg.get('extra_config', '') - -def is_ios(bot_cfg): - return ('iOS' in bot_cfg.get('extra_config', '') or - bot_cfg.get('os') == 'iOS') - - -def is_valgrind(bot_cfg): - return 'Valgrind' in bot_cfg.get('extra_config', '') - - -def is_xsan(bot_cfg): - return (bot_cfg.get('extra_config') == 'ASAN' or - bot_cfg.get('extra_config') == 'MSAN' or - bot_cfg.get('extra_config') == 'TSAN') - - def download_dir(skia_dir, tmp_dir, version_file, gs_path_tmpl, dst_dir): # Ensure that the tmp_dir exists. if not os.path.isdir(tmp_dir): @@ -124,368 +53,3 @@ def download_dir(skia_dir, tmp_dir, version_file, gs_path_tmpl, dst_dir): subprocess.check_call(['gsutil', 'cp', '-R', gs_path, dst_dir]) with open(actual_version_file, 'w') as f: f.write(expected_version) - - -def get_uninteresting_hashes(hashes_file): - retries = 5 - timeout = 60 - wait_base = 15 - - socket.setdefaulttimeout(timeout) - for retry in range(retries): - try: - with contextlib.closing( - urllib2.urlopen(GOLD_UNINTERESTING_HASHES_URL, timeout=timeout)) as w: - hashes = w.read() - with open(hashes_file, 'w') as f: - f.write(hashes) - break - except Exception as e: - print >> sys.stderr, 'Failed to get uninteresting hashes from %s:\n%s' % ( - GOLD_UNINTERESTING_HASHES_URL, e) - if retry == retries: - raise - waittime = wait_base * math.pow(2, retry) - print 'Retry in %d seconds.' % waittime - time.sleep(waittime) - - -class BotInfo(object): - def __init__(self, bot_name, swarm_out_dir): - """Initialize the bot, given its name. - - Assumes that CWD is the directory containing this file. - """ - self.name = bot_name - self.skia_dir = os.path.abspath(os.path.join( - os.path.dirname(os.path.realpath(__file__)), - os.pardir, os.pardir)) - self.swarm_out_dir = swarm_out_dir - os.chdir(self.skia_dir) - self.build_dir = os.path.abspath(os.path.join(self.skia_dir, os.pardir)) - self.infrabots_dir = os.path.join(self.skia_dir, 'infra', 'bots') - self.home_dir = os.path.expanduser('~') - - self.spec = self.get_bot_spec(bot_name) - self.bot_cfg = self.spec['builder_cfg'] - self.out_dir = os.path.join(os.pardir, 'out') - self.configuration = self.spec['configuration'] - self.default_env = { - 'CHROME_HEADLESS': '1', - 'SKIA_OUT': self.out_dir, - 'BUILDTYPE': self.configuration, - 'PATH': os.environ['PATH'], - } - if 'Win' in self.bot_cfg['os']: - self.default_env['SystemRoot'] = 'C:\\Windows' - self.default_env['TEMP'] = os.path.join( - self.home_dir, 'AppData', 'Local', 'Temp') - self.default_env['TMP'] = self.default_env['TEMP'] - self.default_env.update(self.spec['env']) - self.build_targets = [str(t) for t in self.spec['build_targets']] - self.is_trybot = self.bot_cfg['is_trybot'] - self.upload_dm_results = self.spec['upload_dm_results'] - self.upload_perf_results = self.spec['upload_perf_results'] - self.perf_data_dir = os.path.join(self.swarm_out_dir, 'perfdata', - self.name, 'data') - self.resource_dir = os.path.join(self.skia_dir, 'resources') - self.images_dir = os.path.join(self.build_dir, 'images') - self.local_skp_dir = os.path.join(self.build_dir, 'playback', 'skps') - self.dm_flags = self.spec['dm_flags'] - self.nanobench_flags = self.spec['nanobench_flags'] - self._ccache = None - self._checked_for_ccache = False - self._already_ran = {} - self.tmp_dir = os.path.join(self.build_dir, 'tmp') - self.flavor = self.get_flavor(self.bot_cfg) - - # These get filled in during subsequent steps. - self.device_dirs = None - self.build_number = None - self.got_revision = None - self.master_name = None - self.slave_name = None - - @property - def ccache(self): - if not self._checked_for_ccache: - self._checked_for_ccache = True - if sys.platform != 'win32': - try: - result = subprocess.check_output(['which', 'ccache']) - self._ccache = result.rstrip() - except subprocess.CalledProcessError: - pass - - return self._ccache - - def get_bot_spec(self, bot_name): - """Retrieve the bot spec for this bot.""" - sys.path.append(self.skia_dir) - from tools import buildbot_spec - return buildbot_spec.get_builder_spec(bot_name) - - def get_flavor(self, bot_cfg): - """Return a flavor utils object specific to the given bot.""" - if is_android(bot_cfg): - return android_flavor.AndroidFlavorUtils(self) - elif is_chromeos(bot_cfg): - return chromeos_flavor.ChromeOSFlavorUtils(self) - elif is_cmake(bot_cfg): - return cmake_flavor.CMakeFlavorUtils(self) - elif is_ios(bot_cfg): - return ios_flavor.iOSFlavorUtils(self) - elif is_valgrind(bot_cfg): - return valgrind_flavor.ValgrindFlavorUtils(self) - elif is_xsan(bot_cfg): - return xsan_flavor.XSanFlavorUtils(self) - elif bot_cfg.get('configuration') == CONFIG_COVERAGE: - return coverage_flavor.CoverageFlavorUtils(self) - else: - return default_flavor.DefaultFlavorUtils(self) - - def run(self, cmd, env=None, cwd=None): - _env = {} - _env.update(self.default_env) - _env.update(env or {}) - cwd = cwd or self.skia_dir - print '============' - print 'CMD: %s' % cmd - print 'CWD: %s' % cwd - print 'ENV: %s' % _env - print '============' - subprocess.check_call(cmd, env=_env, cwd=cwd) - - def compile_steps(self): - for t in self.build_targets: - self.flavor.compile(t) - dst = os.path.join(self.swarm_out_dir, 'out', self.configuration) - os.makedirs(dst) - for pattern in BUILD_PRODUCTS_WHITELIST: - path = os.path.join(self.out_dir, self.configuration, pattern) - for f in glob.glob(path): - print 'Copying build product %s' % f - shutil.copy(f, dst) - self.cleanup() - - def _run_once(self, fn, *args, **kwargs): - if not fn.__name__ in self._already_ran: - self._already_ran[fn.__name__] = True - fn(*args, **kwargs) - - def install(self): - """Copy the required executables and files to the device.""" - self.device_dirs = self.flavor.get_device_dirs() - - # Run any device-specific installation. - self.flavor.install() - - # TODO(borenet): Only copy files which have changed. - # Resources - self.flavor.copy_directory_contents_to_device(self.resource_dir, - self.device_dirs.resource_dir) - - def _key_params(self): - """Build a unique key from the builder name (as a list). - - E.g. arch x86 gpu GeForce320M mode MacMini4.1 os Mac10.6 - """ - # Don't bother to include role, which is always Test. - # TryBots are uploaded elsewhere so they can use the same key. - blacklist = ['role', 'is_trybot'] - - flat = [] - for k in sorted(self.bot_cfg.keys()): - if k not in blacklist: - flat.append(k) - flat.append(self.bot_cfg[k]) - return flat - - def test_steps(self, got_revision, master_name, slave_name, build_number, - issue=None, patchset=None): - """Run the DM test.""" - self.build_number = build_number - self.got_revision = got_revision - self.master_name = master_name - self.slave_name = slave_name - self._run_once(self.install) - - use_hash_file = False - if self.upload_dm_results: - # This must run before we write anything into self.device_dirs.dm_dir - # or we may end up deleting our output on machines where they're the same. - host_dm_dir = os.path.join(self.swarm_out_dir, 'dm') - print 'host dm dir: %s' % host_dm_dir - self.flavor.create_clean_host_dir(host_dm_dir) - if str(host_dm_dir) != str(self.device_dirs.dm_dir): - self.flavor.create_clean_device_dir(self.device_dirs.dm_dir) - - # Obtain the list of already-generated hashes. - if not os.path.isdir(self.tmp_dir): - os.makedirs(self.tmp_dir) - hash_filename = 'uninteresting_hashes.txt' - host_hashes_file = os.path.join(self.tmp_dir, hash_filename) - hashes_file = self.flavor.device_path_join( - self.device_dirs.tmp_dir, hash_filename) - - try: - get_uninteresting_hashes(host_hashes_file) - except Exception: - pass - - if os.path.exists(host_hashes_file): - self.flavor.copy_file_to_device(host_hashes_file, hashes_file) - use_hash_file = True - - # Run DM. - properties = [ - 'gitHash', self.got_revision, - 'master', self.master_name, - 'builder', self.name, - 'build_number', self.build_number, - ] - if self.is_trybot: - if not issue: - raise Exception('issue is required for trybots.') - if not patchset: - raise Exception('patchset is required for trybots.') - properties.extend([ - 'issue', issue, - 'patchset', patchset, - ]) - - args = [ - 'dm', - '--undefok', # This helps branches that may not know new flags. - '--resourcePath', self.device_dirs.resource_dir, - '--skps', self.device_dirs.skp_dir, - '--images', self.flavor.device_path_join( - self.device_dirs.images_dir, 'dm'), - '--nameByHash', - '--properties' - ] + properties - - args.append('--key') - args.extend(self._key_params()) - if use_hash_file: - args.extend(['--uninterestingHashesFile', hashes_file]) - if self.upload_dm_results: - args.extend(['--writePath', self.device_dirs.dm_dir]) - - skip_flag = None - if self.bot_cfg.get('cpu_or_gpu') == 'CPU': - skip_flag = '--nogpu' - elif self.bot_cfg.get('cpu_or_gpu') == 'GPU': - skip_flag = '--nocpu' - if skip_flag: - args.append(skip_flag) - args.extend(self.dm_flags) - - self.flavor.run(args, env=self.default_env) - - if self.upload_dm_results: - # Copy images and JSON to host machine if needed. - self.flavor.copy_directory_contents_to_host(self.device_dirs.dm_dir, - host_dm_dir) - - # See skia:2789. - if ('Valgrind' in self.name and - self.bot_cfg.get('cpu_or_gpu') == 'GPU'): - abandonGpuContext = list(args) - abandonGpuContext.append('--abandonGpuContext') - self.flavor.run(abandonGpuContext) - preAbandonGpuContext = list(args) - preAbandonGpuContext.append('--preAbandonGpuContext') - self.flavor.run(preAbandonGpuContext) - - self.cleanup() - - def perf_steps(self, got_revision, master_name, slave_name, build_number, - issue=None, patchset=None): - """Run Skia benchmarks.""" - self.build_number = build_number - self.got_revision = got_revision - self.master_name = master_name - self.slave_name = slave_name - self._run_once(self.install) - if self.upload_perf_results: - self.flavor.create_clean_device_dir(self.device_dirs.perf_data_dir) - - # Run nanobench. - properties = [ - '--properties', - 'gitHash', self.got_revision, - 'build_number', self.build_number, - ] - if self.is_trybot: - if not issue: - raise Exception('issue is required for trybots.') - if not patchset: - raise Exception('patchset is required for trybots.') - properties.extend([ - 'issue', issue, - 'patchset', patchset, - ]) - - target = 'nanobench' - if 'VisualBench' in self.name: - target = 'visualbench' - args = [ - target, - '--undefok', # This helps branches that may not know new flags. - '-i', self.device_dirs.resource_dir, - '--skps', self.device_dirs.skp_dir, - '--images', self.flavor.device_path_join( - self.device_dirs.images_dir, 'dm'), # Using DM images for now. - ] - - skip_flag = None - if self.bot_cfg.get('cpu_or_gpu') == 'CPU': - skip_flag = '--nogpu' - elif self.bot_cfg.get('cpu_or_gpu') == 'GPU': - skip_flag = '--nocpu' - if skip_flag: - args.append(skip_flag) - args.extend(self.nanobench_flags) - - if self.upload_perf_results: - json_path = self.flavor.device_path_join( - self.device_dirs.perf_data_dir, - 'nanobench_%s.json' % self.got_revision) - args.extend(['--outResultsFile', json_path]) - args.extend(properties) - - keys_blacklist = ['configuration', 'role', 'is_trybot'] - args.append('--key') - for k in sorted(self.bot_cfg.keys()): - if not k in keys_blacklist: - args.extend([k, self.bot_cfg[k]]) - - self.flavor.run(args, env=self.default_env) - - # See skia:2789. - if ('Valgrind' in self.name and - self.bot_cfg.get('cpu_or_gpu') == 'GPU'): - abandonGpuContext = list(args) - abandonGpuContext.extend(['--abandonGpuContext', '--nocpu']) - self.flavor.run(abandonGpuContext, env=self.default_env) - - # Copy results to host. - if self.upload_perf_results: - if not os.path.exists(self.perf_data_dir): - os.makedirs(self.perf_data_dir) - self.flavor.copy_directory_contents_to_host( - self.device_dirs.perf_data_dir, self.perf_data_dir) - - self.cleanup() - - def cleanup(self): - if sys.platform == 'win32': - # Kill mspdbsrv.exe, which tends to hang around after the build finishes. - for p in psutil.process_iter(): - try: - if p.name == 'mspdbsrv.exe': - p.kill() - except psutil._error.AccessDenied: - pass - self.flavor.cleanup_steps() diff --git a/infra/bots/compile_skia.isolate b/infra/bots/compile_skia.isolate index 529ca151b9..db0d579ccc 100644 --- a/infra/bots/compile_skia.isolate +++ b/infra/bots/compile_skia.isolate @@ -2,9 +2,4 @@ 'includes': [ 'skia_repo.isolate', ], - 'variables': { - 'command': [ - 'python', 'compile_skia.py', '--builder_name', '<(BUILDER_NAME)', '--swarm_out_dir', '${ISOLATED_OUTDIR}', - ], - }, } diff --git a/infra/bots/compile_skia.py b/infra/bots/compile_skia.py deleted file mode 100644 index 2887461351..0000000000 --- a/infra/bots/compile_skia.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Google Inc. -# -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - - -import argparse -import common -import os -import sys -import utils - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('--builder_name', required=True) - parser.add_argument('--swarm_out_dir', required=True) - args = parser.parse_args() - with utils.print_timings(): - bot = common.BotInfo(args.builder_name, os.path.abspath(args.swarm_out_dir)) - bot.compile_steps() - - -if __name__ == '__main__': - main() diff --git a/infra/bots/flavor/__init__.py b/infra/bots/flavor/__init__.py deleted file mode 100644 index 78953f5f51..0000000000 --- a/infra/bots/flavor/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Google Inc. -# -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. diff --git a/infra/bots/flavor/android_devices.py b/infra/bots/flavor/android_devices.py deleted file mode 100644 index 37ceabef3a..0000000000 --- a/infra/bots/flavor/android_devices.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Google Inc. -# -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - - -import collections -import json - - -DEFAULT_SDK_ROOT = '/home/chrome-bot/android-sdk-linux' -MAC_SDK_ROOT = '/Users/chrome-bot/adt-bundle-mac-x86_64-20140702/sdk' -MACMINI_SDK_ROOT = '/Users/chrome-bot/android-sdk-macosx' - -SlaveInfo = collections.namedtuple('SlaveInfo', - 'serial android_sdk_root has_root') - -SLAVE_INFO = { - 'skiabot-mac-10_8-compile-000': - SlaveInfo('noserial', MAC_SDK_ROOT, True), - 'skiabot-mac-10_8-compile-001': - SlaveInfo('noserial', MAC_SDK_ROOT, True), - 'skiabot-mac-10_8-compile-002': - SlaveInfo('noserial', MAC_SDK_ROOT, True), - 'skiabot-mac-10_8-compile-003': - SlaveInfo('noserial', MAC_SDK_ROOT, True), - 'skiabot-mac-10_8-compile-004': - SlaveInfo('noserial', MAC_SDK_ROOT, True), - 'skiabot-mac-10_8-compile-005': - SlaveInfo('noserial', MAC_SDK_ROOT, True), - 'skiabot-mac-10_8-compile-006': - SlaveInfo('noserial', MAC_SDK_ROOT, True), - 'skiabot-mac-10_8-compile-007': - SlaveInfo('noserial', MAC_SDK_ROOT, True), - 'skiabot-mac-10_8-compile-008': - SlaveInfo('noserial', MAC_SDK_ROOT, True), - 'skiabot-mac-10_8-compile-009': - SlaveInfo('noserial', MAC_SDK_ROOT, True), - 'skiabot-shuttle-ubuntu15-androidone-001': - SlaveInfo('AG86044202A04GC', DEFAULT_SDK_ROOT, True), - 'skiabot-shuttle-ubuntu15-androidone-002': - SlaveInfo('AG8404EC06G02GC', DEFAULT_SDK_ROOT, True), - 'skiabot-shuttle-ubuntu15-androidone-003': - SlaveInfo('AG8404EC0688EGC', DEFAULT_SDK_ROOT, True), - 'skiabot-shuttle-ubuntu12-galaxys3-001': - SlaveInfo('4df713b8244a21cf', DEFAULT_SDK_ROOT, False), - 'skiabot-shuttle-ubuntu12-galaxys3-002': - SlaveInfo('32309a56e9b3a09f', DEFAULT_SDK_ROOT, False), - 'skiabot-shuttle-ubuntu12-galaxys4-001': - SlaveInfo('4d0032a5d8cb6125', MACMINI_SDK_ROOT, False), - 'skiabot-shuttle-ubuntu12-galaxys4-002': - SlaveInfo('4d00353cd8ed61c3', MACMINI_SDK_ROOT, False), - 'skiabot-shuttle-ubuntu12-nexus5-001': - SlaveInfo('03f61449437cc47b', DEFAULT_SDK_ROOT, True), - 'skiabot-shuttle-ubuntu12-nexus5-002': - SlaveInfo('018dff3520c970f6', DEFAULT_SDK_ROOT, True), - 'skiabot-shuttle-ubuntu15-nexus6-001': - SlaveInfo('ZX1G22JJWS', DEFAULT_SDK_ROOT, True), - 'skiabot-shuttle-ubuntu15-nexus6-002': - SlaveInfo('ZX1G22JN35', DEFAULT_SDK_ROOT, True), - 'skiabot-shuttle-ubuntu15-nexus6-003': - SlaveInfo('ZX1G22JXXM', DEFAULT_SDK_ROOT, True), - 'skiabot-shuttle-ubuntu12-nexus7-001': - SlaveInfo('015d210a13480604', DEFAULT_SDK_ROOT, True), - 'skiabot-shuttle-ubuntu12-nexus7-002': - SlaveInfo('015d18848c280217', DEFAULT_SDK_ROOT, True), - 'skiabot-shuttle-ubuntu12-nexus7-003': - SlaveInfo('015d16897c401e17', DEFAULT_SDK_ROOT, True), - 'skiabot-shuttle-ubuntu12-nexus9-001': - SlaveInfo('HT43RJT00022', DEFAULT_SDK_ROOT, True), - 'skiabot-shuttle-ubuntu12-nexus9-002': - SlaveInfo('HT4AEJT03112', DEFAULT_SDK_ROOT, True), - 'skiabot-shuttle-ubuntu12-nexus9-003': - SlaveInfo('HT4ADJT03339', DEFAULT_SDK_ROOT, True), - 'skiabot-shuttle-ubuntu12-nexus10-001': - SlaveInfo('R32C801B5LH', DEFAULT_SDK_ROOT, True), - 'skiabot-shuttle-ubuntu12-nexus10-003': - SlaveInfo('R32CB017X2L', DEFAULT_SDK_ROOT, True), - 'skiabot-shuttle-ubuntu12-nexusplayer-001': - SlaveInfo('D76C708B', DEFAULT_SDK_ROOT, True), - 'skiabot-shuttle-ubuntu12-nexusplayer-002': - SlaveInfo('8AB5139A', DEFAULT_SDK_ROOT, True), - 'skiabot-shuttle-ubuntu15-nvidia-shield-001': - SlaveInfo('04217150066510000078', MACMINI_SDK_ROOT, False), - 'skiabot-linux-housekeeper-003': - SlaveInfo('noserial', DEFAULT_SDK_ROOT, False), - 'vm690-m3': SlaveInfo('noserial', MACMINI_SDK_ROOT, False), - 'vm691-m3': SlaveInfo('noserial', MACMINI_SDK_ROOT, False), - 'vm692-m3': SlaveInfo('noserial', MACMINI_SDK_ROOT, False), - 'vm693-m3': SlaveInfo('noserial', MACMINI_SDK_ROOT, False), - 'default': - SlaveInfo('noserial', DEFAULT_SDK_ROOT, False), -} - - -if __name__ == '__main__': - print json.dumps(SLAVE_INFO) # pragma: no cover - diff --git a/infra/bots/flavor/android_flavor.py b/infra/bots/flavor/android_flavor.py deleted file mode 100644 index 754e23ca30..0000000000 --- a/infra/bots/flavor/android_flavor.py +++ /dev/null @@ -1,247 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Google Inc. -# -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - - -import android_devices -import default_flavor -import os - - -"""Android flavor utils, used for building for and running tests on Android.""" - - -class _ADBWrapper(object): - """Wrapper for ADB.""" - def __init__(self, path_to_adb, serial, android_flavor): - self._adb = path_to_adb - self._serial = serial - self._wait_count = 0 - self._android_flavor = android_flavor - - def wait_for_device(self): - """Run 'adb wait-for-device'.""" - self._wait_count += 1 - cmd = [ - os.path.join(self._android_flavor.android_bin, 'adb_wait_for_device'), - '-s', self._serial, - ] - self._android_flavor._bot_info.run( - cmd, env=self._android_flavor._default_env) - cmd = [ - os.path.join(self._android_flavor.android_bin, 'adb_wait_for_charge'), - '-s', self._serial, - ] - self._android_flavor._bot_info.run( - cmd, env=self._android_flavor._default_env) - - def maybe_wait_for_device(self): - """Run 'adb wait-for-device' if it hasn't already been run.""" - if self._wait_count == 0: - self.wait_for_device() - - def __call__(self, *args, **kwargs): - self.maybe_wait_for_device() - return self._android_flavor._bot_info.run(self._adb + args, **kwargs) - - -class AndroidFlavorUtils(default_flavor.DefaultFlavorUtils): - def __init__(self, skia_api): - super(AndroidFlavorUtils, self).__init__(skia_api) - self.device = self._bot_info.spec['device_cfg'] - slave_info = android_devices.SLAVE_INFO.get( - self._bot_info.slave_name, - android_devices.SLAVE_INFO['default']) - self.serial = slave_info.serial - self.android_bin = os.path.join( - self._bot_info.skia_dir, 'platform_tools', 'android', 'bin') - self._android_sdk_root = slave_info.android_sdk_root - self._adb = _ADBWrapper( - os.path.join(self._android_sdk_root, 'platform-tools', 'adb'), - self.serial, - self) - self._has_root = slave_info.has_root - self._default_env = {'ANDROID_SDK_ROOT': self._android_sdk_root, - 'ANDROID_HOME': self._android_sdk_root, - 'SKIA_ANDROID_VERBOSE_SETUP': '1'} - - def step(self, name, cmd, env=None, **kwargs): - self._adb.maybe_wait_for_device() - args = [self.android_bin.join('android_run_skia'), - '--verbose', - '--logcat', - '-d', self.device, - '-s', self.serial, - '-t', self._bot_info.configuration, - ] - env = dict(env or {}) - env.update(self._default_env) - - return self._bot_info.run(self._bot_info.m.step, name=name, cmd=args + cmd, - env=env, **kwargs) - - def compile(self, target): - """Build the given target.""" - env = dict(self._default_env) - ccache = self._bot_info.ccache - if ccache: - env['ANDROID_MAKE_CCACHE'] = ccache - - cmd = [os.path.join(self.android_bin, 'android_ninja'), target, - '-d', self.device] - if 'Clang' in self._bot_info.name: - cmd.append('--clang') - self._bot_info.run(cmd, env=env) - - def device_path_join(self, *args): - """Like os.path.join(), but for paths on a connected Android device.""" - return '/'.join(args) - - def device_path_exists(self, path): - """Like os.path.exists(), but for paths on a connected device.""" - exists_str = 'FILE_EXISTS' - return exists_str in self._adb( - name='exists %s' % self._bot_info.m.path.basename(path), - serial=self.serial, - cmd=['shell', 'if', '[', '-e', path, '];', - 'then', 'echo', exists_str + ';', 'fi'], - stdout=self._bot_info.m.raw_io.output(), - infra_step=True - ).stdout - - def _remove_device_dir(self, path): - """Remove the directory on the device.""" - self._adb(name='rmdir %s' % self._bot_info.m.path.basename(path), - serial=self.serial, - cmd=['shell', 'rm', '-r', path], - infra_step=True) - # Sometimes the removal fails silently. Verify that it worked. - if self.device_path_exists(path): - raise Exception('Failed to remove %s!' % path) # pragma: no cover - - def _create_device_dir(self, path): - """Create the directory on the device.""" - self._adb(name='mkdir %s' % self._bot_info.m.path.basename(path), - serial=self.serial, - cmd=['shell', 'mkdir', '-p', path], - infra_step=True) - - def copy_directory_contents_to_device(self, host_dir, device_dir): - """Like shutil.copytree(), but for copying to a connected device.""" - self._bot_info.run( - self._bot_info.m.step, - name='push %s' % self._bot_info.m.path.basename(host_dir), - cmd=[self.android_bin.join('adb_push_if_needed'), '--verbose', - '-s', self.serial, host_dir, device_dir], - env=self._default_env, - infra_step=True) - - def copy_directory_contents_to_host(self, device_dir, host_dir): - """Like shutil.copytree(), but for copying from a connected device.""" - self._bot_info.run( - self._bot_info.m.step, - name='pull %s' % self._bot_info.m.path.basename(device_dir), - cmd=[self.android_bin.join('adb_pull_if_needed'), '--verbose', - '-s', self.serial, device_dir, host_dir], - env=self._default_env, - infra_step=True) - - def copy_file_to_device(self, host_path, device_path): - """Like shutil.copyfile, but for copying to a connected device.""" - self._adb(name='push %s' % self._bot_info.m.path.basename(host_path), - serial=self.serial, - cmd=['push', host_path, device_path], - infra_step=True) - - def create_clean_device_dir(self, path): - """Like shutil.rmtree() + os.makedirs(), but on a connected device.""" - self._remove_device_dir(path) - self._create_device_dir(path) - - def install(self): - """Run device-specific installation steps.""" - if self._has_root: - self._adb(name='adb root', - serial=self.serial, - cmd=['root'], - infra_step=True) - # Wait for the device to reconnect. - self._bot_info.run( - self._bot_info.m.step, - name='wait', - cmd=['sleep', '10'], - infra_step=True) - self._adb.wait_for_device() - - # TODO(borenet): Set CPU scaling mode to 'performance'. - self._bot_info.run(self._bot_info.m.step, - name='kill skia', - cmd=[self.android_bin.join('android_kill_skia'), - '--verbose', '-s', self.serial], - env=self._default_env, - infra_step=True) - if self._has_root: - self._adb(name='stop shell', - serial=self.serial, - cmd=['shell', 'stop'], - infra_step=True) - - # Print out battery stats. - self._adb(name='starting battery stats', - serial=self.serial, - cmd=['shell', 'dumpsys', 'batteryproperties'], - infra_step=True) - - def cleanup_steps(self): - """Run any device-specific cleanup steps.""" - self._adb(name='final battery stats', - serial=self.serial, - cmd=['shell', 'dumpsys', 'batteryproperties'], - infra_step=True) - self._adb(name='reboot', - serial=self.serial, - cmd=['reboot'], - infra_step=True) - self._bot_info.run( - self._bot_info.m.step, - name='wait for reboot', - cmd=['sleep', '10'], - infra_step=True) - self._adb.wait_for_device() - - def read_file_on_device(self, path, *args, **kwargs): - """Read the given file.""" - return self._adb(name='read %s' % self._bot_info.m.path.basename(path), - serial=self.serial, - cmd=['shell', 'cat', path], - stdout=self._bot_info.m.raw_io.output(), - infra_step=True).stdout.rstrip() - - def remove_file_on_device(self, path, *args, **kwargs): - """Delete the given file.""" - return self._adb(name='rm %s' % self._bot_info.m.path.basename(path), - serial=self.serial, - cmd=['shell', 'rm', '-f', path], - infra_step=True, - *args, - **kwargs) - - def get_device_dirs(self): - """ Set the directories which will be used by the build steps.""" - device_scratch_dir = self._adb( - name='get EXTERNAL_STORAGE dir', - serial=self.serial, - cmd=['shell', 'echo', '$EXTERNAL_STORAGE'], - ) - prefix = self.device_path_join(device_scratch_dir, 'skiabot', 'skia_') - return default_flavor.DeviceDirs( - dm_dir=prefix + 'dm', - perf_data_dir=prefix + 'perf', - resource_dir=prefix + 'resources', - images_dir=prefix + 'images', - skp_dir=prefix + 'skp/skps', - tmp_dir=prefix + 'tmp_dir') - diff --git a/infra/bots/flavor/chromeos_flavor.py b/infra/bots/flavor/chromeos_flavor.py deleted file mode 100644 index 8e8221489d..0000000000 --- a/infra/bots/flavor/chromeos_flavor.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Google Inc. -# -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - - -import default_flavor -import os -import ssh_flavor - - -"""Utils for building for and running tests on ChromeOS.""" - - -class ChromeOSFlavorUtils(ssh_flavor.SSHFlavorUtils): - def __init__(self, bot_info): - super(ChromeOSFlavorUtils, self).__init__(bot_info) - self.board = self._bot_info.spec['device_cfg'] - self.device_root_dir = '/usr/local/skiabot' - self.device_bin_dir = self.device_path_join(self.device_root_dir, 'bin') - - def step(self, name, cmd, **kwargs): - """Wrapper for the Step API; runs a step as appropriate for this flavor.""" - local_path = self._bot_info.out_dir.join( - 'config', 'chromeos-%s' % self.board, - self._bot_info.configuration, cmd[0]) - remote_path = self.device_path_join(self.device_bin_dir, cmd[0]) - self.copy_file_to_device(local_path, remote_path) - super(ChromeOSFlavorUtils, self).step(name=name, - cmd=[remote_path]+cmd[1:], - **kwargs) - - def compile(self, target): - """Build the given target.""" - cmd = [os.path.join(self._bot_info.skia_dir, 'platform_tools', 'chromeos', - 'bin', 'chromeos_make'), - '-d', self.board, - target] - self._bot_info.run(cmd) - - def install(self): - """Run any device-specific installation steps.""" - self.create_clean_device_dir(self.device_bin_dir) - - def get_device_dirs(self): - """ Set the directories which will be used by the build steps.""" - prefix = self.device_path_join(self.device_root_dir, 'skia_') - def join(suffix): - return ''.join((prefix, suffix)) - return default_flavor.DeviceDirs( - dm_dir=join('dm_out'), # 'dm' conflicts with the binary - perf_data_dir=join('perf'), - resource_dir=join('resources'), - images_dir=join('images'), - skp_dir=self.device_path_join(join('skp'), 'skps'), - tmp_dir=join('tmp_dir')) - diff --git a/infra/bots/flavor/cmake_flavor.py b/infra/bots/flavor/cmake_flavor.py deleted file mode 100644 index cfba2bbefd..0000000000 --- a/infra/bots/flavor/cmake_flavor.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Google Inc. -# -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - - -import default_flavor -import os - - -"""CMake flavor utils, used for building Skia with CMake.""" - - -class CMakeFlavorUtils(default_flavor.DefaultFlavorUtils): - def compile(self, target): - """Build Skia with CMake. Ignores `target`.""" - cmake_build = os.path.join(self._bot_info.skia_dir, 'cmake', 'cmake_build') - self._bot_info.run([cmake_build, target]) diff --git a/infra/bots/flavor/coverage_flavor.py b/infra/bots/flavor/coverage_flavor.py deleted file mode 100644 index 0a11d08ee8..0000000000 --- a/infra/bots/flavor/coverage_flavor.py +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Google Inc. -# -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - - -import default_flavor -import os -import subprocess -import time - - -"""Utils for running coverage tests.""" - - -class CoverageFlavorUtils(default_flavor.DefaultFlavorUtils): - def compile(self, target): - """Build the given target.""" - cmd = [os.path.join(self._bot_info.skia_dir, 'tools', - 'llvm_coverage_build'), - target] - self._bot_info.run(cmd) - - def step(self, cmd, **kwargs): - """Run the given step through coverage.""" - # Slice out the 'key' and 'properties' arguments to be reused. - key = [] - properties = [] - current = None - for i in xrange(0, len(cmd)): - if isinstance(cmd[i], basestring) and cmd[i] == '--key': - current = key - elif isinstance(cmd[i], basestring) and cmd[i] == '--properties': - current = properties - elif isinstance(cmd[i], basestring) and cmd[i].startswith('--'): - current = None - if current is not None: - current.append(cmd[i]) - - results_dir = self._bot_info.out_dir.join('coverage_results') - self.create_clean_host_dir(results_dir) - - # Run DM under coverage. - report_file_basename = '%s.cov' % self._bot_info.got_revision - report_file = os.path.join(results_dir, report_file_basename) - args = [ - 'python', - os.path.join(self._bot_info.skia_dir, 'tools', 'llvm_coverage_run.py'), - ] + cmd + ['--outResultsFile', report_file] - self._bot_info.run(args, **kwargs) - - # Generate nanobench-style JSON output from the coverage report. - git_timestamp = subprocess.check_output(['git', 'log', '-n1', - self._bot_info.got_revision, '--format=%%ci']).rstrip() - nanobench_json = results_dir.join('nanobench_%s_%s.json' % ( - self._bot_info.got_revision, git_timestamp)) - line_by_line_basename = ('coverage_by_line_%s_%s.json' % ( - self._bot_info.got_revision, git_timestamp)) - line_by_line = results_dir.join(line_by_line_basename) - args = [ - 'python', - os.path.join(self._bot_info.skia_dir, 'tools', - 'parse_llvm_coverage.py'), - '--report', report_file, '--nanobench', nanobench_json, - '--linebyline', line_by_line] - args.extend(key) - args.extend(properties) - self._bot_info.run(args) - - # Upload raw coverage data. - now = time.utcnow() - gs_json_path = '/'.join(( - str(now.year).zfill(4), str(now.month).zfill(2), - str(now.day).zfill(2), str(now.hour).zfill(2), - self._bot_info.name, - str(self._bot_info.build_number))) - if self._bot_info.is_trybot: - gs_json_path = '/'.join(('trybot', gs_json_path, - str(self._bot_info.issue))) - - self._bot_info.gsutil_upload( - 'upload raw coverage data', - source=report_file, - bucket='skia-infra', - dest='/'.join(('coverage-raw-v1', gs_json_path, report_file_basename))) - - # Upload nanobench JSON data. - gsutil_path = self._bot_info.m.path['depot_tools'].join( - 'third_party', 'gsutil', 'gsutil') - upload_args = [self._bot_info.name, - self._bot_info.m.properties['buildnumber'], - results_dir, - self._bot_info.got_revision, gsutil_path] - if self._bot_info.is_trybot: - upload_args.append(self._bot_info.m.properties['issue']) - self._bot_info.run( - self._bot_info.m.python, - 'upload nanobench coverage results', - script=self._bot_info.resource('upload_bench_results.py'), - args=upload_args, - cwd=self._bot_info.m.path['checkout'], - abort_on_failure=False, - infra_step=True) - - # Upload line-by-line coverage data. - self._bot_info.gsutil_upload( - 'upload line-by-line coverage data', - source=line_by_line, - bucket='skia-infra', - dest='/'.join(('coverage-json-v1', gs_json_path, - line_by_line_basename))) - diff --git a/infra/bots/flavor/default_flavor.py b/infra/bots/flavor/default_flavor.py deleted file mode 100644 index e59887301f..0000000000 --- a/infra/bots/flavor/default_flavor.py +++ /dev/null @@ -1,190 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Google Inc. -# -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - - -"""Default flavor utils class, used for desktop bots.""" - - -import os -import shutil -import sys - - -class DeviceDirs(object): - def __init__(self, - dm_dir, - perf_data_dir, - resource_dir, - images_dir, - skp_dir, - tmp_dir): - self._dm_dir = dm_dir - self._perf_data_dir = perf_data_dir - self._resource_dir = resource_dir - self._images_dir = images_dir - self._skp_dir = skp_dir - self._tmp_dir = tmp_dir - - @property - def dm_dir(self): - """Where DM writes.""" - return self._dm_dir - - @property - def perf_data_dir(self): - return self._perf_data_dir - - @property - def resource_dir(self): - return self._resource_dir - - @property - def images_dir(self): - return self._images_dir - - @property - def skp_dir(self): - return self._skp_dir - - @property - def tmp_dir(self): - return self._tmp_dir - - -class DefaultFlavorUtils(object): - """Utilities to be used by build steps. - - The methods in this class define how certain high-level functions should - work. Each build step flavor should correspond to a subclass of - DefaultFlavorUtils which may override any of these functions as appropriate - for that flavor. - - For example, the AndroidFlavorUtils will override the functions for - copying files between the host and Android device, as well as the - 'step' function, so that commands may be run through ADB. - """ - def __init__(self, bot_info, *args, **kwargs): - self._bot_info = bot_info - self.chrome_path = os.path.join(self._bot_info.build_dir, 'src') - - def run(self, cmd, **kwargs): - """Runs a step as appropriate for this flavor.""" - path_to_app = os.path.join(self._bot_info.out_dir, - self._bot_info.configuration, cmd[0]) - if ('linux' in sys.platform and - 'x86_64' in self._bot_info.name and - not 'TSAN' in self._bot_info.name): - new_cmd = ['catchsegv', path_to_app] - else: - if sys.platform == 'win32': - path_to_app += '.exe' - new_cmd = [path_to_app] - new_cmd.extend(cmd[1:]) - return self._bot_info.run(new_cmd, **kwargs) - - def bootstrap_win_toolchain(self): - """Run bootstrapping script for the Windows toolchain.""" - bootstrap_script = os.path.join(self._bot_info.infrabots_dir, - 'bootstrap_win_toolchain_json.py') - win_toolchain_json = os.path.join( - self._bot_info.build_dir, 'src', 'build', 'win_toolchain.json') - self._bot_info.run([ - 'python', bootstrap_script, - '--win_toolchain_json', win_toolchain_json, - '--depot_tools_parent_dir', self._bot_info.build_dir]) - - def compile(self, target): - """Build the given target.""" - # The CHROME_PATH environment variable is needed for bots that use - # toolchains downloaded by Chrome. - env = {} - if sys.platform == 'win32': - make_cmd = ['python', 'make.py'] - env['CHROME_PATH'] = self.chrome_path - self._bot_info._run_once(self.bootstrap_win_toolchain) - else: - # Ensure depot_tools is in PATH. - depot_tools = os.path.join('/b', 'depot_tools') - env['PATH'] = os.pathsep.join([depot_tools, os.environ['PATH']]) - make_cmd = ['make'] - cmd = make_cmd + [target] - self._bot_info.run(cmd, env=env) - - def device_path_join(self, *args): - """Like os.path.join(), but for paths on a connected device.""" - return os.path.join(*args) - - def device_path_exists(self, path): - """Like os.path.exists(), but for paths on a connected device.""" - return os.path.exists(path, infra_step=True) # pragma: no cover - - def copy_directory_contents_to_device(self, host_dir, device_dir): - """Like shutil.copytree(), but for copying to a connected device.""" - # For "normal" bots who don't have an attached device, we expect - # host_dir and device_dir to be the same. - if str(host_dir) != str(device_dir): - raise ValueError('For bots who do not have attached devices, copying ' - 'from host to device is undefined and only allowed if ' - 'host_path and device_path are the same (%s vs %s).' % ( - str(host_dir), str(device_dir))) # pragma: no cover - - def copy_directory_contents_to_host(self, device_dir, host_dir): - """Like shutil.copytree(), but for copying from a connected device.""" - # For "normal" bots who don't have an attached device, we expect - # host_dir and device_dir to be the same. - if str(host_dir) != str(device_dir): - raise ValueError('For bots who do not have attached devices, copying ' - 'from device to host is undefined and only allowed if ' - 'host_path and device_path are the same (%s vs %s).' % ( - str(host_dir), str(device_dir))) # pragma: no cover - - def copy_file_to_device(self, host_path, device_path): - """Like shutil.copyfile, but for copying to a connected device.""" - # For "normal" bots who don't have an attached device, we expect - # host_dir and device_dir to be the same. - if str(host_path) != str(device_path): # pragma: no cover - raise ValueError('For bots who do not have attached devices, copying ' - 'from host to device is undefined and only allowed if ' - 'host_path and device_path are the same (%s vs %s).' % ( - str(host_path), str(device_path))) - - def create_clean_device_dir(self, path): - """Like shutil.rmtree() + os.makedirs(), but on a connected device.""" - self.create_clean_host_dir(path) - - def create_clean_host_dir(self, path): - """Convenience function for creating a clean directory.""" - if os.path.exists(path): - shutil.rmtree(path) - os.makedirs(path) - - def install(self): - """Run device-specific installation steps.""" - pass - - def cleanup_steps(self): - """Run any device-specific cleanup steps.""" - pass - - def get_device_dirs(self): - """ Set the directories which will be used by the build steps. - - These refer to paths on the same device where the test executables will - run, for example, for Android bots these are paths on the Android device - itself. For desktop bots, these are just local paths. - """ - join = lambda p: os.path.join(self._bot_info.build_dir, p) - return DeviceDirs( - dm_dir=os.path.join(self._bot_info.swarm_out_dir, 'dm'), - perf_data_dir=self._bot_info.perf_data_dir, - resource_dir=self._bot_info.resource_dir, - images_dir=join('images'), - skp_dir=self._bot_info.local_skp_dir, - tmp_dir=join('tmp')) - - def __repr__(self): - return '<%s object>' % self.__class__.__name__ # pragma: no cover diff --git a/infra/bots/flavor/ios_flavor.py b/infra/bots/flavor/ios_flavor.py deleted file mode 100644 index c2d8737750..0000000000 --- a/infra/bots/flavor/ios_flavor.py +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Google Inc. -# -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - - -import default_flavor -import os -import subprocess - - -"""iOS flavor utils, used for building for and running tests on iOS.""" - - -class iOSFlavorUtils(default_flavor.DefaultFlavorUtils): - def __init__(self, bot_info): - super(iOSFlavorUtils, self).__init__(bot_info) - self.ios_bin = os.path.join(self._bot_info.skia_dir, 'platform_tools', - 'ios', 'bin') - - def step(self, cmd, **kwargs): - args = [os.path.join(self.ios_bin, 'ios_run_skia')] - - # Convert 'dm' and 'nanobench' from positional arguments - # to flags, which is what iOSShell expects to select which - # one is being run. - cmd = ["--" + c if c in ['dm', 'nanobench'] else c - for c in cmd] - return self._bot_info.run(args + cmd, **kwargs) - - def compile(self, target): - """Build the given target.""" - cmd = [os.path.join(self.ios_bin, 'ios_ninja')] - self._bot_info.run(cmd) - - def device_path_join(self, *args): - """Like os.path.join(), but for paths on a connected iOS device.""" - return '/'.join(args) - - def device_path_exists(self, path): - """Like os.path.exists(), but for paths on a connected device.""" - return self._bot_info.run( - [os.path.join(self.ios_bin, 'ios_path_exists'), path], - ) # pragma: no cover - - def _remove_device_dir(self, path): - """Remove the directory on the device.""" - return self._bot_info.run( - [os.path.join(self.ios_bin, 'ios_rm'), path], - ) - - def _create_device_dir(self, path): - """Create the directory on the device.""" - return self._bot_info.run( - [os.path.join(self.ios_bin, 'ios_mkdir'), path], - ) - - def copy_directory_contents_to_device(self, host_dir, device_dir): - """Like shutil.copytree(), but for copying to a connected device.""" - return self._bot_info.run([ - os.path.join(self.ios_bin, 'ios_push_if_needed'), - host_dir, device_dir - ]) - - def copy_directory_contents_to_host(self, device_dir, host_dir): - """Like shutil.copytree(), but for copying from a connected device.""" - return self._bot_info.run( - [os.path.join(self.ios_bin, 'ios_pull_if_needed'), - device_dir, host_dir], - ) - - def copy_file_to_device(self, host_path, device_path): - """Like shutil.copyfile, but for copying to a connected device.""" - self._bot_info.run( - [os.path.join(self.ios_bin, 'ios_push_file'), host_path, device_path], - ) # pragma: no cover - - def create_clean_device_dir(self, path): - """Like shutil.rmtree() + os.makedirs(), but on a connected device.""" - self._remove_device_dir(path) - self._create_device_dir(path) - - def install(self): - """Run device-specific installation steps.""" - self._bot_info.run([os.path.join(self.ios_bin, 'ios_install')]) - - def cleanup_steps(self): - """Run any device-specific cleanup steps.""" - self._bot_info.run([os.path.join(self.ios_bin, 'ios_restart')]) - self._bot_info.run(['sleep', '20']) - - def read_file_on_device(self, path): - """Read the given file.""" - return subprocess.check_output( - [os.path.join(self.ios_bin, 'ios_cat_file'), path]).rstrip() - - def remove_file_on_device(self, path): - """Remove the file on the device.""" - return self._bot_info.run( - [os.path.join(self.ios_bin, 'ios_rm'), path], - ) - - def get_device_dirs(self): - """ Set the directories which will be used by the build steps.""" - prefix = self.device_path_join('skiabot', 'skia_') - return default_flavor.DeviceDirs( - dm_dir=prefix + 'dm', - perf_data_dir=prefix + 'perf', - resource_dir=prefix + 'resources', - images_dir=prefix + 'images', - skp_dir=prefix + 'skp/skps', - tmp_dir=prefix + 'tmp_dir') diff --git a/infra/bots/flavor/ssh_devices.py b/infra/bots/flavor/ssh_devices.py deleted file mode 100644 index f113fdb8a7..0000000000 --- a/infra/bots/flavor/ssh_devices.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Google Inc. -# -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - - -import collections -import json - - -DEFAULT_PORT = '22' -DEFAULT_USER = 'chrome-bot' - - -SlaveInfo = collections.namedtuple('SlaveInfo', - 'ssh_user ssh_host ssh_port') - -SLAVE_INFO = { - 'skiabot-shuttle-ubuntu12-003': - SlaveInfo('root', '192.168.1.123', DEFAULT_PORT), - 'skiabot-shuttle-ubuntu12-004': - SlaveInfo('root', '192.168.1.134', DEFAULT_PORT), - 'default': - SlaveInfo('nouser', 'noip', 'noport'), -} - - -if __name__ == '__main__': - print json.dumps(SLAVE_INFO) # pragma: no cover - diff --git a/infra/bots/flavor/ssh_flavor.py b/infra/bots/flavor/ssh_flavor.py deleted file mode 100644 index 07c383f603..0000000000 --- a/infra/bots/flavor/ssh_flavor.py +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Google Inc. -# -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - - -import default_flavor -import os -import posixpath -import subprocess -import ssh_devices - - -"""Utils for running tests remotely over SSH.""" - - -class SSHFlavorUtils(default_flavor.DefaultFlavorUtils): - def __init__(self, *args, **kwargs): - super(SSHFlavorUtils, self).__init__(*args, **kwargs) - slave_info = ssh_devices.SLAVE_INFO.get(self._bot_info.slave_name, - ssh_devices.SLAVE_INFO['default']) - self._host = slave_info.ssh_host - self._port = slave_info.ssh_port - self._user = slave_info.ssh_user - - @property - def host(self): - return self._host - - @property - def port(self): - return self._port - - @property - def user(self): - return self._user - - def ssh(self, cmd, **kwargs): - """Run the given SSH command.""" - ssh_cmd = ['ssh'] - if self.port: - ssh_cmd.extend(['-p', self.port]) - dest = self.host - if self.user: - dest = self.user + '@' + dest - ssh_cmd.append(dest) - ssh_cmd.extend(cmd) - return self._bot_info.run(ssh_cmd, **kwargs) - - def step(self, *args, **kwargs): - """Run the given step over SSH.""" - self.ssh(*args, **kwargs) - - def device_path_join(self, *args): - """Like os.path.join(), but for paths on a remote machine.""" - return posixpath.join(*args) - - def device_path_exists(self, path): # pragma: no cover - """Like os.path.exists(), but for paths on a remote device.""" - try: - self.ssh(['test', '-e', path]) - return True - except subprocess.CalledProcessError: - return False - - def _remove_device_dir(self, path): - """Remove the directory on the device.""" - self.ssh(['rm', '-rf', path]) - - def _create_device_dir(self, path): - """Create the directory on the device.""" - self.ssh(['mkdir', '-p', path]) - - def create_clean_device_dir(self, path): - """Like shutil.rmtree() + os.makedirs(), but on a remote device.""" - self._remove_device_dir(path) - self._create_device_dir(path) - - def _make_scp_cmd(self, remote_path, recurse=True): - """Prepare an SCP command. - - Returns a partial SCP command and an adjusted remote path. - """ - cmd = ['scp'] - if recurse: - cmd.append('-r') - if self.port: - cmd.extend(['-P', self.port]) - adj_remote_path = self.host + ':' + remote_path - if self.user: - adj_remote_path = self.user + '@' + adj_remote_path - return cmd, adj_remote_path - - def copy_directory_contents_to_device(self, host_dir, device_dir): - """Like shutil.copytree(), but for copying to a remote device.""" - _, remote_path = self._make_scp_cmd(device_dir) - cmd = [os.path.join(self._bot_info.skia_dir, 'tools', - 'scp_dir_contents.sh'), - host_dir, remote_path] - self._bot_info.run(cmd) - - def copy_directory_contents_to_host(self, device_dir, host_dir): - """Like shutil.copytree(), but for copying from a remote device.""" - _, remote_path = self._make_scp_cmd(device_dir) - cmd = [os.path.join(self._bot_info.skia_dir, 'tools', - 'scp_dir_contents.sh'), - remote_path, host_dir] - self._bot_info.run(cmd) - - def copy_file_to_device(self, host_path, device_path): - """Like shutil.copyfile, but for copying to a connected device.""" - cmd, remote_path = self._make_scp_cmd(device_path, recurse=False) - cmd.extend([host_path, remote_path]) - self._bot_info.run(cmd) - - def read_file_on_device(self, path): - return self.ssh(['cat', path]).rstrip() - - def remove_file_on_device(self, path): - """Delete the given file.""" - return self.ssh(['rm', '-f', path]) diff --git a/infra/bots/flavor/valgrind_flavor.py b/infra/bots/flavor/valgrind_flavor.py deleted file mode 100644 index 129a7c08ed..0000000000 --- a/infra/bots/flavor/valgrind_flavor.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Google Inc. -# -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - - -import default_flavor -import os - - -"""Utils for running under Valgrind.""" - - -class ValgrindFlavorUtils(default_flavor.DefaultFlavorUtils): - def __init__(self, *args, **kwargs): - super(ValgrindFlavorUtils, self).__init__(*args, **kwargs) - self._suppressions_file = os.path.join(self._bot_info.skia_dir, - 'tools', 'valgrind.supp') - - def step(self, name, cmd, **kwargs): - new_cmd = ['valgrind', '--gen-suppressions=all', '--leak-check=full', - '--track-origins=yes', '--error-exitcode=1', '--num-callers=40', - '--suppressions=%s' % self._suppressions_file] - path_to_app = os.path.join(self._bot_info.out_dir, - self._bot_info.configuration, cmd[0]) - new_cmd.append(path_to_app) - new_cmd.extend(cmd[1:]) - return self._bot_info.run(new_cmd, **kwargs) - diff --git a/infra/bots/flavor/xsan_flavor.py b/infra/bots/flavor/xsan_flavor.py deleted file mode 100644 index 5807be0180..0000000000 --- a/infra/bots/flavor/xsan_flavor.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Google Inc. -# -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - - -"""Utils for running under *SAN""" - - -import default_flavor -import os - - -class XSanFlavorUtils(default_flavor.DefaultFlavorUtils): - def __init__(self, *args, **kwargs): - super(XSanFlavorUtils, self).__init__(*args, **kwargs) - self._sanitizer = { - # We'd love to just pass 'address,undefined' and get all the checks, but - # we're not anywhere close to being able to do that. Instead we start - # with a set of checks that we know pass or nearly pass. See here for - # more information: - # http://clang.llvm.org/docs/UsersManual.html#controlling-code-generation - 'ASAN': ('address,bool,function,integer-divide-by-zero,nonnull-attribute,' - 'null,object-size,return,returns-nonnull-attribute,shift,' - 'signed-integer-overflow,unreachable,vla-bound,vptr'), - # MSAN and TSAN can't run together with ASAN, so they're their own bots. - 'MSAN': 'memory', - 'TSAN': 'thread', - }[self._bot_info.bot_cfg['extra_config']] - - def compile(self, target): - cmd = [os.path.join(self._bot_info.skia_dir, 'tools', 'xsan_build'), - self._sanitizer, target] - self._bot_info.run(cmd) - - def step(self, cmd, env=None, **kwargs): - """Wrapper for the Step API; runs a step as appropriate for this flavor.""" - lsan_suppressions = self._bot_info.skia_dir.join('tools', 'lsan.supp') - tsan_suppressions = self._bot_info.skia_dir.join('tools', 'tsan.supp') - ubsan_suppressions = self._bot_info.skia_dir.join('tools', 'ubsan.supp') - env = dict(env or {}) - env['ASAN_OPTIONS'] = 'symbolize=1 detect_leaks=1' - env['LSAN_OPTIONS'] = ('symbolize=1 print_suppressions=1 suppressions=%s' % - lsan_suppressions) - env['TSAN_OPTIONS'] = 'suppressions=%s' % tsan_suppressions - env['UBSAN_OPTIONS'] = 'suppressions=%s' % ubsan_suppressions - - path_to_app = os.path.join(self._bot_info.out_dir, - self._bot_info.configuration, cmd[0]) - new_cmd = [path_to_app] - new_cmd.extend(cmd[1:]) - return self._bot_info.run(new_cmd, env=env, **kwargs) diff --git a/infra/bots/images.isolate b/infra/bots/images.isolate index b93d95d76a..573342b8e8 100644 --- a/infra/bots/images.isolate +++ b/infra/bots/images.isolate @@ -2,6 +2,7 @@ 'variables': { 'files': [ '../../../images/', + '../../../tmp/SK_IMAGE_VERSION', ], }, } diff --git a/infra/bots/perf_skia.isolate b/infra/bots/perf_skia.isolate index 75b73adca1..4235354b21 100644 --- a/infra/bots/perf_skia.isolate +++ b/infra/bots/perf_skia.isolate @@ -5,17 +5,4 @@ 'resources.isolate', 'skps.isolate', ], - 'variables': { - 'command': [ - 'python', 'perf_skia.py', - '--master_name', '<(MASTER_NAME)', - '--builder_name', '<(BUILDER_NAME)', - '--build_number', '<(BUILD_NUMBER)', - '--slave_name', '<(SLAVE_NAME)', - '--revision', '<(REVISION)', - '--swarm_out_dir', '${ISOLATED_OUTDIR}', - '--issue', '<(ISSUE)', - '--patchset', '<(PATCHSET)', - ], - }, } diff --git a/infra/bots/perf_skia.py b/infra/bots/perf_skia.py deleted file mode 100644 index ec3f99ca2e..0000000000 --- a/infra/bots/perf_skia.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Google Inc. -# -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - - -import argparse -import common -import os -import sys -import utils - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('--master_name', required=True) - parser.add_argument('--builder_name', required=True) - parser.add_argument('--build_number', required=True) - parser.add_argument('--slave_name', required=True) - parser.add_argument('--revision', required=True) - parser.add_argument('--swarm_out_dir', required=True) - parser.add_argument('--issue') - parser.add_argument('--patchset') - args = parser.parse_args() - with utils.print_timings(): - bot = common.BotInfo(args.builder_name, os.path.abspath(args.swarm_out_dir)) - bot.perf_steps(args.revision, args.master_name, args.slave_name, - args.build_number, issue=args.issue or None, - patchset=args.patchset or None) - - -if __name__ == '__main__': - main() diff --git a/infra/bots/skps.isolate b/infra/bots/skps.isolate index 15b2b1a5e8..84e986f4be 100644 --- a/infra/bots/skps.isolate +++ b/infra/bots/skps.isolate @@ -2,6 +2,7 @@ 'variables': { 'files': [ '../../../skps/', + '../../../tmp/SKP_VERSION', ], }, } diff --git a/infra/bots/test_skia.isolate b/infra/bots/test_skia.isolate index e3e4735a9d..4235354b21 100644 --- a/infra/bots/test_skia.isolate +++ b/infra/bots/test_skia.isolate @@ -5,17 +5,4 @@ 'resources.isolate', 'skps.isolate', ], - 'variables': { - 'command': [ - 'python', 'test_skia.py', - '--master_name', '<(MASTER_NAME)', - '--builder_name', '<(BUILDER_NAME)', - '--build_number', '<(BUILD_NUMBER)', - '--slave_name', '<(SLAVE_NAME)', - '--revision', '<(REVISION)', - '--swarm_out_dir', '${ISOLATED_OUTDIR}', - '--issue', '<(ISSUE)', - '--patchset', '<(PATCHSET)', - ], - }, } diff --git a/infra/bots/test_skia.py b/infra/bots/test_skia.py deleted file mode 100644 index 77121d6813..0000000000 --- a/infra/bots/test_skia.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Google Inc. -# -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - - -import argparse -import common -import os -import sys -import utils - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('--master_name', required=True) - parser.add_argument('--builder_name', required=True) - parser.add_argument('--build_number', required=True) - parser.add_argument('--slave_name', required=True) - parser.add_argument('--revision', required=True) - parser.add_argument('--swarm_out_dir', required=True) - parser.add_argument('--issue') - parser.add_argument('--patchset') - args = parser.parse_args() - with utils.print_timings(): - bot = common.BotInfo(args.builder_name, os.path.abspath(args.swarm_out_dir)) - bot.test_steps(args.revision, args.master_name, args.slave_name, - args.build_number, issue=args.issue or None, - patchset=args.patchset or None) - - -if __name__ == '__main__': - main() -- cgit v1.2.3