#!/usr/bin/env python # # Copyright 2016 Google Inc. # # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import contextlib import math import os import shutil import socket import subprocess import sys import time import urllib2 from flavor import android_flavor from flavor import chromeos_flavor from flavor import cmake_flavor from flavor import coverage_flavor from flavor import default_flavor from flavor import ios_flavor from flavor import valgrind_flavor from flavor import xsan_flavor CONFIG_COVERAGE = 'Coverage' CONFIG_DEBUG = 'Debug' CONFIG_RELEASE = 'Release' VALID_CONFIGS = (CONFIG_COVERAGE, CONFIG_DEBUG, CONFIG_RELEASE) BUILD_PRODUCTS_WHITELIST = [ 'dm', 'dm.exe', 'nanobench', 'nanobench.exe', ] GM_ACTUAL_FILENAME = 'actual-results.json' GM_EXPECTATIONS_FILENAME = 'expected-results.json' GM_IGNORE_TESTS_FILENAME = 'ignored-tests.txt' GOLD_UNINTERESTING_HASHES_URL = 'https://gold.skia.org/_/hashes' GS_GM_BUCKET = 'chromium-skia-gm' GS_SUMMARIES_BUCKET = 'chromium-skia-gm-summaries' GS_SUBDIR_TMPL_SK_IMAGE = 'skimage/v%s' GS_SUBDIR_TMPL_SKP = 'playback_%s/skps' SKIA_REPO = 'https://skia.googlesource.com/skia.git' INFRA_REPO = 'https://skia.googlesource.com/buildbot.git' SERVICE_ACCOUNT_FILE = 'service-account-skia.json' SERVICE_ACCOUNT_INTERNAL_FILE = 'service-account-skia-internal.json' VERSION_FILE_SK_IMAGE = 'SK_IMAGE_VERSION' VERSION_FILE_SKP = 'SKP_VERSION' def is_android(bot_cfg): """Determine whether the given bot is an Android bot.""" return ('Android' in bot_cfg.get('extra_config', '') or bot_cfg.get('os') == 'Android') def is_chromeos(bot_cfg): return ('CrOS' in bot_cfg.get('extra_config', '') or bot_cfg.get('os') == 'ChromeOS') def is_cmake(bot_cfg): return 'CMake' in bot_cfg.get('extra_config', '') def is_ios(bot_cfg): return ('iOS' in bot_cfg.get('extra_config', '') or bot_cfg.get('os') == 'iOS') def is_valgrind(bot_cfg): return 'Valgrind' in bot_cfg.get('extra_config', '') def is_xsan(bot_cfg): return (bot_cfg.get('extra_config') == 'ASAN' or bot_cfg.get('extra_config') == 'MSAN' or bot_cfg.get('extra_config') == 'TSAN') def download_dir(skia_dir, tmp_dir, version_file, gs_path_tmpl, dst_dir): # Ensure that the tmp_dir exists. if not os.path.isdir(tmp_dir): os.makedirs(tmp_dir) # Get the expected version. with open(os.path.join(skia_dir, version_file)) as f: expected_version = f.read().rstrip() print 'Expected %s = %s' % (version_file, expected_version) # Get the actually-downloaded version, if we have one. actual_version_file = os.path.join(tmp_dir, version_file) try: with open(actual_version_file) as f: actual_version = f.read().rstrip() except IOError: actual_version = -1 print 'Actual %s = %s' % (version_file, actual_version) # If we don't have the desired version, download it. if actual_version != expected_version: if actual_version != -1: os.remove(actual_version_file) if os.path.isdir(dst_dir): shutil.rmtree(dst_dir) os.makedirs(dst_dir) gs_path = 'gs://%s/%s/*' % (GS_GM_BUCKET, gs_path_tmpl % expected_version) print 'Downloading from %s' % gs_path subprocess.check_call(['gsutil', 'cp', '-R', gs_path, dst_dir]) with open(actual_version_file, 'w') as f: f.write(expected_version) def get_uninteresting_hashes(hashes_file): retries = 5 timeout = 60 wait_base = 15 socket.setdefaulttimeout(timeout) for retry in range(retries): try: with contextlib.closing( urllib2.urlopen(GOLD_UNINTERESTING_HASHES_URL, timeout=timeout)) as w: hashes = w.read() with open(hashes_file, 'w') as f: f.write(hashes) break except Exception as e: print >> sys.stderr, 'Failed to get uninteresting hashes from %s:\n%s' % ( GOLD_UNINTERESTING_HASHES_URL, e) if retry == retries: raise waittime = wait_base * math.pow(2, retry) print 'Retry in %d seconds.' % waittime time.sleep(waittime) class BotInfo(object): def __init__(self, bot_name, swarm_out_dir): """Initialize the bot, given its name. Assumes that CWD is the directory containing this file. """ self.name = bot_name self.skia_dir = os.path.abspath(os.path.join( os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir)) self.swarm_out_dir = swarm_out_dir os.chdir(self.skia_dir) self.build_dir = os.path.abspath(os.path.join(self.skia_dir, os.pardir)) self.spec = self.get_bot_spec(bot_name) self.bot_cfg = self.spec['builder_cfg'] self.out_dir = os.path.join(os.pardir, 'out') self.configuration = self.spec['configuration'] self.default_env = { 'SKIA_OUT': self.out_dir, 'BUILDTYPE': self.configuration, 'PATH': os.environ['PATH'], } self.default_env.update(self.spec['env']) self.build_targets = [str(t) for t in self.spec['build_targets']] self.is_trybot = self.bot_cfg['is_trybot'] self.upload_dm_results = self.spec['upload_dm_results'] self.upload_perf_results = self.spec['upload_perf_results'] self.perf_data_dir = os.path.join(self.swarm_out_dir, 'perfdata', self.name, 'data') self.resource_dir = os.path.join(self.skia_dir, 'resources') self.images_dir = os.path.join(self.build_dir, 'images') self.local_skp_dir = os.path.join(self.build_dir, 'playback', 'skps') self.dm_flags = self.spec['dm_flags'] self.nanobench_flags = self.spec['nanobench_flags'] self._ccache = None self._checked_for_ccache = False self._already_ran = {} self.tmp_dir = os.path.join(self.build_dir, 'tmp') self.flavor = self.get_flavor(self.bot_cfg) # These get filled in during subsequent steps. self.device_dirs = None self.build_number = None self.got_revision = None self.master_name = None self.slave_name = None @property def ccache(self): if not self._checked_for_ccache: self._checked_for_ccache = True if sys.platform != 'win32': try: result = subprocess.check_output(['which', 'ccache']) self._ccache = result.rstrip() except subprocess.CalledProcessError: pass return self._ccache def get_bot_spec(self, bot_name): """Retrieve the bot spec for this bot.""" sys.path.append(self.skia_dir) from tools import buildbot_spec return buildbot_spec.get_builder_spec(bot_name) def get_flavor(self, bot_cfg): """Return a flavor utils object specific to the given bot.""" if is_android(bot_cfg): return android_flavor.AndroidFlavorUtils(self) elif is_chromeos(bot_cfg): return chromeos_flavor.ChromeOSFlavorUtils(self) elif is_cmake(bot_cfg): return cmake_flavor.CMakeFlavorUtils(self) elif is_ios(bot_cfg): return ios_flavor.iOSFlavorUtils(self) elif is_valgrind(bot_cfg): return valgrind_flavor.ValgrindFlavorUtils(self) elif is_xsan(bot_cfg): return xsan_flavor.XSanFlavorUtils(self) elif bot_cfg.get('configuration') == CONFIG_COVERAGE: return coverage_flavor.CoverageFlavorUtils(self) else: return default_flavor.DefaultFlavorUtils(self) def run(self, cmd, env=None, cwd=None): _env = {} _env.update(self.default_env) _env.update(env or {}) cwd = cwd or self.skia_dir print '============' print 'CMD: %s' % cmd print 'CWD: %s' % cwd print 'ENV: %s' % _env print '============' subprocess.check_call(cmd, env=_env, cwd=cwd) def compile_steps(self): for t in self.build_targets: self.flavor.compile(t) dst = os.path.join(self.swarm_out_dir, 'out', self.configuration) os.makedirs(dst) for f in BUILD_PRODUCTS_WHITELIST: path = os.path.join(self.out_dir, self.configuration, f) if os.path.exists(path): print 'Copying build product %s' % path shutil.copy(path, dst) def _run_once(self, fn, *args, **kwargs): if not fn.__name__ in self._already_ran: self._already_ran[fn.__name__] = True fn(*args, **kwargs) def install(self): """Copy the required executables and files to the device.""" self.device_dirs = self.flavor.get_device_dirs() # Run any device-specific installation. self.flavor.install() # TODO(borenet): Only copy files which have changed. # Resources self.flavor.copy_directory_contents_to_device(self.resource_dir, self.device_dirs.resource_dir) def _key_params(self): """Build a unique key from the builder name (as a list). E.g. arch x86 gpu GeForce320M mode MacMini4.1 os Mac10.6 """ # Don't bother to include role, which is always Test. # TryBots are uploaded elsewhere so they can use the same key. blacklist = ['role', 'is_trybot'] flat = [] for k in sorted(self.bot_cfg.keys()): if k not in blacklist: flat.append(k) flat.append(self.bot_cfg[k]) return flat def test_steps(self, got_revision, master_name, slave_name, build_number, issue=None, patchset=None): """Run the DM test.""" self.build_number = build_number self.got_revision = got_revision self.master_name = master_name self.slave_name = slave_name self._run_once(self.install) use_hash_file = False if self.upload_dm_results: # This must run before we write anything into self.device_dirs.dm_dir # or we may end up deleting our output on machines where they're the same. host_dm_dir = os.path.join(self.swarm_out_dir, 'dm') print 'host dm dir: %s' % host_dm_dir self.flavor.create_clean_host_dir(host_dm_dir) if str(host_dm_dir) != str(self.device_dirs.dm_dir): self.flavor.create_clean_device_dir(self.device_dirs.dm_dir) # Obtain the list of already-generated hashes. if not os.path.isdir(self.tmp_dir): os.makedirs(self.tmp_dir) hash_filename = 'uninteresting_hashes.txt' host_hashes_file = os.path.join(self.tmp_dir, hash_filename) hashes_file = self.flavor.device_path_join( self.device_dirs.tmp_dir, hash_filename) try: get_uninteresting_hashes(host_hashes_file) except Exception: pass if os.path.exists(host_hashes_file): self.flavor.copy_file_to_device(host_hashes_file, hashes_file) use_hash_file = True # Run DM. properties = [ 'gitHash', self.got_revision, 'master', self.master_name, 'builder', self.name, 'build_number', self.build_number, ] if self.is_trybot: if not issue: raise Exception('issue is required for trybots.') if not patchset: raise Exception('patchset is required for trybots.') properties.extend([ 'issue', issue, 'patchset', patchset, ]) args = [ 'dm', '--undefok', # This helps branches that may not know new flags. '--verbose', '--resourcePath', self.device_dirs.resource_dir, '--skps', self.device_dirs.skp_dir, '--images', self.flavor.device_path_join( self.device_dirs.images_dir, 'dm'), '--nameByHash', '--properties' ] + properties args.append('--key') args.extend(self._key_params()) if use_hash_file: args.extend(['--uninterestingHashesFile', hashes_file]) if self.upload_dm_results: args.extend(['--writePath', self.device_dirs.dm_dir]) skip_flag = None if self.bot_cfg.get('cpu_or_gpu') == 'CPU': skip_flag = '--nogpu' elif self.bot_cfg.get('cpu_or_gpu') == 'GPU': skip_flag = '--nocpu' if skip_flag: args.append(skip_flag) args.extend(self.dm_flags) self.flavor.run(args, env=self.default_env) if self.upload_dm_results: # Copy images and JSON to host machine if needed. self.flavor.copy_directory_contents_to_host(self.device_dirs.dm_dir, host_dm_dir) # See skia:2789. if ('Valgrind' in self.name and self.bot_cfg.get('cpu_or_gpu') == 'GPU'): abandonGpuContext = list(args) abandonGpuContext.append('--abandonGpuContext') self.flavor.run(abandonGpuContext) preAbandonGpuContext = list(args) preAbandonGpuContext.append('--preAbandonGpuContext') self.flavor.run(preAbandonGpuContext) self.flavor.cleanup_steps() def perf_steps(self, got_revision, master_name, slave_name, build_number, issue=None, patchset=None): """Run Skia benchmarks.""" self.build_number = build_number self.got_revision = got_revision self.master_name = master_name self.slave_name = slave_name self._run_once(self.install) if self.upload_perf_results: self.flavor.create_clean_device_dir(self.device_dirs.perf_data_dir) # Run nanobench. properties = [ '--properties', 'gitHash', self.got_revision, 'build_number', self.build_number, ] if self.is_trybot: if not issue: raise Exception('issue is required for trybots.') if not patchset: raise Exception('patchset is required for trybots.') properties.extend([ 'issue', issue, 'patchset', patchset, ]) target = 'nanobench' if 'VisualBench' in self.name: target = 'visualbench' args = [ target, '--undefok', # This helps branches that may not know new flags. '-i', self.device_dirs.resource_dir, '--skps', self.device_dirs.skp_dir, '--images', self.flavor.device_path_join( self.device_dirs.images_dir, 'dm'), # Using DM images for now. ] skip_flag = None if self.bot_cfg.get('cpu_or_gpu') == 'CPU': skip_flag = '--nogpu' elif self.bot_cfg.get('cpu_or_gpu') == 'GPU': skip_flag = '--nocpu' if skip_flag: args.append(skip_flag) args.extend(self.nanobench_flags) if self.upload_perf_results: json_path = self.flavor.device_path_join( self.device_dirs.perf_data_dir, 'nanobench_%s.json' % self.got_revision) args.extend(['--outResultsFile', json_path]) args.extend(properties) keys_blacklist = ['configuration', 'role', 'is_trybot'] args.append('--key') for k in sorted(self.bot_cfg.keys()): if not k in keys_blacklist: args.extend([k, self.bot_cfg[k]]) self.flavor.run(args, env=self.default_env) # See skia:2789. if ('Valgrind' in self.name and self.bot_cfg.get('cpu_or_gpu') == 'GPU'): abandonGpuContext = list(args) abandonGpuContext.extend(['--abandonGpuContext', '--nocpu']) self.flavor.run(abandonGpuContext, env=self.default_env) # Copy results to host. if self.upload_perf_results: if not os.path.exists(self.perf_data_dir): os.makedirs(self.perf_data_dir) self.flavor.copy_directory_contents_to_host( self.device_dirs.perf_data_dir, self.perf_data_dir) self.flavor.cleanup_steps()