# # Copyright 2015 Google Inc. # # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. # #!/usr/bin/env python usage = ''' Write buildbot spec to outfile based on the bot name: $ python buildbot_spec.py outfile Test-Ubuntu-GCC-GCE-CPU-AVX2-x86-Debug Or run self-tests: $ python buildbot_spec.py test ''' import inspect import json import os import sys import builder_name_schema import dm_flags import nanobench_flags CONFIG_DEBUG = 'Debug' CONFIG_RELEASE = 'Release' def lineno(): caller = inspect.stack()[1] # Up one level to our caller. return inspect.getframeinfo(caller[0]).lineno # Since we don't actually start coverage until we're in the self-test, # some function def lines aren't reported as covered. Add them to this # list so that we can ignore them. cov_skip = [] cov_start = lineno()+1 # We care about coverage starting just past this def. def gyp_defines(builder_dict): gyp_defs = {} # skia_arch_type. if builder_dict['role'] == builder_name_schema.BUILDER_ROLE_BUILD: arch = builder_dict['target_arch'] elif builder_dict['role'] == builder_name_schema.BUILDER_ROLE_HOUSEKEEPER: arch = None else: arch = builder_dict['arch'] arch_types = { 'x86': 'x86', 'x86_64': 'x86_64', 'Arm7': 'arm', 'Arm64': 'arm64', 'Mips': 'mips32', 'Mips64': 'mips64', 'MipsDSP2': 'mips32', } if arch in arch_types: gyp_defs['skia_arch_type'] = arch_types[arch] # housekeeper: build shared lib. if builder_dict['role'] == builder_name_schema.BUILDER_ROLE_HOUSEKEEPER: gyp_defs['skia_shared_lib'] = '1' # skia_gpu. if builder_dict.get('cpu_or_gpu') == 'CPU': gyp_defs['skia_gpu'] = '0' # skia_warnings_as_errors. werr = False if builder_dict['role'] == builder_name_schema.BUILDER_ROLE_BUILD: if 'Win' in builder_dict.get('os', ''): if not ('GDI' in builder_dict.get('extra_config', '') or 'Exceptions' in builder_dict.get('extra_config', '')): werr = True elif ('Mac' in builder_dict.get('os', '') and 'Android' in builder_dict.get('extra_config', '')): werr = False else: werr = True gyp_defs['skia_warnings_as_errors'] = str(int(werr)) # True/False -> '1'/'0' # Win debugger. if 'Win' in builder_dict.get('os', ''): gyp_defs['skia_win_debuggers_path'] = 'c:/DbgHelp' # Qt SDK (Win). if 'Win' in builder_dict.get('os', ''): if builder_dict.get('os') == 'Win8': gyp_defs['qt_sdk'] = 'C:/Qt/Qt5.1.0/5.1.0/msvc2012_64/' else: gyp_defs['qt_sdk'] = 'C:/Qt/4.8.5/' # ANGLE. if builder_dict.get('extra_config') == 'ANGLE': gyp_defs['skia_angle'] = '1' if builder_dict.get('os', '') in ('Ubuntu', 'Linux'): gyp_defs['use_x11'] = '1' gyp_defs['chromeos'] = '0' # GDI. if builder_dict.get('extra_config') == 'GDI': gyp_defs['skia_gdi'] = '1' # Build with Exceptions on Windows. if ('Win' in builder_dict.get('os', '') and builder_dict.get('extra_config') == 'Exceptions'): gyp_defs['skia_win_exceptions'] = '1' # iOS. if (builder_dict.get('os') == 'iOS' or builder_dict.get('extra_config') == 'iOS'): gyp_defs['skia_os'] = 'ios' # Shared library build. if builder_dict.get('extra_config') == 'Shared': gyp_defs['skia_shared_lib'] = '1' # PDF viewer in GM. if (builder_dict.get('os') == 'Mac10.8' and builder_dict.get('arch') == 'x86_64' and builder_dict.get('configuration') == 'Release'): gyp_defs['skia_run_pdfviewer_in_gm'] = '1' # Clang. if builder_dict.get('compiler') == 'Clang': gyp_defs['skia_clang_build'] = '1' # Valgrind. if 'Valgrind' in builder_dict.get('extra_config', ''): gyp_defs['skia_release_optimization_level'] = '1' # Link-time code generation just wastes time on compile-only bots. if (builder_dict.get('role') == builder_name_schema.BUILDER_ROLE_BUILD and builder_dict.get('compiler') == 'MSVC'): gyp_defs['skia_win_ltcg'] = '0' # Mesa. if (builder_dict.get('extra_config') == 'Mesa' or builder_dict.get('cpu_or_gpu_value') == 'Mesa'): gyp_defs['skia_mesa'] = '1' # VisualBench if builder_dict.get('extra_config') == 'VisualBench': gyp_defs['skia_use_sdl'] = '1' # skia_use_android_framework_defines. if builder_dict.get('extra_config') == 'Android_FrameworkDefs': gyp_defs['skia_use_android_framework_defines'] = '1' # Skia dump stats for perf tests and gpu if (builder_dict.get('cpu_or_gpu') == 'GPU' and builder_dict.get('role') == 'Perf'): gyp_defs['skia_dump_stats'] = '1' return gyp_defs cov_skip.extend([lineno(), lineno() + 1]) def get_extra_env_vars(builder_dict): env = {} if builder_dict.get('configuration') == 'Coverage': # We have to use Clang 3.6 because earlier versions do not support the # compile flags we use and 3.7 and 3.8 hit asserts during compilation. env['CC'] = '/usr/bin/clang-3.6' env['CXX'] = '/usr/bin/clang++-3.6' elif builder_dict.get('compiler') == 'Clang': env['CC'] = '/usr/bin/clang' env['CXX'] = '/usr/bin/clang++' # SKNX_NO_SIMD, SK_USE_DISCARDABLE_SCALEDIMAGECACHE, etc. extra_config = builder_dict.get('extra_config', '') if extra_config.startswith('SK') and extra_config.isupper(): env['CPPFLAGS'] = '-D' + extra_config return env cov_skip.extend([lineno(), lineno() + 1]) def build_targets_from_builder_dict(builder_dict, do_test_steps, do_perf_steps): """Return a list of targets to build, depending on the builder type.""" if builder_dict['role'] in ('Test', 'Perf') and builder_dict['os'] == 'iOS': return ['iOSShell'] if builder_dict.get('extra_config') == 'Appurify': return ['VisualBenchTest_APK'] t = [] if do_test_steps: t.append('dm') if do_perf_steps and builder_dict.get('extra_config') == 'VisualBench': t.append('visualbench') elif do_perf_steps: t.append('nanobench') if t: return t else: return ['most'] cov_skip.extend([lineno(), lineno() + 1]) def device_cfg(builder_dict): # Android. if 'Android' in builder_dict.get('extra_config', ''): if 'NoNeon' in builder_dict['extra_config']: return 'arm_v7' return { 'Arm64': 'arm64', 'x86': 'x86', 'x86_64': 'x86_64', 'Mips': 'mips', 'Mips64': 'mips64', 'MipsDSP2': 'mips_dsp2', }.get(builder_dict['target_arch'], 'arm_v7_neon') elif builder_dict.get('os') == 'Android': return { 'AndroidOne': 'arm_v7_neon', 'GalaxyS3': 'arm_v7_neon', 'GalaxyS4': 'arm_v7_neon', 'NVIDIA_Shield': 'arm64', 'Nexus10': 'arm_v7_neon', 'Nexus5': 'arm_v7_neon', 'Nexus6': 'arm_v7_neon', 'Nexus7': 'arm_v7_neon', 'Nexus9': 'arm64', 'NexusPlayer': 'x86', }[builder_dict['model']] # ChromeOS. if 'CrOS' in builder_dict.get('extra_config', ''): if 'Link' in builder_dict['extra_config']: return 'link' if 'Daisy' in builder_dict['extra_config']: return 'daisy' elif builder_dict.get('os') == 'ChromeOS': return { 'Link': 'link', 'Daisy': 'daisy', }[builder_dict['model']] return None cov_skip.extend([lineno(), lineno() + 1]) def get_builder_spec(builder_name): builder_dict = builder_name_schema.DictForBuilderName(builder_name) env = get_extra_env_vars(builder_dict) gyp_defs = gyp_defines(builder_dict) gyp_defs_list = ['%s=%s' % (k, v) for k, v in gyp_defs.iteritems()] gyp_defs_list.sort() env['GYP_DEFINES'] = ' '.join(gyp_defs_list) rv = { 'builder_cfg': builder_dict, 'dm_flags': dm_flags.get_args(builder_name), 'env': env, 'nanobench_flags': nanobench_flags.get_args(builder_name), } device = device_cfg(builder_dict) if device: rv['device_cfg'] = device role = builder_dict['role'] if role == builder_name_schema.BUILDER_ROLE_HOUSEKEEPER: configuration = CONFIG_RELEASE else: configuration = builder_dict.get( 'configuration', CONFIG_DEBUG) arch = (builder_dict.get('arch') or builder_dict.get('target_arch')) if ('Win' in builder_dict.get('os', '') and arch == 'x86_64'): configuration += '_x64' rv['configuration'] = configuration rv['do_test_steps'] = role == builder_name_schema.BUILDER_ROLE_TEST rv['do_perf_steps'] = (role == builder_name_schema.BUILDER_ROLE_PERF or (role == builder_name_schema.BUILDER_ROLE_TEST and configuration == CONFIG_DEBUG)) if 'Valgrind' in builder_name: rv['do_perf_steps'] = True if 'GalaxyS4' in builder_name: rv['do_perf_steps'] = False rv['build_targets'] = build_targets_from_builder_dict( builder_dict, rv['do_test_steps'], rv['do_perf_steps']) # Do we upload perf results? upload_perf_results = False if role == builder_name_schema.BUILDER_ROLE_PERF: upload_perf_results = True rv['upload_perf_results'] = upload_perf_results # Do we upload correctness results? skip_upload_bots = [ 'ASAN', 'Coverage', 'MSAN', 'TSAN', 'UBSAN', 'Valgrind', ] upload_dm_results = True for s in skip_upload_bots: if s in builder_name: upload_dm_results = False break rv['upload_dm_results'] = upload_dm_results return rv cov_end = lineno() # Don't care about code coverage past here. def self_test(): import coverage # This way the bots don't need coverage.py to be installed. args = {} cases = [ 'Build-Mac10.8-Clang-Arm7-Debug-Android', 'Build-Win-MSVC-x86-Debug', 'Build-Win-MSVC-x86-Debug-GDI', 'Build-Win-MSVC-x86-Debug-Exceptions', 'Build-Ubuntu-GCC-Arm7-Debug-Android_FrameworkDefs', 'Build-Ubuntu-GCC-Arm7-Debug-Android_NoNeon', 'Build-Ubuntu-GCC-Arm7-Debug-CrOS_Daisy', 'Build-Ubuntu-GCC-x86_64-Debug-CrOS_Link', 'Build-Ubuntu-GCC-x86_64-Release-Mesa', 'Build-Ubuntu-GCC-x86_64-Release-ANGLE', 'Housekeeper-PerCommit', 'Perf-Win8-MSVC-ShuttleB-GPU-HD4600-x86_64-Release-Trybot', 'Perf-Ubuntu-GCC-ShuttleA-GPU-GTX660-x86_64-Release-VisualBench', 'Test-Android-GCC-GalaxyS4-GPU-SGX544-Arm7-Debug', 'Perf-Android-GCC-Nexus5-GPU-Adreno330-Arm7-Release-Appurify', 'Test-Android-GCC-Nexus6-GPU-Adreno420-Arm7-Debug', 'Test-ChromeOS-GCC-Link-CPU-AVX-x86_64-Debug', 'Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Debug', 'Test-Mac10.8-Clang-MacMini4.1-GPU-GeForce320M-x86_64-Release', 'Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Coverage', ('Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-' 'SK_USE_DISCARDABLE_SCALEDIMAGECACHE'), 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-SKNX_NO_SIMD', 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared', 'Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind', 'Test-Win8-MSVC-ShuttleB-GPU-HD4600-x86-Release-ANGLE', 'Test-Win8-MSVC-ShuttleA-CPU-AVX-x86_64-Debug', ] cov = coverage.coverage() cov.start() for case in cases: args[case] = get_builder_spec(case) cov.stop() this_file = os.path.basename(__file__) _, _, not_run, _ = cov.analysis(this_file) filtered = [line for line in not_run if line > cov_start and line < cov_end and line not in cov_skip] if filtered: print 'Lines not covered by test cases: ', filtered sys.exit(1) golden = this_file.replace('.py', '.json') with open(os.path.join(os.path.dirname(__file__), golden), 'w') as f: json.dump(args, f, indent=2, sort_keys=True) if __name__ == '__main__': if len(sys.argv) == 2 and sys.argv[1] == 'test': self_test() sys.exit(0) if len(sys.argv) != 3: print usage sys.exit(1) with open(sys.argv[1], 'w') as out: json.dump(get_builder_spec(sys.argv[2]), out)