diff options
author | borenet <borenet@google.com> | 2015-07-29 07:38:49 -0700 |
---|---|---|
committer | Commit bot <commit-bot@chromium.org> | 2015-07-29 07:38:49 -0700 |
commit | b76c1f706c40693740a8f7cc376bd603b417e8a3 (patch) | |
tree | e555f761afe5515cd098e3244b43ebc67c0304b2 | |
parent | f1f8bd58513848440d76e49cc3fde585bc888ff5 (diff) |
Add builder_spec.py
Works like dm_flags.py and nanobench_flags.py; adds things like
GYP_DEFINES, additional environment variables, and build targets.
Required copying builder_name_schema from the tools/build repo.
BUG=skia:4132
Review URL: https://codereview.chromium.org/1265623002
-rw-r--r-- | tools/buildbot_spec.json | 129 | ||||
-rwxr-xr-x | tools/buildbot_spec.py | 240 | ||||
-rw-r--r-- | tools/builder_name_schema.json | 40 | ||||
-rw-r--r-- | tools/builder_name_schema.py | 194 |
4 files changed, 603 insertions, 0 deletions
diff --git a/tools/buildbot_spec.json b/tools/buildbot_spec.json new file mode 100644 index 0000000000..b6fae8a64c --- /dev/null +++ b/tools/buildbot_spec.json @@ -0,0 +1,129 @@ +{ + "Build-Mac10.8-Clang-Arm7-Debug-Android": { + "build_targets": [ + "most" + ], + "env": { + "CC": "/usr/bin/clang", + "CXX": "/usr/bin/clang++", + "GYP_DEFINES": "skia_arch_type=arm skia_clang_build=1 skia_warnings_as_errors=0" + } + }, + "Build-Ubuntu-GCC-Arm7-Debug-Android_FrameworkDefs": { + "build_targets": [ + "most" + ], + "env": { + "GYP_DEFINES": "skia_arch_type=arm skia_use_android_framework_defines=1 skia_warnings_as_errors=1" + } + }, + "Build-Ubuntu-GCC-x86_64-Release-Mesa": { + "build_targets": [ + "most" + ], + "env": { + "GYP_DEFINES": "skia_arch_type=x86_64 skia_mesa=1 skia_warnings_as_errors=1" + } + }, + "Build-Win-MSVC-x86-Debug": { + "build_targets": [ + "most" + ], + "env": { + "GYP_DEFINES": "qt_sdk=C:/Qt/4.8.5/ skia_arch_type=x86 skia_warnings_as_errors=1 skia_win_debuggers_path=c:/DbgHelp skia_win_ltcg=0" + } + }, + "Build-Win-MSVC-x86-Debug-Exceptions": { + "build_targets": [ + "most" + ], + "env": { + "GYP_DEFINES": "qt_sdk=C:/Qt/4.8.5/ skia_arch_type=x86 skia_warnings_as_errors=0 skia_win_debuggers_path=c:/DbgHelp skia_win_exceptions=1 skia_win_ltcg=0" + } + }, + "Build-Win-MSVC-x86-Debug-GDI": { + "build_targets": [ + "most" + ], + "env": { + "GYP_DEFINES": "qt_sdk=C:/Qt/4.8.5/ skia_arch_type=x86 skia_gdi=1 skia_warnings_as_errors=0 skia_win_debuggers_path=c:/DbgHelp skia_win_ltcg=0" + } + }, + "Housekeeper-PerCommit": { + "build_targets": [ + "most" + ], + "env": { + "GYP_DEFINES": "skia_shared_lib=1 skia_warnings_as_errors=0" + } + }, + "Perf-Win8-MSVC-ShuttleB-GPU-HD4600-x86_64-Release-Trybot": { + "build_targets": [ + "nanobench" + ], + "env": { + "GYP_DEFINES": "qt_sdk=C:/Qt/Qt5.1.0/5.1.0/msvc2012_64/ skia_arch_type=x86_64 skia_warnings_as_errors=0 skia_win_debuggers_path=c:/DbgHelp" + } + }, + "Test-Mac10.8-Clang-MacMini4.1-GPU-GeForce320M-x86_64-Release": { + "build_targets": [ + "dm" + ], + "env": { + "CC": "/usr/bin/clang", + "CXX": "/usr/bin/clang++", + "GYP_DEFINES": "skia_arch_type=x86_64 skia_clang_build=1 skia_run_pdfviewer_in_gm=1 skia_warnings_as_errors=0" + } + }, + "Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-SKNX_NO_SIMD": { + "build_targets": [ + "dm" + ], + "env": { + "GYP_DEFINES": "skia_arch_type=x86_64 skia_gpu=0 skia_warnings_as_errors=0 sknx_no_simd=1" + } + }, + "Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared": { + "build_targets": [ + "dm" + ], + "env": { + "GYP_DEFINES": "skia_arch_type=x86_64 skia_gpu=0 skia_shared_lib=1 skia_warnings_as_errors=0" + } + }, + "Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind": { + "build_targets": [ + "dm" + ], + "env": { + "GYP_DEFINES": "skia_arch_type=x86_64 skia_release_optimization_level=1 skia_warnings_as_errors=0" + } + }, + "Test-Win8-MSVC-ShuttleA-CPU-AVX-x86_64-Debug": { + "build_targets": [ + "dm", + "nanobench" + ], + "env": { + "GYP_DEFINES": "qt_sdk=C:/Qt/Qt5.1.0/5.1.0/msvc2012_64/ skia_arch_type=x86_64 skia_gpu=0 skia_warnings_as_errors=0 skia_win_debuggers_path=c:/DbgHelp" + } + }, + "Test-Win8-MSVC-ShuttleB-GPU-HD4600-x86-Release-ANGLE": { + "build_targets": [ + "dm" + ], + "env": { + "GYP_DEFINES": "qt_sdk=C:/Qt/Qt5.1.0/5.1.0/msvc2012_64/ skia_angle=1 skia_arch_type=x86 skia_warnings_as_errors=0 skia_win_debuggers_path=c:/DbgHelp" + } + }, + "Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Debug": { + "build_targets": [ + "iOSShell" + ], + "env": { + "CC": "/usr/bin/clang", + "CXX": "/usr/bin/clang++", + "GYP_DEFINES": "skia_arch_type=arm skia_clang_build=1 skia_os=ios skia_warnings_as_errors=0" + } + } +}
\ No newline at end of file diff --git a/tools/buildbot_spec.py b/tools/buildbot_spec.py new file mode 100755 index 0000000000..92ff170ddd --- /dev/null +++ b/tools/buildbot_spec.py @@ -0,0 +1,240 @@ +# +# Copyright 2015 Google Inc. +# +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +# + +#!/usr/bin/env python + +usage = ''' +Write buildbot spec to outfile based on the bot name: + $ python buildbot_spec.py outfile Test-Ubuntu-GCC-GCE-CPU-AVX2-x86-Debug +Or run self-tests: + $ python buildbot_spec.py test +''' + +import inspect +import json +import os +import sys + +import builder_name_schema + + +def lineno(): + caller = inspect.stack()[1] # Up one level to our caller. + return inspect.getframeinfo(caller[0]).lineno + +# Since we don't actually start coverage until we're in the self-test, +# some function def lines aren't reported as covered. Add them to this +# list so that we can ignore them. +cov_skip = [] + +cov_start = lineno()+1 # We care about coverage starting just past this def. +def gyp_defines(builder_dict): + gyp_defs = {} + + # skia_arch_type. + if builder_dict['role'] == builder_name_schema.BUILDER_ROLE_BUILD: + arch = builder_dict['target_arch'] + elif builder_dict['role'] == builder_name_schema.BUILDER_ROLE_HOUSEKEEPER: + arch = None + else: + arch = builder_dict['arch'] + + arch_types = { + 'x86': 'x86', + 'x86_64': 'x86_64', + 'Arm7': 'arm', + 'Arm64': 'arm64', + 'Mips': 'mips32', + 'Mips64': 'mips64', + 'MipsDSP2': 'mips32', + } + if arch in arch_types: + gyp_defs['skia_arch_type'] = arch_types[arch] + + # housekeeper: build shared lib. + if builder_dict['role'] == builder_name_schema.BUILDER_ROLE_HOUSEKEEPER: + gyp_defs['skia_shared_lib'] = '1' + + # skia_gpu. + if builder_dict.get('cpu_or_gpu') == 'CPU': + gyp_defs['skia_gpu'] = '0' + + # skia_warnings_as_errors. + werr = False + if builder_dict['role'] == builder_name_schema.BUILDER_ROLE_BUILD: + if 'Win' in builder_dict.get('os', ''): + if not ('GDI' in builder_dict.get('extra_config', '') or + 'Exceptions' in builder_dict.get('extra_config', '')): + werr = True + elif ('Mac' in builder_dict.get('os', '') and + 'Android' in builder_dict.get('extra_config', '')): + werr = False + else: + werr = True + gyp_defs['skia_warnings_as_errors'] = str(int(werr)) # True/False -> '1'/'0' + + # Win debugger. + if 'Win' in builder_dict.get('os', ''): + gyp_defs['skia_win_debuggers_path'] = 'c:/DbgHelp' + + # Qt SDK (Win). + if 'Win' in builder_dict.get('os', ''): + if builder_dict.get('os') == 'Win8': + gyp_defs['qt_sdk'] = 'C:/Qt/Qt5.1.0/5.1.0/msvc2012_64/' + else: + gyp_defs['qt_sdk'] = 'C:/Qt/4.8.5/' + + # ANGLE. + if builder_dict.get('extra_config') == 'ANGLE': + gyp_defs['skia_angle'] = '1' + + # GDI. + if builder_dict.get('extra_config') == 'GDI': + gyp_defs['skia_gdi'] = '1' + + # Build with Exceptions on Windows. + if ('Win' in builder_dict.get('os', '') and + builder_dict.get('extra_config') == 'Exceptions'): + gyp_defs['skia_win_exceptions'] = '1' + + # iOS. + if (builder_dict.get('os') == 'iOS' or + builder_dict.get('extra_config') == 'iOS'): + gyp_defs['skia_os'] = 'ios' + + # Shared library build. + if builder_dict.get('extra_config') == 'Shared': + gyp_defs['skia_shared_lib'] = '1' + + # PDF viewer in GM. + if (builder_dict.get('os') == 'Mac10.8' and + builder_dict.get('arch') == 'x86_64' and + builder_dict.get('configuration') == 'Release'): + gyp_defs['skia_run_pdfviewer_in_gm'] = '1' + + # Clang. + if builder_dict.get('compiler') == 'Clang': + gyp_defs['skia_clang_build'] = '1' + + # Valgrind. + if 'Valgrind' in builder_dict.get('extra_config', ''): + gyp_defs['skia_release_optimization_level'] = '1' + + # Link-time code generation just wastes time on compile-only bots. + if (builder_dict.get('role') == builder_name_schema.BUILDER_ROLE_BUILD and + builder_dict.get('compiler') == 'MSVC'): + gyp_defs['skia_win_ltcg'] = '0' + + # Mesa. + if (builder_dict.get('extra_config') == 'Mesa' or + builder_dict.get('cpu_or_gpu_value') == 'Mesa'): + gyp_defs['skia_mesa'] = '1' + + # SKNX_NO_SIMD + if builder_dict.get('extra_config') == 'SKNX_NO_SIMD': + gyp_defs['sknx_no_simd'] = '1' + + # skia_use_android_framework_defines. + if builder_dict.get('extra_config') == 'Android_FrameworkDefs': + gyp_defs['skia_use_android_framework_defines'] = '1' + + return gyp_defs + + +cov_skip.extend([lineno(), lineno() + 1]) +def get_extra_env_vars(builder_dict): + env = {} + if builder_dict.get('compiler') == 'Clang': + env['CC'] = '/usr/bin/clang' + env['CXX'] = '/usr/bin/clang++' + return env + + +cov_skip.extend([lineno(), lineno() + 1]) +def build_targets_from_builder_dict(builder_dict): + """Return a list of targets to build, depending on the builder type.""" + if builder_dict['role'] in ('Test', 'Perf') and builder_dict['os'] == 'iOS': + return ['iOSShell'] + elif builder_dict['role'] == builder_name_schema.BUILDER_ROLE_TEST: + t = ['dm'] + if builder_dict.get('configuration') == 'Debug': + t.append('nanobench') + return t + elif builder_dict['role'] == builder_name_schema.BUILDER_ROLE_PERF: + return ['nanobench'] + else: + return ['most'] + + +cov_skip.extend([lineno(), lineno() + 1]) +def get_builder_spec(builder_name): + builder_dict = builder_name_schema.DictForBuilderName(builder_name) + env = get_extra_env_vars(builder_dict) + gyp_defs = gyp_defines(builder_dict) + gyp_defs_list = ['%s=%s' % (k, v) for k, v in gyp_defs.iteritems()] + gyp_defs_list.sort() + env['GYP_DEFINES'] = ' '.join(gyp_defs_list) + return { + 'build_targets': build_targets_from_builder_dict(builder_dict), + 'env': env, + } + + +cov_end = lineno() # Don't care about code coverage past here. + + +def self_test(): + import coverage # This way the bots don't need coverage.py to be installed. + args = {} + cases = [ + 'Build-Mac10.8-Clang-Arm7-Debug-Android', + 'Build-Win-MSVC-x86-Debug', + 'Build-Win-MSVC-x86-Debug-GDI', + 'Build-Win-MSVC-x86-Debug-Exceptions', + 'Build-Ubuntu-GCC-Arm7-Debug-Android_FrameworkDefs', + 'Build-Ubuntu-GCC-x86_64-Release-Mesa', + 'Housekeeper-PerCommit', + 'Perf-Win8-MSVC-ShuttleB-GPU-HD4600-x86_64-Release-Trybot', + 'Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Debug', + 'Test-Mac10.8-Clang-MacMini4.1-GPU-GeForce320M-x86_64-Release', + 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-SKNX_NO_SIMD', + 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared', + 'Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind', + 'Test-Win8-MSVC-ShuttleB-GPU-HD4600-x86-Release-ANGLE', + 'Test-Win8-MSVC-ShuttleA-CPU-AVX-x86_64-Debug', + ] + + cov = coverage.coverage() + cov.start() + for case in cases: + args[case] = get_builder_spec(case) + cov.stop() + + this_file = os.path.basename(__file__) + _, _, not_run, _ = cov.analysis(this_file) + filtered = [line for line in not_run if + line > cov_start and line < cov_end and line not in cov_skip] + if filtered: + print 'Lines not covered by test cases: ', filtered + sys.exit(1) + + golden = this_file.replace('.py', '.json') + with open(os.path.join(os.path.dirname(__file__), golden), 'w') as f: + json.dump(args, f, indent=2, sort_keys=True) + + +if __name__ == '__main__': + if len(sys.argv) == 2 and sys.argv[1] == 'test': + self_test() + sys.exit(0) + + if len(sys.argv) != 3: + print usage + sys.exit(1) + + with open(sys.argv[1], 'w') as out: + json.dump(get_builder_spec(sys.argv[2]), out) diff --git a/tools/builder_name_schema.json b/tools/builder_name_schema.json new file mode 100644 index 0000000000..cffe9bfd89 --- /dev/null +++ b/tools/builder_name_schema.json @@ -0,0 +1,40 @@ +{ + "builder_name_schema": { + "Test": [ + "os", + "compiler", + "model", + "cpu_or_gpu", + "cpu_or_gpu_value", + "arch", + "configuration" + ], + "Housekeeper": [ + "frequency" + ], + "Build": [ + "os", + "compiler", + "target_arch", + "configuration" + ], + "Perf": [ + "os", + "compiler", + "model", + "cpu_or_gpu", + "cpu_or_gpu_value", + "arch", + "configuration" + ], + "Canary": [ + "project", + "os", + "compiler", + "target_arch", + "configuration" + ] + }, + "builder_name_sep": "-", + "trybot_name_suffix": "Trybot" +} diff --git a/tools/builder_name_schema.py b/tools/builder_name_schema.py new file mode 100644 index 0000000000..990aa6447c --- /dev/null +++ b/tools/builder_name_schema.py @@ -0,0 +1,194 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +""" Utilities for dealing with builder names. This module obtains its attributes +dynamically from builder_name_schema.json. """ + + +import json +import os + + +# All of these global variables are filled in by _LoadSchema(). + +# The full schema. +BUILDER_NAME_SCHEMA = None + +# Character which separates parts of a builder name. +BUILDER_NAME_SEP = None + +# Builder roles. +BUILDER_ROLE_CANARY = 'Canary' +BUILDER_ROLE_BUILD = 'Build' +BUILDER_ROLE_HOUSEKEEPER = 'Housekeeper' +BUILDER_ROLE_PERF = 'Perf' +BUILDER_ROLE_TEST = 'Test' +BUILDER_ROLES = (BUILDER_ROLE_CANARY, + BUILDER_ROLE_BUILD, + BUILDER_ROLE_HOUSEKEEPER, + BUILDER_ROLE_PERF, + BUILDER_ROLE_TEST) + +# Suffix which distinguishes trybots from normal bots. +TRYBOT_NAME_SUFFIX = None + + +def _LoadSchema(): + """ Load the builder naming schema from the JSON file. """ + + def _UnicodeToStr(obj): + """ Convert all unicode strings in obj to Python strings. """ + if isinstance(obj, unicode): + return str(obj) + elif isinstance(obj, dict): + return dict(map(_UnicodeToStr, obj.iteritems())) + elif isinstance(obj, list): + return list(map(_UnicodeToStr, obj)) + elif isinstance(obj, tuple): + return tuple(map(_UnicodeToStr, obj)) + else: + return obj + + builder_name_json_filename = os.path.join( + os.path.dirname(__file__), 'builder_name_schema.json') + builder_name_schema_json = json.load(open(builder_name_json_filename)) + + global BUILDER_NAME_SCHEMA + BUILDER_NAME_SCHEMA = _UnicodeToStr( + builder_name_schema_json['builder_name_schema']) + + global BUILDER_NAME_SEP + BUILDER_NAME_SEP = _UnicodeToStr( + builder_name_schema_json['builder_name_sep']) + + global TRYBOT_NAME_SUFFIX + TRYBOT_NAME_SUFFIX = _UnicodeToStr( + builder_name_schema_json['trybot_name_suffix']) + + # Since the builder roles are dictionary keys, just assert that the global + # variables above account for all of them. + assert len(BUILDER_ROLES) == len(BUILDER_NAME_SCHEMA) + for role in BUILDER_ROLES: + assert role in BUILDER_NAME_SCHEMA + + +_LoadSchema() + + +def MakeBuilderName(role, extra_config=None, is_trybot=False, **kwargs): + schema = BUILDER_NAME_SCHEMA.get(role) + if not schema: + raise ValueError('%s is not a recognized role.' % role) + for k, v in kwargs.iteritems(): + if BUILDER_NAME_SEP in v: + raise ValueError('%s not allowed in %s.' % (BUILDER_NAME_SEP, v)) + if not k in schema: + raise ValueError('Schema does not contain "%s": %s' %(k, schema)) + if extra_config and BUILDER_NAME_SEP in extra_config: + raise ValueError('%s not allowed in %s.' % (BUILDER_NAME_SEP, + extra_config)) + name_parts = [role] + name_parts.extend([kwargs[attribute] for attribute in schema]) + if extra_config: + name_parts.append(extra_config) + if is_trybot: + name_parts.append(TRYBOT_NAME_SUFFIX) + return BUILDER_NAME_SEP.join(name_parts) + + +def BuilderNameFromObject(obj, is_trybot=False): + """Create a builder name based on properties of the given object. + + Args: + obj: the object from which to create the builder name. The object must + have as properties: + - A valid builder role, as defined in the JSON file + - All properties listed in the JSON file for that role + - Optionally, an extra_config property + is_trybot: bool; whether or not the builder is a trybot. + Returns: + string which combines the properties of the given object into a valid + builder name. + """ + schema = BUILDER_NAME_SCHEMA.get(obj.role) + if not schema: + raise ValueError('%s is not a recognized role.' % obj.role) + name_parts = [obj.role] + for attr_name in schema: + attr_val = getattr(obj, attr_name) + name_parts.append(attr_val) + extra_config = getattr(obj, 'extra_config', None) + if extra_config: + name_parts.append(extra_config) + if is_trybot: + name_parts.append(TRYBOT_NAME_SUFFIX) + return BUILDER_NAME_SEP.join(name_parts) + + +def IsTrybot(builder_name): + """ Returns true if builder_name refers to a trybot (as opposed to a + waterfall bot). """ + return builder_name.endswith(TRYBOT_NAME_SUFFIX) + + +def GetWaterfallBot(builder_name): + """Returns the name of the waterfall bot for this builder. If it is not a + trybot, builder_name is returned unchanged. If it is a trybot the name is + returned without the trybot suffix.""" + if not IsTrybot(builder_name): + return builder_name + return _WithoutSuffix(builder_name, BUILDER_NAME_SEP + TRYBOT_NAME_SUFFIX) + + +def TrybotName(builder_name): + """Returns the name of the trybot clone of this builder. + + If the given builder is a trybot, the name is returned unchanged. If not, the + TRYBOT_NAME_SUFFIX is appended. + """ + if builder_name.endswith(TRYBOT_NAME_SUFFIX): + return builder_name + return builder_name + BUILDER_NAME_SEP + TRYBOT_NAME_SUFFIX + + +def _WithoutSuffix(string, suffix): + """ Returns a copy of string 'string', but with suffix 'suffix' removed. + Raises ValueError if string does not end with suffix. """ + if not string.endswith(suffix): + raise ValueError('_WithoutSuffix: string %s does not end with suffix %s' % ( + string, suffix)) + return string[:-len(suffix)] + + +def DictForBuilderName(builder_name): + """Makes a dictionary containing details about the builder from its name.""" + split_name = builder_name.split(BUILDER_NAME_SEP) + + def pop_front(): + try: + return split_name.pop(0) + except: + raise ValueError('Invalid builder name: %s' % builder_name) + + result = {'is_trybot': False} + + if split_name[-1] == TRYBOT_NAME_SUFFIX: + result['is_trybot'] = True + split_name.pop() + + if split_name[0] in BUILDER_NAME_SCHEMA.keys(): + key_list = BUILDER_NAME_SCHEMA[split_name[0]] + result['role'] = pop_front() + for key in key_list: + result[key] = pop_front() + if split_name: + result['extra_config'] = pop_front() + if split_name: + raise ValueError('Invalid builder name: %s' % builder_name) + else: + raise ValueError('Invalid builder name: %s' % builder_name) + return result + + |