diff options
| -rwxr-xr-x | tools/jenkins/build_artifacts.sh | 5 | ||||
| -rw-r--r-- | tools/jenkins/build_packages.sh | 42 | ||||
| -rw-r--r--[-rwxr-xr-x] | tools/run_tests/artifact_targets.py (renamed from tools/run_tests/build_artifacts.py) | 121 | ||||
| -rw-r--r-- | tools/run_tests/package_targets.py | 73 | ||||
| -rw-r--r-- | tools/run_tests/task_runner.py | 124 |
5 files changed, 253 insertions, 112 deletions
diff --git a/tools/jenkins/build_artifacts.sh b/tools/jenkins/build_artifacts.sh index d5912010e3..9af553ae48 100755 --- a/tools/jenkins/build_artifacts.sh +++ b/tools/jenkins/build_artifacts.sh @@ -36,4 +36,7 @@ # NOTE: No empty lines should appear in this file before igncr is set! set -ex -o igncr || set -ex -python tools/run_tests/build_artifacts.py $@ +curr_platform="$platform" +unset platform # variable named 'platform' breaks the windows build + +python tools/run_tests/task_runner.py -f artifact $language $curr_platform $architecture diff --git a/tools/jenkins/build_packages.sh b/tools/jenkins/build_packages.sh new file mode 100644 index 0000000000..d795e355c7 --- /dev/null +++ b/tools/jenkins/build_packages.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# This script is invoked by Jenkins and triggers build of artifacts. +# +# To prevent cygwin bash complaining about empty lines ending with \r +# we set the igncr option. The option doesn't exist on Linux, so we fallback +# to just 'set -ex' there. +# NOTE: No empty lines should appear in this file before igncr is set! +set -ex -o igncr || set -ex + +curr_platform="$platform" +unset platform # variable named 'platform' breaks the windows build + +python tools/run_tests/task_runner.py -f package $curr_platform diff --git a/tools/run_tests/build_artifacts.py b/tools/run_tests/artifact_targets.py index 0337f1b3d7..a34fa8e4fa 100755..100644 --- a/tools/run_tests/build_artifacts.py +++ b/tools/run_tests/artifact_targets.py @@ -28,28 +28,9 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -"""Builds gRPC distribution artifacts.""" +"""Definition of targets to build artifacts.""" -import argparse -import atexit -import dockerjob -import itertools import jobset -import json -import multiprocessing -import os -import re -import subprocess -import sys -import time -import uuid - -# Docker doesn't clean up after itself, so we do it on exit. -if jobset.platform_string() == 'linux': - atexit.register(lambda: subprocess.call(['stty', 'echo'])) - -ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) -os.chdir(ROOT) def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={}, @@ -58,7 +39,6 @@ def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={}, environ = environ.copy() environ['RUN_COMMAND'] = shell_command - #docker_args = ['-v', '%s/artifacts:/var/local/jenkins/grpc/artifacts' % ROOT] docker_args=[] for k,v in environ.iteritems(): docker_args += ['-e', '%s=%s' % (k, v)] @@ -107,7 +87,7 @@ class CSharpExtArtifact: self.name = 'csharp_ext_%s_%s' % (platform, arch) self.platform = platform self.arch = arch - self.labels = ['csharp', platform, arch] + self.labels = ['artifact', 'csharp', platform, arch] def pre_build_jobspecs(self): if self.platform == 'windows': @@ -147,92 +127,11 @@ class CSharpExtArtifact: return self.name -_ARTIFACTS = [ - CSharpExtArtifact('linux', 'x86'), - CSharpExtArtifact('linux', 'x64'), - CSharpExtArtifact('macos', 'x86'), - CSharpExtArtifact('macos', 'x64'), - CSharpExtArtifact('windows', 'x86'), - CSharpExtArtifact('windows', 'x64') -] - - -def _create_build_map(): - """Maps artifact names and labels to list of artifacts to be built.""" - artifact_build_map = dict([(artifact.name, [artifact]) - for artifact in _ARTIFACTS]) - if len(_ARTIFACTS) > len(artifact_build_map.keys()): - raise Exception('Artifact names need to be unique') - - label_build_map = {} - label_build_map['all'] = [a for a in _ARTIFACTS] # to build all artifacts - for artifact in _ARTIFACTS: - for label in artifact.labels: - if label in label_build_map: - label_build_map[label].append(artifact) - else: - label_build_map[label] = [artifact] - - if set(artifact_build_map.keys()).intersection(label_build_map.keys()): - raise Exception('Artifact names need to be distinct from label names') - return dict( artifact_build_map.items() + label_build_map.items()) - - -_BUILD_MAP = _create_build_map() - -argp = argparse.ArgumentParser(description='Builds distribution artifacts.') -argp.add_argument('-b', '--build', - choices=sorted(_BUILD_MAP.keys()), - nargs='+', - default=['all'], - help='Artifact name or artifact label to build.') -argp.add_argument('-f', '--filter', - choices=sorted(_BUILD_MAP.keys()), - nargs='+', - default=[], - help='Filter artifacts to build with AND semantics.') -argp.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int) -argp.add_argument('-t', '--travis', - default=False, - action='store_const', - const=True) - -args = argp.parse_args() - -# Figure out which artifacts to build -artifacts = [] -for label in args.build: - artifacts += _BUILD_MAP[label] - -# Among target selected by -b, filter out those that don't match the filter -artifacts = [a for a in artifacts if all(f in a.labels for f in args.filter)] -artifacts = sorted(set(artifacts)) - -# Execute pre-build phase -prebuild_jobs = [] -for artifact in artifacts: - prebuild_jobs += artifact.pre_build_jobspecs() -if prebuild_jobs: - num_failures, _ = jobset.run( - prebuild_jobs, newline_on_success=True, maxjobs=args.jobs) - if num_failures != 0: - jobset.message('FAILED', 'Pre-build phase failed.', do_newline=True) - sys.exit(1) - -build_jobs = [] -for artifact in artifacts: - build_jobs.append(artifact.build_jobspec()) -if not build_jobs: - print 'Nothing to build.' - sys.exit(1) - -jobset.message('START', 'Building artifacts.', do_newline=True) -num_failures, _ = jobset.run( - build_jobs, newline_on_success=True, maxjobs=args.jobs) -if num_failures == 0: - jobset.message('SUCCESS', 'All artifacts built successfully.', - do_newline=True) -else: - jobset.message('FAILED', 'Failed to build artifacts.', - do_newline=True) - sys.exit(1) +def targets(): + """Gets list of supported targets""" + return [CSharpExtArtifact('linux', 'x86'), + CSharpExtArtifact('linux', 'x64'), + CSharpExtArtifact('macos', 'x86'), + CSharpExtArtifact('macos', 'x64'), + CSharpExtArtifact('windows', 'x86'), + CSharpExtArtifact('windows', 'x64')] diff --git a/tools/run_tests/package_targets.py b/tools/run_tests/package_targets.py new file mode 100644 index 0000000000..839991e270 --- /dev/null +++ b/tools/run_tests/package_targets.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Definition of targets to build distribution packages.""" + +import jobset + + +def create_jobspec(name, cmdline, environ=None, cwd=None, shell=False, + flake_retries=0, timeout_retries=0): + """Creates jobspec.""" + jobspec = jobset.JobSpec( + cmdline=cmdline, + environ=environ, + cwd=cwd, + shortname='build_package.%s' % (name), + timeout_seconds=10*60, + flake_retries=flake_retries, + timeout_retries=timeout_retries, + shell=shell) + return jobspec + + +class CSharpNugetTarget: + """Builds C# nuget packages.""" + + def __init__(self): + self.name = 'csharp_nuget' + self.labels = ['package', 'csharp', 'windows'] + + def pre_build_jobspecs(self): + return [] + + def build_jobspec(self): + return create_jobspec(self.name, + ['build_packages.bat'], + cwd='src\\csharp', + shell=True) + + def __str__(self): + return self.name + + +def targets(): + """Gets list of supported targets""" + return [CSharpNugetTarget()] diff --git a/tools/run_tests/task_runner.py b/tools/run_tests/task_runner.py new file mode 100644 index 0000000000..39b15cc8b2 --- /dev/null +++ b/tools/run_tests/task_runner.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Runs selected gRPC test/build tasks.""" + +import argparse +import atexit +import jobset +import multiprocessing +import sys + +import artifact_targets +import package_targets + +_TARGETS = [] +_TARGETS += artifact_targets.targets() +_TARGETS += package_targets.targets() + +def _create_build_map(): + """Maps task names and labels to list of tasks to be built.""" + target_build_map = dict([(target.name, [target]) + for target in _TARGETS]) + if len(_TARGETS) > len(target_build_map.keys()): + raise Exception('Target names need to be unique') + + label_build_map = {} + label_build_map['all'] = [t for t in _TARGETS] # to build all targets + for target in _TARGETS: + for label in target.labels: + if label in label_build_map: + label_build_map[label].append(target) + else: + label_build_map[label] = [target] + + if set(target_build_map.keys()).intersection(label_build_map.keys()): + raise Exception('Target names need to be distinct from label names') + return dict( target_build_map.items() + label_build_map.items()) + + +_BUILD_MAP = _create_build_map() + +argp = argparse.ArgumentParser(description='Runs build/test targets.') +argp.add_argument('-b', '--build', + choices=sorted(_BUILD_MAP.keys()), + nargs='+', + default=['all'], + help='Target name or target label to build.') +argp.add_argument('-f', '--filter', + choices=sorted(_BUILD_MAP.keys()), + nargs='+', + default=[], + help='Filter targets to build with AND semantics.') +argp.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int) +argp.add_argument('-t', '--travis', + default=False, + action='store_const', + const=True) + +args = argp.parse_args() + +# Figure out which targets to build +targets = [] +for label in args.build: + targets += _BUILD_MAP[label] + +# Among targets selected by -b, filter out those that don't match the filter +targets = [t for t in targets if all(f in t.labels for f in args.filter)] +targets = sorted(set(targets)) + +# Execute pre-build phase +prebuild_jobs = [] +for target in targets: + prebuild_jobs += target.pre_build_jobspecs() +if prebuild_jobs: + num_failures, _ = jobset.run( + prebuild_jobs, newline_on_success=True, maxjobs=args.jobs) + if num_failures != 0: + jobset.message('FAILED', 'Pre-build phase failed.', do_newline=True) + sys.exit(1) + +build_jobs = [] +for target in targets: + build_jobs.append(target.build_jobspec()) +if not build_jobs: + print 'Nothing to build.' + sys.exit(1) + +jobset.message('START', 'Building targets.', do_newline=True) +num_failures, _ = jobset.run( + build_jobs, newline_on_success=True, maxjobs=args.jobs) +if num_failures == 0: + jobset.message('SUCCESS', 'All targets built successfully.', + do_newline=True) +else: + jobset.message('FAILED', 'Failed to build targets.', + do_newline=True) + sys.exit(1) |