diff options
Diffstat (limited to 'tools/run_tests/artifacts')
18 files changed, 1714 insertions, 0 deletions
diff --git a/tools/run_tests/artifacts/__init__.py b/tools/run_tests/artifacts/__init__.py new file mode 100644 index 0000000000..100a624dc9 --- /dev/null +++ b/tools/run_tests/artifacts/__init__.py @@ -0,0 +1,28 @@ +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py new file mode 100644 index 0000000000..005d99790a --- /dev/null +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -0,0 +1,356 @@ +#!/usr/bin/env python2.7 +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Definition of targets to build artifacts.""" + +import os.path +import random +import string +import sys + +sys.path.insert(0, os.path.abspath('..')) +import python_utils.jobset as jobset + + +def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={}, + flake_retries=0, timeout_retries=0, timeout_seconds=30*60): + """Creates jobspec for a task running under docker.""" + environ = environ.copy() + environ['RUN_COMMAND'] = shell_command + + docker_args=[] + for k,v in environ.items(): + docker_args += ['-e', '%s=%s' % (k, v)] + docker_env = {'DOCKERFILE_DIR': dockerfile_dir, + 'DOCKER_RUN_SCRIPT': 'tools/run_tests/dockerize/docker_run.sh', + 'OUTPUT_DIR': 'artifacts'} + jobspec = jobset.JobSpec( + cmdline=['tools/run_tests/dockerize/build_and_run_docker.sh'] + docker_args, + environ=docker_env, + shortname='build_artifact.%s' % (name), + timeout_seconds=timeout_seconds, + flake_retries=flake_retries, + timeout_retries=timeout_retries) + return jobspec + + +def create_jobspec(name, cmdline, environ=None, shell=False, + flake_retries=0, timeout_retries=0, timeout_seconds=30*60): + """Creates jobspec.""" + jobspec = jobset.JobSpec( + cmdline=cmdline, + environ=environ, + shortname='build_artifact.%s' % (name), + timeout_seconds=timeout_seconds, + flake_retries=flake_retries, + timeout_retries=timeout_retries, + shell=shell) + return jobspec + + +_MACOS_COMPAT_FLAG = '-mmacosx-version-min=10.7' + +_ARCH_FLAG_MAP = { + 'x86': '-m32', + 'x64': '-m64' +} + + +class PythonArtifact: + """Builds Python artifacts.""" + + def __init__(self, platform, arch, py_version): + self.name = 'python_%s_%s_%s' % (platform, arch, py_version) + self.platform = platform + self.arch = arch + self.labels = ['artifact', 'python', platform, arch, py_version] + self.py_version = py_version + + def pre_build_jobspecs(self): + return [] + + def build_jobspec(self): + environ = {} + if self.platform == 'linux': + if self.arch == 'x86': + environ['SETARCH_CMD'] = 'linux32' + # Inside the manylinux container, the python installations are located in + # special places... + environ['PYTHON'] = '/opt/python/{}/bin/python'.format(self.py_version) + environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.py_version) + # Platform autodetection for the manylinux1 image breaks so we set the + # defines ourselves. + # TODO(atash) get better platform-detection support in core so we don't + # need to do this manually... + environ['CFLAGS'] = '-DGPR_MANYLINUX1=1' + environ['GRPC_BUILD_GRPCIO_TOOLS_DEPENDENTS'] = 'TRUE' + environ['GRPC_BUILD_MANYLINUX_WHEEL'] = 'TRUE' + return create_docker_jobspec(self.name, + 'tools/dockerfile/grpc_artifact_python_manylinux_%s' % self.arch, + 'tools/run_tests/artifacts/build_artifact_python.sh', + environ=environ, + timeout_seconds=60*60) + elif self.platform == 'windows': + if 'Python27' in self.py_version or 'Python34' in self.py_version: + environ['EXT_COMPILER'] = 'mingw32' + else: + environ['EXT_COMPILER'] = 'msvc' + # For some reason, the batch script %random% always runs with the same + # seed. We create a random temp-dir here + dir = ''.join(random.choice(string.ascii_uppercase) for _ in range(10)) + return create_jobspec(self.name, + ['tools\\run_tests\\artifacts\\build_artifact_python.bat', + self.py_version, + '32' if self.arch == 'x86' else '64', + dir + ], + environ=environ, + shell=True) + else: + environ['PYTHON'] = self.py_version + environ['SKIP_PIP_INSTALL'] = 'TRUE' + return create_jobspec(self.name, + ['tools/run_tests/artifacts/build_artifact_python.sh'], + environ=environ) + + def __str__(self): + return self.name + + +class RubyArtifact: + """Builds ruby native gem.""" + + def __init__(self, platform, arch): + self.name = 'ruby_native_gem_%s_%s' % (platform, arch) + self.platform = platform + self.arch = arch + self.labels = ['artifact', 'ruby', platform, arch] + + def pre_build_jobspecs(self): + return [] + + def build_jobspec(self): + if self.platform == 'windows': + raise Exception("Not supported yet") + else: + if self.platform == 'linux': + environ = {} + if self.arch == 'x86': + environ['SETARCH_CMD'] = 'linux32' + return create_docker_jobspec(self.name, + 'tools/dockerfile/grpc_artifact_linux_%s' % self.arch, + 'tools/run_tests/artifacts/build_artifact_ruby.sh', + environ=environ) + else: + return create_jobspec(self.name, + ['tools/run_tests/artifacts/build_artifact_ruby.sh']) + + +class CSharpExtArtifact: + """Builds C# native extension library""" + + def __init__(self, platform, arch): + self.name = 'csharp_ext_%s_%s' % (platform, arch) + self.platform = platform + self.arch = arch + self.labels = ['artifact', 'csharp', platform, arch] + + def pre_build_jobspecs(self): + if self.platform == 'windows': + return [create_jobspec('prebuild_%s' % self.name, + ['tools\\run_tests\\helper_scripts\\pre_build_c.bat'], + shell=True, + flake_retries=5, + timeout_retries=2)] + else: + return [] + + def build_jobspec(self): + if self.platform == 'windows': + msbuild_platform = 'Win32' if self.arch == 'x86' else self.arch + return create_jobspec(self.name, + ['tools\\run_tests\\artifacts\\build_artifact_csharp.bat', + 'vsprojects\\grpc_csharp_ext.sln', + '/p:Configuration=Release', + '/p:PlatformToolset=v120', + '/p:Platform=%s' % msbuild_platform], + shell=True) + else: + environ = {'CONFIG': 'opt', + 'EMBED_OPENSSL': 'true', + 'EMBED_ZLIB': 'true', + 'CFLAGS': '-DGPR_BACKWARDS_COMPATIBILITY_MODE', + 'LDFLAGS': ''} + if self.platform == 'linux': + return create_docker_jobspec(self.name, + 'tools/dockerfile/grpc_artifact_linux_%s' % self.arch, + 'tools/run_tests/artifacts/build_artifact_csharp.sh', + environ=environ) + else: + archflag = _ARCH_FLAG_MAP[self.arch] + environ['CFLAGS'] += ' %s %s' % (archflag, _MACOS_COMPAT_FLAG) + environ['LDFLAGS'] += ' %s' % archflag + return create_jobspec(self.name, + ['tools/run_tests/artifacts/build_artifact_csharp.sh'], + environ=environ) + + def __str__(self): + return self.name + + +node_gyp_arch_map = { + 'x86': 'ia32', + 'x64': 'x64' +} + +class NodeExtArtifact: + """Builds Node native extension""" + + def __init__(self, platform, arch): + self.name = 'node_ext_{0}_{1}'.format(platform, arch) + self.platform = platform + self.arch = arch + self.gyp_arch = node_gyp_arch_map[arch] + self.labels = ['artifact', 'node', platform, arch] + + def pre_build_jobspecs(self): + return [] + + def build_jobspec(self): + if self.platform == 'windows': + return create_jobspec(self.name, + ['tools\\run_tests\\artifacts\\build_artifact_node.bat', + self.gyp_arch], + shell=True) + else: + if self.platform == 'linux': + return create_docker_jobspec( + self.name, + 'tools/dockerfile/grpc_artifact_linux_{}'.format(self.arch), + 'tools/run_tests/artifacts/build_artifact_node.sh {}'.format(self.gyp_arch)) + else: + return create_jobspec(self.name, + ['tools/run_tests/artifacts/build_artifact_node.sh', + self.gyp_arch]) + +class PHPArtifact: + """Builds PHP PECL package""" + + def __init__(self, platform, arch): + self.name = 'php_pecl_package_{0}_{1}'.format(platform, arch) + self.platform = platform + self.arch = arch + self.labels = ['artifact', 'php', platform, arch] + + def pre_build_jobspecs(self): + return [] + + def build_jobspec(self): + if self.platform == 'linux': + return create_docker_jobspec( + self.name, + 'tools/dockerfile/grpc_artifact_linux_{}'.format(self.arch), + 'tools/run_tests/artifacts/build_artifact_php.sh') + else: + return create_jobspec(self.name, + ['tools/run_tests/artifacts/build_artifact_php.sh']) + +class ProtocArtifact: + """Builds protoc and protoc-plugin artifacts""" + + def __init__(self, platform, arch): + self.name = 'protoc_%s_%s' % (platform, arch) + self.platform = platform + self.arch = arch + self.labels = ['artifact', 'protoc', platform, arch] + + def pre_build_jobspecs(self): + return [] + + def build_jobspec(self): + if self.platform != 'windows': + cxxflags = '-DNDEBUG %s' % _ARCH_FLAG_MAP[self.arch] + ldflags = '%s' % _ARCH_FLAG_MAP[self.arch] + if self.platform != 'macos': + ldflags += ' -static-libgcc -static-libstdc++ -s' + environ={'CONFIG': 'opt', + 'CXXFLAGS': cxxflags, + 'LDFLAGS': ldflags, + 'PROTOBUF_LDFLAGS_EXTRA': ldflags} + if self.platform == 'linux': + return create_docker_jobspec(self.name, + 'tools/dockerfile/grpc_artifact_protoc', + 'tools/run_tests/artifacts/build_artifact_protoc.sh', + environ=environ) + else: + environ['CXXFLAGS'] += ' -std=c++11 -stdlib=libc++ %s' % _MACOS_COMPAT_FLAG + return create_jobspec(self.name, + ['tools/run_tests/artifacts/build_artifact_protoc.sh'], + environ=environ) + else: + generator = 'Visual Studio 12 Win64' if self.arch == 'x64' else 'Visual Studio 12' + vcplatform = 'x64' if self.arch == 'x64' else 'Win32' + return create_jobspec(self.name, + ['tools\\run_tests\\artifacts\\build_artifact_protoc.bat'], + environ={'generator': generator, + 'Platform': vcplatform}) + + def __str__(self): + return self.name + + +def targets(): + """Gets list of supported targets""" + return ([Cls(platform, arch) + for Cls in (CSharpExtArtifact, NodeExtArtifact, ProtocArtifact) + for platform in ('linux', 'macos', 'windows') + for arch in ('x86', 'x64')] + + [PythonArtifact('linux', 'x86', 'cp27-cp27m'), + PythonArtifact('linux', 'x86', 'cp27-cp27mu'), + PythonArtifact('linux', 'x86', 'cp34-cp34m'), + PythonArtifact('linux', 'x86', 'cp35-cp35m'), + PythonArtifact('linux', 'x64', 'cp27-cp27m'), + PythonArtifact('linux', 'x64', 'cp27-cp27mu'), + PythonArtifact('linux', 'x64', 'cp34-cp34m'), + PythonArtifact('linux', 'x64', 'cp35-cp35m'), + PythonArtifact('macos', 'x64', 'python2.7'), + PythonArtifact('macos', 'x64', 'python3.4'), + PythonArtifact('macos', 'x64', 'python3.5'), + PythonArtifact('windows', 'x86', 'Python27_32bits'), + PythonArtifact('windows', 'x86', 'Python34_32bits'), + PythonArtifact('windows', 'x86', 'Python35_32bits'), + PythonArtifact('windows', 'x64', 'Python27'), + PythonArtifact('windows', 'x64', 'Python34'), + PythonArtifact('windows', 'x64', 'Python35'), + RubyArtifact('linux', 'x86'), + RubyArtifact('linux', 'x64'), + RubyArtifact('macos', 'x64'), + PHPArtifact('linux', 'x64'), + PHPArtifact('macos', 'x64')]) diff --git a/tools/run_tests/artifacts/build_artifact_csharp.bat b/tools/run_tests/artifacts/build_artifact_csharp.bat new file mode 100644 index 0000000000..24c8d485f9 --- /dev/null +++ b/tools/run_tests/artifacts/build_artifact_csharp.bat @@ -0,0 +1,41 @@ +@rem Copyright 2016, Google Inc. +@rem All rights reserved. +@rem +@rem Redistribution and use in source and binary forms, with or without +@rem modification, are permitted provided that the following conditions are +@rem met: +@rem +@rem * Redistributions of source code must retain the above copyright +@rem notice, this list of conditions and the following disclaimer. +@rem * Redistributions in binary form must reproduce the above +@rem copyright notice, this list of conditions and the following disclaimer +@rem in the documentation and/or other materials provided with the +@rem distribution. +@rem * Neither the name of Google Inc. nor the names of its +@rem contributors may be used to endorse or promote products derived from +@rem this software without specific prior written permission. +@rem +@rem THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +@rem "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +@rem LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +@rem A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +@rem OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +@rem SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +@rem LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +@rem DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +@rem THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +@rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +@rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +@rem Builds C# artifacts on Windows + +@call vsprojects\build_vs2013.bat %* || goto :error + +mkdir artifacts +copy /Y vsprojects\Release\grpc_csharp_ext.dll artifacts || copy /Y vsprojects\x64\Release\grpc_csharp_ext.dll artifacts || goto :error + +goto :EOF + +:error +echo Failed! +exit /b %errorlevel% diff --git a/tools/run_tests/artifacts/build_artifact_csharp.sh b/tools/run_tests/artifacts/build_artifact_csharp.sh new file mode 100755 index 0000000000..aed04b2745 --- /dev/null +++ b/tools/run_tests/artifacts/build_artifact_csharp.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set -ex + +cd $(dirname $0)/../../.. + +make grpc_csharp_ext + +mkdir -p artifacts +cp libs/opt/libgrpc_csharp_ext.so artifacts || cp libs/opt/libgrpc_csharp_ext.dylib artifacts diff --git a/tools/run_tests/artifacts/build_artifact_node.bat b/tools/run_tests/artifacts/build_artifact_node.bat new file mode 100644 index 0000000000..2e0ecd21d0 --- /dev/null +++ b/tools/run_tests/artifacts/build_artifact_node.bat @@ -0,0 +1,55 @@ +@rem Copyright 2016, Google Inc. +@rem All rights reserved. +@rem +@rem Redistribution and use in source and binary forms, with or without +@rem modification, are permitted provided that the following conditions are +@rem met: +@rem +@rem * Redistributions of source code must retain the above copyright +@rem notice, this list of conditions and the following disclaimer. +@rem * Redistributions in binary form must reproduce the above +@rem copyright notice, this list of conditions and the following disclaimer +@rem in the documentation and/or other materials provided with the +@rem distribution. +@rem * Neither the name of Google Inc. nor the names of its +@rem contributors may be used to endorse or promote products derived from +@rem this software without specific prior written permission. +@rem +@rem THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +@rem "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +@rem LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +@rem A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +@rem OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +@rem SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +@rem LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +@rem DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +@rem THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +@rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +@rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set node_versions=0.12.0 1.0.0 1.1.0 2.0.0 3.0.0 4.0.0 5.0.0 6.0.0 7.0.0 + +set PATH=%PATH%;C:\Program Files\nodejs\;%APPDATA%\npm + +del /f /q BUILD || rmdir build /s /q + +call npm update || goto :error + +mkdir artifacts + +for %%v in (%node_versions%) do ( + call .\node_modules\.bin\node-pre-gyp.cmd configure build --target=%%v --target_arch=%1 + +@rem Try again after removing openssl headers + rmdir "%HOMEDRIVE%%HOMEPATH%\.node-gyp\%%v\include\node\openssl" /S /Q + rmdir "%HOMEDRIVE%%HOMEPATH%\.node-gyp\iojs-%%v\include\node\openssl" /S /Q + call .\node_modules\.bin\node-pre-gyp.cmd build package testpackage --target=%%v --target_arch=%1 || goto :error + + xcopy /Y /I /S build\stage\* artifacts\ || goto :error +) +if %errorlevel% neq 0 exit /b %errorlevel% + +goto :EOF + +:error +exit /b 1 diff --git a/tools/run_tests/artifacts/build_artifact_node.sh b/tools/run_tests/artifacts/build_artifact_node.sh new file mode 100755 index 0000000000..1066ebde19 --- /dev/null +++ b/tools/run_tests/artifacts/build_artifact_node.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +NODE_TARGET_ARCH=$1 +source ~/.nvm/nvm.sh + +nvm use 4 +set -ex + +cd $(dirname $0)/../../.. + +rm -rf build || true + +mkdir -p artifacts + +npm update + +node_versions=( 0.12.0 1.0.0 1.1.0 2.0.0 3.0.0 4.0.0 5.0.0 6.0.0 7.0.0 ) + +for version in ${node_versions[@]} +do + ./node_modules/.bin/node-pre-gyp configure rebuild package testpackage --target=$version --target_arch=$NODE_TARGET_ARCH + cp -r build/stage/* artifacts/ +done diff --git a/tools/run_tests/artifacts/build_artifact_php.sh b/tools/run_tests/artifacts/build_artifact_php.sh new file mode 100755 index 0000000000..c8d55860c1 --- /dev/null +++ b/tools/run_tests/artifacts/build_artifact_php.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +PHP_TARGET_ARCH=$1 +set -ex + +cd $(dirname $0)/../../.. + +mkdir -p artifacts + +pear package + +cp -r grpc-*.tgz artifacts/ diff --git a/tools/run_tests/artifacts/build_artifact_protoc.bat b/tools/run_tests/artifacts/build_artifact_protoc.bat new file mode 100644 index 0000000000..fd93318833 --- /dev/null +++ b/tools/run_tests/artifacts/build_artifact_protoc.bat @@ -0,0 +1,48 @@ +@rem Copyright 2016, Google Inc. +@rem All rights reserved. +@rem +@rem Redistribution and use in source and binary forms, with or without +@rem modification, are permitted provided that the following conditions are +@rem met: +@rem +@rem * Redistributions of source code must retain the above copyright +@rem notice, this list of conditions and the following disclaimer. +@rem * Redistributions in binary form must reproduce the above +@rem copyright notice, this list of conditions and the following disclaimer +@rem in the documentation and/or other materials provided with the +@rem distribution. +@rem * Neither the name of Google Inc. nor the names of its +@rem contributors may be used to endorse or promote products derived from +@rem this software without specific prior written permission. +@rem +@rem THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +@rem "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +@rem LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +@rem A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +@rem OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +@rem SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +@rem LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +@rem DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +@rem THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +@rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +@rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +mkdir artifacts + +setlocal +cd third_party/protobuf/cmake + +mkdir build & cd build +mkdir solution & cd solution +cmake -G "%generator%" -Dprotobuf_BUILD_TESTS=OFF ../../.. || goto :error +endlocal + +call vsprojects/build_plugins.bat || goto :error + +xcopy /Y third_party\protobuf\cmake\build\solution\Release\protoc.exe artifacts\ || goto :error +xcopy /Y vsprojects\Release\*_plugin.exe artifacts\ || xcopy /Y vsprojects\x64\Release\*_plugin.exe artifacts\ || goto :error + +goto :EOF + +:error +exit /b 1
\ No newline at end of file diff --git a/tools/run_tests/artifacts/build_artifact_protoc.sh b/tools/run_tests/artifacts/build_artifact_protoc.sh new file mode 100755 index 0000000000..26c2280eff --- /dev/null +++ b/tools/run_tests/artifacts/build_artifact_protoc.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Use devtoolset environment that has GCC 4.7 before set -ex +source scl_source enable devtoolset-1.1 + +set -ex + +cd $(dirname $0)/../../.. + +make plugins + +mkdir -p artifacts +cp bins/opt/protobuf/protoc bins/opt/*_plugin artifacts/ diff --git a/tools/run_tests/artifacts/build_artifact_python.bat b/tools/run_tests/artifacts/build_artifact_python.bat new file mode 100644 index 0000000000..246713a6ce --- /dev/null +++ b/tools/run_tests/artifacts/build_artifact_python.bat @@ -0,0 +1,78 @@ +@rem Copyright 2016, Google Inc. +@rem All rights reserved. +@rem +@rem Redistribution and use in source and binary forms, with or without +@rem modification, are permitted provided that the following conditions are +@rem met: +@rem +@rem * Redistributions of source code must retain the above copyright +@rem notice, this list of conditions and the following disclaimer. +@rem * Redistributions in binary form must reproduce the above +@rem copyright notice, this list of conditions and the following disclaimer +@rem in the documentation and/or other materials provided with the +@rem distribution. +@rem * Neither the name of Google Inc. nor the names of its +@rem contributors may be used to endorse or promote products derived from +@rem this software without specific prior written permission. +@rem +@rem THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +@rem "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +@rem LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +@rem A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +@rem OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +@rem SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +@rem LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +@rem DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +@rem THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +@rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +@rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +set PATH=C:\%1;C:\%1\scripts;C:\msys64\mingw%2\bin;%PATH% + +pip install --upgrade six +pip install --upgrade setuptools +pip install -rrequirements.txt + +set GRPC_PYTHON_BUILD_WITH_CYTHON=1 + +@rem Multiple builds are running simultaneously, so to avoid distutils +@rem file collisions, we build everything in a tmp directory +if not exist "artifacts" mkdir "artifacts" +set ARTIFACT_DIR=%cd%\artifacts +set BUILD_DIR=C:\Windows\Temp\pygrpc-%3\ +mkdir %BUILD_DIR% +xcopy /s/e/q %cd%\* %BUILD_DIR% +pushd %BUILD_DIR% + + +@rem Set up gRPC Python tools +python tools\distrib\python\make_grpcio_tools.py + +@rem Build gRPC Python extensions +python setup.py build_ext -c %EXT_COMPILER% || goto :error + +pushd tools\distrib\python\grpcio_tools +python setup.py build_ext -c %EXT_COMPILER% || goto :error +popd + +@rem Build gRPC Python distributions +python setup.py bdist_wheel || goto :error + +pushd tools\distrib\python\grpcio_tools +python setup.py bdist_wheel || goto :error +popd + + +xcopy /Y /I /S dist\* %ARTIFACT_DIR% || goto :error +xcopy /Y /I /S tools\distrib\python\grpcio_tools\dist\* %ARTIFACT_DIR% || goto :error + +popd +rmdir /s /q %BUILD_DIR% + +goto :EOF + +:error +popd +rmdir /s /q %BUILD_DIR% +exit /b 1 diff --git a/tools/run_tests/artifacts/build_artifact_python.sh b/tools/run_tests/artifacts/build_artifact_python.sh new file mode 100755 index 0000000000..5a5506029a --- /dev/null +++ b/tools/run_tests/artifacts/build_artifact_python.sh @@ -0,0 +1,103 @@ +#!/bin/bash +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set -ex + +cd $(dirname $0)/../../.. + +export GRPC_PYTHON_USE_CUSTOM_BDIST=0 +export GRPC_PYTHON_BUILD_WITH_CYTHON=1 +export PYTHON=${PYTHON:-python} +export PIP=${PIP:-pip} +export AUDITWHEEL=${AUDITWHEEL:-auditwheel} + +# Because multiple builds run in parallel, some distutils file +# operations may collide. To avoid this, each build is run in +# a temp directory +mkdir -p artifacts +ARTIFACT_DIR="$PWD/artifacts" +BUILD_DIR=`mktemp -d "${TMPDIR:-/tmp}/pygrpc.XXXXXX"` +trap "rm -rf $BUILD_DIR" EXIT +cp -r * "$BUILD_DIR" +cd "$BUILD_DIR" + +# Build the source distribution first because MANIFEST.in cannot override +# exclusion of built shared objects among package resources (for some +# inexplicable reason). +${SETARCH_CMD} ${PYTHON} setup.py sdist + +# Wheel has a bug where directories don't get excluded. +# https://bitbucket.org/pypa/wheel/issues/99/cannot-exclude-directory +${SETARCH_CMD} ${PYTHON} setup.py bdist_wheel + +# Build gRPC tools package distribution +${PYTHON} tools/distrib/python/make_grpcio_tools.py + +# Build gRPC tools package source distribution +${SETARCH_CMD} ${PYTHON} tools/distrib/python/grpcio_tools/setup.py sdist + +# Build gRPC tools package binary distribution +${SETARCH_CMD} ${PYTHON} tools/distrib/python/grpcio_tools/setup.py bdist_wheel + +if [ "$GRPC_BUILD_MANYLINUX_WHEEL" != "" ] +then + for wheel in dist/*.whl; do + ${AUDITWHEEL} repair $wheel -w "$ARTIFACT_DIR" + rm $wheel + done + for wheel in tools/distrib/python/grpcio_tools/dist/*.whl; do + ${AUDITWHEEL} repair $wheel -w "$ARTIFACT_DIR" + rm $wheel + done +fi + +# We need to use the built grpcio-tools/grpcio to compile the health proto +# Wheels are not supported by setup_requires/dependency_links, so we +# manually install the dependency. Note we should only do this if we +# are in a docker image or in a virtualenv. +if [ "$GRPC_BUILD_GRPCIO_TOOLS_DEPENDENTS" != "" ] +then + ${PIP} install -rrequirements.txt + ${PIP} install grpcio --no-index --find-links "file://$ARTIFACT_DIR/" + ${PIP} install grpcio-tools --no-index --find-links "file://$ARTIFACT_DIR/" + + # Build gRPC health-checking source distribution + ${SETARCH_CMD} ${PYTHON} src/python/grpcio_health_checking/setup.py \ + preprocess build_package_protos sdist + cp -r src/python/grpcio_health_checking/dist/* "$ARTIFACT_DIR" + + # Build gRPC reflection source distribution + ${SETARCH_CMD} ${PYTHON} src/python/grpcio_reflection/setup.py \ + preprocess build_package_protos sdist + cp -r src/python/grpcio_reflection/dist/* "$ARTIFACT_DIR" +fi + +cp -r dist/* "$ARTIFACT_DIR" +cp -r tools/distrib/python/grpcio_tools/dist/* "$ARTIFACT_DIR" diff --git a/tools/run_tests/artifacts/build_artifact_ruby.sh b/tools/run_tests/artifacts/build_artifact_ruby.sh new file mode 100755 index 0000000000..019efb01fd --- /dev/null +++ b/tools/run_tests/artifacts/build_artifact_ruby.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +set -ex + +SYSTEM=`uname | cut -f 1 -d_` + +cd $(dirname $0)/../../.. +set +ex +[[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh +[[ -s "$HOME/.rvm/scripts/rvm" ]] && source "$HOME/.rvm/scripts/rvm" +set -ex + +if [ "$SYSTEM" == "MSYS" ] ; then + SYSTEM=MINGW32 +fi +if [ "$SYSTEM" == "MINGW64" ] ; then + SYSTEM=MINGW32 +fi + +if [ "$SYSTEM" == "MINGW32" ] ; then + echo "Need Linux to build the Windows ruby gem." + exit 1 +fi + +set +ex +rvm use default +gem install bundler --update +bundle install +set -ex + +rake gem:native + +if [ "$SYSTEM" == "Darwin" ] ; then + rm `ls pkg/*.gem | grep -v darwin` +fi + +mkdir -p artifacts + +cp pkg/*.gem artifacts diff --git a/tools/run_tests/artifacts/build_package_node.sh b/tools/run_tests/artifacts/build_package_node.sh new file mode 100755 index 0000000000..8b5e8c0bc1 --- /dev/null +++ b/tools/run_tests/artifacts/build_package_node.sh @@ -0,0 +1,99 @@ +#!/bin/bash +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +source ~/.nvm/nvm.sh + +nvm use 4 +set -ex + +cd $(dirname $0)/../../.. + +base=$(pwd) + +artifacts=$base/artifacts + +mkdir -p $artifacts +cp -r $EXTERNAL_GIT_ROOT/architecture={x86,x64},language=node,platform={windows,linux,macos}/artifacts/* $artifacts/ || true + +npm update +npm pack + +cp grpc-*.tgz $artifacts/grpc.tgz + +mkdir -p bin + +cd $base/src/node/health_check +npm pack +cp grpc-health-check-*.tgz $artifacts/ + +cd $base/src/node/tools +npm update +npm pack +cp grpc-tools-*.tgz $artifacts/ +tools_version=$(npm list | grep -oP '(?<=grpc-tools@)\S+') + +output_dir=$artifacts/grpc-precompiled-binaries/node/grpc-tools/v$tools_version +mkdir -p $output_dir + +well_known_protos=( any api compiler/plugin descriptor duration empty field_mask source_context struct timestamp type wrappers ) + +for arch in {x86,x64}; do + case $arch in + x86) + node_arch=ia32 + ;; + *) + node_arch=$arch + ;; + esac + for plat in {windows,linux,macos}; do + case $plat in + windows) + node_plat=win32 + ;; + macos) + node_plat=darwin + ;; + *) + node_plat=$plat + ;; + esac + rm -r bin/* + input_dir="$EXTERNAL_GIT_ROOT/architecture=$arch,language=protoc,platform=$plat/artifacts" + cp $input_dir/protoc* bin/ + cp $input_dir/grpc_node_plugin* bin/ + mkdir -p bin/google/protobuf + mkdir -p bin/google/protobuf/compiler # needed for plugin.proto + for proto in "${well_known_protos[@]}"; do + cp $base/third_party/protobuf/src/google/protobuf/$proto.proto bin/google/protobuf/$proto.proto + done + tar -czf $output_dir/$node_plat-$node_arch.tar.gz bin/ + done +done diff --git a/tools/run_tests/artifacts/build_package_php.sh b/tools/run_tests/artifacts/build_package_php.sh new file mode 100755 index 0000000000..42a8d9f8df --- /dev/null +++ b/tools/run_tests/artifacts/build_package_php.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set -ex + +cd $(dirname $0)/../../.. + +mkdir -p artifacts/ +cp -r $EXTERNAL_GIT_ROOT/architecture={x86,x64},language=php,platform={windows,linux,macos}/artifacts/* artifacts/ || true diff --git a/tools/run_tests/artifacts/build_package_python.sh b/tools/run_tests/artifacts/build_package_python.sh new file mode 100755 index 0000000000..4a1c15ceee --- /dev/null +++ b/tools/run_tests/artifacts/build_package_python.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set -ex + +cd $(dirname $0)/../../.. + +mkdir -p artifacts/ + +# All the python packages have been built in the artifact phase already +# and we only collect them here to deliver them to the distribtest phase. +cp -r $EXTERNAL_GIT_ROOT/architecture={x86,x64},language=python,platform={windows,linux,macos}/artifacts/* artifacts/ || true + +# TODO: all the artifact builder configurations generate a grpcio-VERSION.tar.gz +# source distribution package, and only one of them will end up +# in the artifacts/ directory. They should be all equivalent though. diff --git a/tools/run_tests/artifacts/build_package_ruby.sh b/tools/run_tests/artifacts/build_package_ruby.sh new file mode 100755 index 0000000000..b4d20d8a4c --- /dev/null +++ b/tools/run_tests/artifacts/build_package_ruby.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set -ex + +cd $(dirname $0)/../../.. + +base=$(pwd) + +mkdir -p artifacts/ + +# All the ruby packages have been built in the artifact phase already +# and we only collect them here to deliver them to the distribtest phase. +cp -r $EXTERNAL_GIT_ROOT/architecture={x86,x64},language=ruby,platform={windows,linux,macos}/artifacts/* artifacts/ || true + +well_known_protos=( any api compiler/plugin descriptor duration empty field_mask source_context struct timestamp type wrappers ) + +# TODO: all the artifact builder configurations generate a grpc-VERSION.gem +# source distribution package, and only one of them will end up +# in the artifacts/ directory. They should be all equivalent though. + +for arch in {x86,x64}; do + case $arch in + x64) + ruby_arch=x86_64 + ;; + *) + ruby_arch=$arch + ;; + esac + for plat in {windows,linux,macos}; do + input_dir="$EXTERNAL_GIT_ROOT/architecture=$arch,language=protoc,platform=$plat/artifacts" + output_dir="$base/src/ruby/tools/bin/${ruby_arch}-${plat}" + mkdir -p $output_dir/google/protobuf + mkdir -p $output_dir/google/protobuf/compiler # needed for plugin.proto + cp $input_dir/protoc* $output_dir/ + cp $input_dir/grpc_ruby_plugin* $output_dir/ + for proto in "${well_known_protos[@]}"; do + cp $base/third_party/protobuf/src/google/protobuf/$proto.proto $output_dir/google/protobuf/$proto.proto + done + done +done + +cd $base/src/ruby/tools +gem build grpc-tools.gemspec +cp ./grpc-tools*.gem $base/artifacts/ diff --git a/tools/run_tests/artifacts/distribtest_targets.py b/tools/run_tests/artifacts/distribtest_targets.py new file mode 100644 index 0000000000..a7535b3852 --- /dev/null +++ b/tools/run_tests/artifacts/distribtest_targets.py @@ -0,0 +1,335 @@ +#!/usr/bin/env python2.7 +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Definition of targets run distribution package tests.""" + +import os.path +import sys + +sys.path.insert(0, os.path.abspath('..')) +import python_utils.jobset as jobset + + +def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={}, + flake_retries=0, timeout_retries=0): + """Creates jobspec for a task running under docker.""" + environ = environ.copy() + environ['RUN_COMMAND'] = shell_command + environ['RELATIVE_COPY_PATH'] = 'test/distrib' + + docker_args=[] + for k,v in environ.items(): + docker_args += ['-e', '%s=%s' % (k, v)] + docker_env = {'DOCKERFILE_DIR': dockerfile_dir, + 'DOCKER_RUN_SCRIPT': 'tools/run_tests/dockerize/docker_run.sh'} + jobspec = jobset.JobSpec( + cmdline=['tools/run_tests/dockerize/build_and_run_docker.sh'] + docker_args, + environ=docker_env, + shortname='distribtest.%s' % (name), + timeout_seconds=30*60, + flake_retries=flake_retries, + timeout_retries=timeout_retries) + return jobspec + + +def create_jobspec(name, cmdline, environ=None, shell=False, + flake_retries=0, timeout_retries=0): + """Creates jobspec.""" + jobspec = jobset.JobSpec( + cmdline=cmdline, + environ=environ, + shortname='distribtest.%s' % (name), + timeout_seconds=10*60, + flake_retries=flake_retries, + timeout_retries=timeout_retries, + shell=shell) + return jobspec + + +class CSharpDistribTest(object): + """Tests C# NuGet package""" + + def __init__(self, platform, arch, docker_suffix=None, use_dotnet_cli=False): + self.name = 'csharp_nuget_%s_%s' % (platform, arch) + self.platform = platform + self.arch = arch + self.docker_suffix = docker_suffix + self.labels = ['distribtest', 'csharp', platform, arch] + self.script_suffix = '' + if docker_suffix: + self.name += '_%s' % docker_suffix + self.labels.append(docker_suffix) + if use_dotnet_cli: + self.name += '_dotnetcli' + self.script_suffix = '_dotnetcli' + self.labels.append('dotnetcli') + else: + self.labels.append('olddotnet') + + def pre_build_jobspecs(self): + return [] + + def build_jobspec(self): + if self.platform == 'linux': + return create_docker_jobspec(self.name, + 'tools/dockerfile/distribtest/csharp_%s_%s' % ( + self.docker_suffix, + self.arch), + 'test/distrib/csharp/run_distrib_test%s.sh' % self.script_suffix) + elif self.platform == 'macos': + return create_jobspec(self.name, + ['test/distrib/csharp/run_distrib_test%s.sh' % self.script_suffix], + environ={'EXTERNAL_GIT_ROOT': '../../..'}) + elif self.platform == 'windows': + if self.arch == 'x64': + environ={'MSBUILD_EXTRA_ARGS': '/p:Platform=x64', + 'DISTRIBTEST_OUTPATH': 'DistribTest\\bin\\x64\\Debug'} + else: + environ={'DISTRIBTEST_OUTPATH': 'DistribTest\\bin\\\Debug'} + return create_jobspec(self.name, + ['test\\distrib\\csharp\\run_distrib_test%s.bat' % self.script_suffix], + environ=environ) + else: + raise Exception("Not supported yet.") + + def __str__(self): + return self.name + +class NodeDistribTest(object): + """Tests Node package""" + + def __init__(self, platform, arch, docker_suffix, node_version): + self.name = 'node_npm_%s_%s_%s' % (platform, arch, node_version) + self.platform = platform + self.arch = arch + self.node_version = node_version + self.labels = ['distribtest', 'node', platform, arch, + 'node-%s' % node_version] + if docker_suffix is not None: + self.name += '_%s' % docker_suffix + self.docker_suffix = docker_suffix + self.labels.append(docker_suffix) + + def pre_build_jobspecs(self): + return [] + + def build_jobspec(self): + if self.platform == 'linux': + linux32 = '' + if self.arch == 'x86': + linux32 = 'linux32' + return create_docker_jobspec(self.name, + 'tools/dockerfile/distribtest/node_%s_%s' % ( + self.docker_suffix, + self.arch), + '%s test/distrib/node/run_distrib_test.sh %s' % ( + linux32, + self.node_version)) + elif self.platform == 'macos': + return create_jobspec(self.name, + ['test/distrib/node/run_distrib_test.sh', + str(self.node_version)], + environ={'EXTERNAL_GIT_ROOT': '../../..'}) + else: + raise Exception("Not supported yet.") + + def __str__(self): + return self.name + + +class PythonDistribTest(object): + """Tests Python package""" + + def __init__(self, platform, arch, docker_suffix): + self.name = 'python_%s_%s_%s' % (platform, arch, docker_suffix) + self.platform = platform + self.arch = arch + self.docker_suffix = docker_suffix + self.labels = ['distribtest', 'python', platform, arch, docker_suffix] + + def pre_build_jobspecs(self): + return [] + + def build_jobspec(self): + if not self.platform == 'linux': + raise Exception("Not supported yet.") + + return create_docker_jobspec(self.name, + 'tools/dockerfile/distribtest/python_%s_%s' % ( + self.docker_suffix, + self.arch), + 'test/distrib/python/run_distrib_test.sh') + + def __str__(self): + return self.name + + +class RubyDistribTest(object): + """Tests Ruby package""" + + def __init__(self, platform, arch, docker_suffix): + self.name = 'ruby_%s_%s_%s' % (platform, arch, docker_suffix) + self.platform = platform + self.arch = arch + self.docker_suffix = docker_suffix + self.labels = ['distribtest', 'ruby', platform, arch, docker_suffix] + + def pre_build_jobspecs(self): + return [] + + def build_jobspec(self): + if not self.platform == 'linux': + raise Exception("Not supported yet.") + + return create_docker_jobspec(self.name, + 'tools/dockerfile/distribtest/ruby_%s_%s' % ( + self.docker_suffix, + self.arch), + 'test/distrib/ruby/run_distrib_test.sh') + + def __str__(self): + return self.name + + +class PHPDistribTest(object): + """Tests PHP package""" + + def __init__(self, platform, arch, docker_suffix=None): + self.name = 'php_%s_%s_%s' % (platform, arch, docker_suffix) + self.platform = platform + self.arch = arch + self.docker_suffix = docker_suffix + self.labels = ['distribtest', 'php', platform, arch, docker_suffix] + + def pre_build_jobspecs(self): + return [] + + def build_jobspec(self): + if self.platform == 'linux': + return create_docker_jobspec(self.name, + 'tools/dockerfile/distribtest/php_%s_%s' % ( + self.docker_suffix, + self.arch), + 'test/distrib/php/run_distrib_test.sh') + elif self.platform == 'macos': + return create_jobspec(self.name, + ['test/distrib/php/run_distrib_test.sh'], + environ={'EXTERNAL_GIT_ROOT': '../../..'}) + else: + raise Exception("Not supported yet.") + + def __str__(self): + return self.name + + +class CppDistribTest(object): + """Tests Cpp make intall by building examples.""" + + def __init__(self, platform, arch, docker_suffix=None): + self.name = 'cpp_%s_%s_%s' % (platform, arch, docker_suffix) + self.platform = platform + self.arch = arch + self.docker_suffix = docker_suffix + self.labels = ['distribtest', 'cpp', platform, arch, docker_suffix] + + def pre_build_jobspecs(self): + return [] + + def build_jobspec(self): + if self.platform == 'linux': + return create_docker_jobspec(self.name, + 'tools/dockerfile/distribtest/cpp_%s_%s' % ( + self.docker_suffix, + self.arch), + 'test/distrib/cpp/run_distrib_test.sh') + else: + raise Exception("Not supported yet.") + + def __str__(self): + return self.name + + +def targets(): + """Gets list of supported targets""" + return [CppDistribTest('linux', 'x64', 'jessie'), + CSharpDistribTest('linux', 'x64', 'wheezy'), + CSharpDistribTest('linux', 'x64', 'jessie'), + CSharpDistribTest('linux', 'x86', 'jessie'), + CSharpDistribTest('linux', 'x64', 'centos7'), + CSharpDistribTest('linux', 'x64', 'ubuntu1404'), + CSharpDistribTest('linux', 'x64', 'ubuntu1504'), + CSharpDistribTest('linux', 'x64', 'ubuntu1510'), + CSharpDistribTest('linux', 'x64', 'ubuntu1604'), + CSharpDistribTest('linux', 'x64', 'ubuntu1404', use_dotnet_cli=True), + CSharpDistribTest('macos', 'x86'), + CSharpDistribTest('windows', 'x86'), + CSharpDistribTest('windows', 'x64'), + PythonDistribTest('linux', 'x64', 'wheezy'), + PythonDistribTest('linux', 'x64', 'jessie'), + PythonDistribTest('linux', 'x86', 'jessie'), + PythonDistribTest('linux', 'x64', 'centos6'), + PythonDistribTest('linux', 'x64', 'centos7'), + PythonDistribTest('linux', 'x64', 'fedora20'), + PythonDistribTest('linux', 'x64', 'fedora21'), + PythonDistribTest('linux', 'x64', 'fedora22'), + PythonDistribTest('linux', 'x64', 'fedora23'), + PythonDistribTest('linux', 'x64', 'opensuse'), + PythonDistribTest('linux', 'x64', 'arch'), + PythonDistribTest('linux', 'x64', 'ubuntu1204'), + PythonDistribTest('linux', 'x64', 'ubuntu1404'), + PythonDistribTest('linux', 'x64', 'ubuntu1504'), + PythonDistribTest('linux', 'x64', 'ubuntu1510'), + PythonDistribTest('linux', 'x64', 'ubuntu1604'), + RubyDistribTest('linux', 'x64', 'wheezy'), + RubyDistribTest('linux', 'x64', 'jessie'), + RubyDistribTest('linux', 'x86', 'jessie'), + RubyDistribTest('linux', 'x64', 'centos6'), + RubyDistribTest('linux', 'x64', 'centos7'), + RubyDistribTest('linux', 'x64', 'fedora20'), + RubyDistribTest('linux', 'x64', 'fedora21'), + RubyDistribTest('linux', 'x64', 'fedora22'), + RubyDistribTest('linux', 'x64', 'fedora23'), + RubyDistribTest('linux', 'x64', 'opensuse'), + RubyDistribTest('linux', 'x64', 'ubuntu1204'), + RubyDistribTest('linux', 'x64', 'ubuntu1404'), + RubyDistribTest('linux', 'x64', 'ubuntu1504'), + RubyDistribTest('linux', 'x64', 'ubuntu1510'), + RubyDistribTest('linux', 'x64', 'ubuntu1604'), + NodeDistribTest('macos', 'x64', None, '4'), + NodeDistribTest('macos', 'x64', None, '5'), + NodeDistribTest('linux', 'x86', 'jessie', '4'), + PHPDistribTest('linux', 'x64', 'jessie'), + PHPDistribTest('macos', 'x64'), + ] + [ + NodeDistribTest('linux', 'x64', os, version) + for os in ('wheezy', 'jessie', 'ubuntu1204', 'ubuntu1404', + 'ubuntu1504', 'ubuntu1510', 'ubuntu1604') + for version in ('0.12', '3', '4', '5') + ] diff --git a/tools/run_tests/artifacts/package_targets.py b/tools/run_tests/artifacts/package_targets.py new file mode 100644 index 0000000000..d490f571c3 --- /dev/null +++ b/tools/run_tests/artifacts/package_targets.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python2.7 +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Definition of targets to build distribution packages.""" + +import os.path +import sys + +sys.path.insert(0, os.path.abspath('..')) +import python_utils.jobset as jobset + + +def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={}, + flake_retries=0, timeout_retries=0): + """Creates jobspec for a task running under docker.""" + environ = environ.copy() + environ['RUN_COMMAND'] = shell_command + + docker_args=[] + for k,v in environ.items(): + docker_args += ['-e', '%s=%s' % (k, v)] + docker_env = {'DOCKERFILE_DIR': dockerfile_dir, + 'DOCKER_RUN_SCRIPT': 'tools/run_tests/dockerize/docker_run.sh', + 'OUTPUT_DIR': 'artifacts'} + jobspec = jobset.JobSpec( + cmdline=['tools/run_tests/dockerize/build_and_run_docker.sh'] + docker_args, + environ=docker_env, + shortname='build_package.%s' % (name), + timeout_seconds=30*60, + flake_retries=flake_retries, + timeout_retries=timeout_retries) + return jobspec + +def create_jobspec(name, cmdline, environ=None, cwd=None, shell=False, + flake_retries=0, timeout_retries=0): + """Creates jobspec.""" + jobspec = jobset.JobSpec( + cmdline=cmdline, + environ=environ, + cwd=cwd, + shortname='build_package.%s' % (name), + timeout_seconds=10*60, + flake_retries=flake_retries, + timeout_retries=timeout_retries, + shell=shell) + return jobspec + + +class CSharpPackage: + """Builds C# nuget packages.""" + + def __init__(self, linux=False): + self.linux = linux + self.labels = ['package', 'csharp'] + if linux: + self.name = 'csharp_package_dotnetcli_linux' + self.labels += ['linux'] + else: + self.name = 'csharp_package_dotnetcli_windows' + self.labels += ['windows'] + + def pre_build_jobspecs(self): + return [] + + def build_jobspec(self): + if self.linux: + return create_docker_jobspec( + self.name, + 'tools/dockerfile/test/csharp_coreclr_x64', + 'src/csharp/build_packages_dotnetcli.sh') + else: + return create_jobspec(self.name, + ['build_packages_dotnetcli.bat'], + cwd='src\\csharp', + shell=True) + + def __str__(self): + return self.name + + +class NodePackage: + """Builds Node NPM package and collects precompiled binaries""" + + def __init__(self): + self.name = 'node_package' + self.labels = ['package', 'node', 'linux'] + + def pre_build_jobspecs(self): + return [] + + def build_jobspec(self): + return create_docker_jobspec( + self.name, + 'tools/dockerfile/grpc_artifact_linux_x64', + 'tools/run_tests/artifacts/build_package_node.sh') + + +class RubyPackage: + """Collects ruby gems created in the artifact phase""" + + def __init__(self): + self.name = 'ruby_package' + self.labels = ['package', 'ruby', 'linux'] + + def pre_build_jobspecs(self): + return [] + + def build_jobspec(self): + return create_docker_jobspec( + self.name, + 'tools/dockerfile/grpc_artifact_linux_x64', + 'tools/run_tests/artifacts/build_package_ruby.sh') + + +class PythonPackage: + """Collects python eggs and wheels created in the artifact phase""" + + def __init__(self): + self.name = 'python_package' + self.labels = ['package', 'python', 'linux'] + + def pre_build_jobspecs(self): + return [] + + def build_jobspec(self): + return create_docker_jobspec( + self.name, + 'tools/dockerfile/grpc_artifact_linux_x64', + 'tools/run_tests/artifacts/build_package_python.sh') + + +class PHPPackage: + """Copy PHP PECL package artifact""" + + def __init__(self): + self.name = 'php_package' + self.labels = ['package', 'php', 'linux'] + + def pre_build_jobspecs(self): + return [] + + def build_jobspec(self): + return create_docker_jobspec( + self.name, + 'tools/dockerfile/grpc_artifact_linux_x64', + 'tools/run_tests/artifacts/build_package_php.sh') + + +def targets(): + """Gets list of supported targets""" + return [CSharpPackage(), + CSharpPackage(linux=True), + NodePackage(), + RubyPackage(), + PythonPackage(), + PHPPackage()] |