diff options
Diffstat (limited to 'tools/run_tests')
22 files changed, 147 insertions, 70 deletions
diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index c90da066c3..5a71b96f9e 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -45,6 +45,7 @@ def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={}, """Creates jobspec for a task running under docker.""" environ = environ.copy() environ['RUN_COMMAND'] = shell_command + environ['ARTIFACTS_OUT'] = 'artifacts/%s' % name docker_args=[] for k,v in environ.items(): @@ -65,9 +66,20 @@ def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={}, return jobspec -def create_jobspec(name, cmdline, environ=None, shell=False, - flake_retries=0, timeout_retries=0, timeout_seconds=30*60): +def create_jobspec(name, cmdline, environ={}, shell=False, + flake_retries=0, timeout_retries=0, timeout_seconds=30*60, + use_workspace=False, + cpu_cost=1.0): """Creates jobspec.""" + environ = environ.copy() + if use_workspace: + environ['WORKSPACE_NAME'] = 'workspace_%s' % name + environ['ARTIFACTS_OUT'] = os.path.join('..', 'artifacts', name) + cmdline = ['bash', + 'tools/run_tests/artifacts/run_in_workspace.sh'] + cmdline + else: + environ['ARTIFACTS_OUT'] = os.path.join('artifacts', name) + jobspec = jobset.JobSpec( cmdline=cmdline, environ=environ, @@ -75,7 +87,8 @@ def create_jobspec(name, cmdline, environ=None, shell=False, timeout_seconds=timeout_seconds, flake_retries=flake_retries, timeout_retries=timeout_retries, - shell=shell) + shell=shell, + cpu_cost=cpu_cost) return jobspec @@ -137,13 +150,14 @@ class PythonArtifact: dir ], environ=environ, - shell=True) + use_workspace=True) else: environ['PYTHON'] = self.py_version environ['SKIP_PIP_INSTALL'] = 'TRUE' return create_jobspec(self.name, ['tools/run_tests/artifacts/build_artifact_python.sh'], - environ=environ) + environ=environ, + use_workspace=True) def __str__(self): return self.name @@ -162,20 +176,11 @@ class RubyArtifact: return [] def build_jobspec(self): - if self.platform == 'windows': - raise Exception("Not supported yet") - else: - if self.platform == 'linux': - environ = {} - if self.arch == 'x86': - environ['SETARCH_CMD'] = 'linux32' - return create_docker_jobspec(self.name, - 'tools/dockerfile/grpc_artifact_linux_%s' % self.arch, - 'tools/run_tests/artifacts/build_artifact_ruby.sh', - environ=environ) - else: - return create_jobspec(self.name, - ['tools/run_tests/artifacts/build_artifact_ruby.sh']) + # Ruby build uses docker internally and docker cannot be nested. + # We are using a custom workspace instead. + return create_jobspec(self.name, + ['tools/run_tests/artifacts/build_artifact_ruby.sh'], + use_workspace=True) class CSharpExtArtifact: @@ -196,7 +201,7 @@ class CSharpExtArtifact: return create_jobspec(self.name, ['tools\\run_tests\\artifacts\\build_artifact_csharp.bat', cmake_arch_option], - shell=True) + use_workspace=True) else: environ = {'CONFIG': 'opt', 'EMBED_OPENSSL': 'true', @@ -216,7 +221,8 @@ class CSharpExtArtifact: environ['LDFLAGS'] += ' %s' % archflag return create_jobspec(self.name, ['tools/run_tests/artifacts/build_artifact_csharp.sh'], - environ=environ) + environ=environ, + use_workspace=True) def __str__(self): return self.name @@ -242,10 +248,15 @@ class NodeExtArtifact: def build_jobspec(self): if self.platform == 'windows': + # Simultaneous builds of node on the same windows machine are flaky. + # Set x86 build as exclusive to make sure there is only one node build + # at a time. See https://github.com/grpc/grpc/issues/8293 + cpu_cost = 1e6 if self.arch != 'x64' else 1.0 return create_jobspec(self.name, ['tools\\run_tests\\artifacts\\build_artifact_node.bat', self.gyp_arch], - shell=True) + use_workspace=True, + cpu_cost=cpu_cost) else: if self.platform == 'linux': return create_docker_jobspec( @@ -255,7 +266,8 @@ class NodeExtArtifact: else: return create_jobspec(self.name, ['tools/run_tests/artifacts/build_artifact_node.sh', - self.gyp_arch]) + self.gyp_arch], + use_workspace=True) class PHPArtifact: """Builds PHP PECL package""" @@ -277,7 +289,8 @@ class PHPArtifact: 'tools/run_tests/artifacts/build_artifact_php.sh') else: return create_jobspec(self.name, - ['tools/run_tests/artifacts/build_artifact_php.sh']) + ['tools/run_tests/artifacts/build_artifact_php.sh'], + use_workspace=True) class ProtocArtifact: """Builds protoc and protoc-plugin artifacts""" @@ -310,14 +323,16 @@ class ProtocArtifact: environ['CXXFLAGS'] += ' -std=c++11 -stdlib=libc++ %s' % _MACOS_COMPAT_FLAG return create_jobspec(self.name, ['tools/run_tests/artifacts/build_artifact_protoc.sh'], - environ=environ) + environ=environ, + use_workspace=True) else: generator = 'Visual Studio 12 Win64' if self.arch == 'x64' else 'Visual Studio 12' vcplatform = 'x64' if self.arch == 'x64' else 'Win32' return create_jobspec(self.name, ['tools\\run_tests\\artifacts\\build_artifact_protoc.bat'], environ={'generator': generator, - 'Platform': vcplatform}) + 'Platform': vcplatform}, + use_workspace=True) def __str__(self): return self.name @@ -351,7 +366,6 @@ def targets(): PythonArtifact('windows', 'x64', 'Python34'), PythonArtifact('windows', 'x64', 'Python35'), PythonArtifact('windows', 'x64', 'Python36'), - RubyArtifact('linux', 'x86'), RubyArtifact('linux', 'x64'), RubyArtifact('macos', 'x64'), PHPArtifact('linux', 'x64'), diff --git a/tools/run_tests/artifacts/build_artifact_csharp.bat b/tools/run_tests/artifacts/build_artifact_csharp.bat index f84ebc5a35..ebb11bf909 100644 --- a/tools/run_tests/artifacts/build_artifact_csharp.bat +++ b/tools/run_tests/artifacts/build_artifact_csharp.bat @@ -37,8 +37,8 @@ cd cmake\build\%ARCHITECTURE% cmake --build . --target grpc_csharp_ext --config Release cd ..\..\.. -mkdir artifacts -copy /Y cmake\build\Win32\Release\grpc_csharp_ext.dll artifacts || copy /Y cmake\build\x64\Release\grpc_csharp_ext.dll artifacts || goto :error +mkdir -p %ARTIFACTS_OUT% +copy /Y cmake\build\Win32\Release\grpc_csharp_ext.dll %ARTIFACTS_OUT% || copy /Y cmake\build\x64\Release\grpc_csharp_ext.dll %ARTIFACTS_OUT% || goto :error goto :EOF diff --git a/tools/run_tests/artifacts/build_artifact_csharp.sh b/tools/run_tests/artifacts/build_artifact_csharp.sh index aed04b2745..2bfa749fa3 100755 --- a/tools/run_tests/artifacts/build_artifact_csharp.sh +++ b/tools/run_tests/artifacts/build_artifact_csharp.sh @@ -34,5 +34,5 @@ cd $(dirname $0)/../../.. make grpc_csharp_ext -mkdir -p artifacts -cp libs/opt/libgrpc_csharp_ext.so artifacts || cp libs/opt/libgrpc_csharp_ext.dylib artifacts +mkdir -p "${ARTIFACTS_OUT}" +cp libs/opt/libgrpc_csharp_ext.so "${ARTIFACTS_OUT}" || cp libs/opt/libgrpc_csharp_ext.dylib "${ARTIFACTS_OUT}" diff --git a/tools/run_tests/artifacts/build_artifact_node.bat b/tools/run_tests/artifacts/build_artifact_node.bat index bfd4461f72..da4d479a8c 100644 --- a/tools/run_tests/artifacts/build_artifact_node.bat +++ b/tools/run_tests/artifacts/build_artifact_node.bat @@ -37,7 +37,7 @@ del /f /q BUILD || rmdir build /s /q call npm update || goto :error -mkdir artifacts +mkdir -p %ARTIFACTS_OUT% for %%v in (%node_versions%) do ( call .\node_modules\.bin\node-pre-gyp.cmd configure build --target=%%v --target_arch=%1 @@ -47,13 +47,13 @@ for %%v in (%node_versions%) do ( rmdir "%HOMEDRIVE%%HOMEPATH%\.node-gyp\iojs-%%v\include\node\openssl" /S /Q call .\node_modules\.bin\node-pre-gyp.cmd build package testpackage --target=%%v --target_arch=%1 || goto :error - xcopy /Y /I /S build\stage\* artifacts\ || goto :error + xcopy /Y /I /S build\stage\* %ARTIFACTS_OUT%\ || goto :error ) for %%v in (%electron_versions%) do ( cmd /V /C "set "HOME=%HOMEDRIVE%%HOMEPATH%\electron-gyp" && call .\node_modules\.bin\node-pre-gyp.cmd configure rebuild package testpackage --runtime=electron --target=%%v --target_arch=%1 --disturl=https://atom.io/download/electron" || goto :error - xcopy /Y /I /S build\stage\* artifacts\ || goto :error + xcopy /Y /I /S build\stage\* %ARTIFACTS_OUT%\ || goto :error ) if %errorlevel% neq 0 exit /b %errorlevel% diff --git a/tools/run_tests/artifacts/build_artifact_node.sh b/tools/run_tests/artifacts/build_artifact_node.sh index 7a747551e8..3947bded6f 100755 --- a/tools/run_tests/artifacts/build_artifact_node.sh +++ b/tools/run_tests/artifacts/build_artifact_node.sh @@ -38,7 +38,7 @@ cd $(dirname $0)/../../.. rm -rf build || true -mkdir -p artifacts +mkdir -p "${ARTIFACTS_OUT}" npm update @@ -49,11 +49,11 @@ electron_versions=( 1.0.0 1.1.0 1.2.0 1.3.0 1.4.0 1.5.0 1.6.0 ) for version in ${node_versions[@]} do ./node_modules/.bin/node-pre-gyp configure rebuild package testpackage --target=$version --target_arch=$NODE_TARGET_ARCH --grpc_alpine=true - cp -r build/stage/* artifacts/ + cp -r build/stage/* "${ARTIFACTS_OUT}"/ done for version in ${electron_versions[@]} do HOME=~/.electron-gyp ./node_modules/.bin/node-pre-gyp configure rebuild package testpackage --runtime=electron --target=$version --target_arch=$NODE_TARGET_ARCH --disturl=https://atom.io/download/electron - cp -r build/stage/* artifacts/ + cp -r build/stage/* "${ARTIFACTS_OUT}"/ done diff --git a/tools/run_tests/artifacts/build_artifact_php.sh b/tools/run_tests/artifacts/build_artifact_php.sh index c8d55860c1..d7f8c8f028 100755 --- a/tools/run_tests/artifacts/build_artifact_php.sh +++ b/tools/run_tests/artifacts/build_artifact_php.sh @@ -33,8 +33,8 @@ set -ex cd $(dirname $0)/../../.. -mkdir -p artifacts +mkdir -p "${ARTIFACTS_OUT}" pear package -cp -r grpc-*.tgz artifacts/ +cp -r grpc-*.tgz "${ARTIFACTS_OUT}"/ diff --git a/tools/run_tests/artifacts/build_artifact_protoc.bat b/tools/run_tests/artifacts/build_artifact_protoc.bat index b2bf86da40..def64bdfee 100644 --- a/tools/run_tests/artifacts/build_artifact_protoc.bat +++ b/tools/run_tests/artifacts/build_artifact_protoc.bat @@ -27,7 +27,7 @@ @rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE @rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -mkdir artifacts +mkdir -p %ARTIFACTS_OUT% setlocal cd third_party/protobuf/cmake @@ -39,8 +39,8 @@ endlocal call vsprojects/build_plugins.bat || goto :error -xcopy /Y third_party\protobuf\cmake\build\solution\Release\protoc.exe artifacts\ || goto :error -xcopy /Y vsprojects\Release\*_plugin.exe artifacts\ || xcopy /Y vsprojects\x64\Release\*_plugin.exe artifacts\ || goto :error +xcopy /Y third_party\protobuf\cmake\build\solution\Release\protoc.exe %ARTIFACTS_OUT%\ || goto :error +xcopy /Y vsprojects\Release\*_plugin.exe %ARTIFACTS_OUT%\ || xcopy /Y vsprojects\x64\Release\*_plugin.exe %ARTIFACTS_OUT%\ || goto :error goto :EOF diff --git a/tools/run_tests/artifacts/build_artifact_protoc.sh b/tools/run_tests/artifacts/build_artifact_protoc.sh index 26c2280eff..df78203739 100755 --- a/tools/run_tests/artifacts/build_artifact_protoc.sh +++ b/tools/run_tests/artifacts/build_artifact_protoc.sh @@ -37,5 +37,5 @@ cd $(dirname $0)/../../.. make plugins -mkdir -p artifacts -cp bins/opt/protobuf/protoc bins/opt/*_plugin artifacts/ +mkdir -p "${ARTIFACTS_OUT}" +cp bins/opt/protobuf/protoc bins/opt/*_plugin "${ARTIFACTS_OUT}"/ diff --git a/tools/run_tests/artifacts/build_artifact_python.bat b/tools/run_tests/artifacts/build_artifact_python.bat index 246713a6ce..860b9e831d 100644 --- a/tools/run_tests/artifacts/build_artifact_python.bat +++ b/tools/run_tests/artifacts/build_artifact_python.bat @@ -38,8 +38,9 @@ set GRPC_PYTHON_BUILD_WITH_CYTHON=1 @rem Multiple builds are running simultaneously, so to avoid distutils @rem file collisions, we build everything in a tmp directory -if not exist "artifacts" mkdir "artifacts" -set ARTIFACT_DIR=%cd%\artifacts +@rem TODO(jtattermusch): it doesn't look like builds are actually running in parallel in the same dir +mkdir -p %ARTIFACTS_OUT% +set ARTIFACT_DIR=%cd%\%ARTIFACTS_OUT% set BUILD_DIR=C:\Windows\Temp\pygrpc-%3\ mkdir %BUILD_DIR% xcopy /s/e/q %cd%\* %BUILD_DIR% diff --git a/tools/run_tests/artifacts/build_artifact_python.sh b/tools/run_tests/artifacts/build_artifact_python.sh index 5a5506029a..a1f0a5857a 100755 --- a/tools/run_tests/artifacts/build_artifact_python.sh +++ b/tools/run_tests/artifacts/build_artifact_python.sh @@ -41,8 +41,9 @@ export AUDITWHEEL=${AUDITWHEEL:-auditwheel} # Because multiple builds run in parallel, some distutils file # operations may collide. To avoid this, each build is run in # a temp directory -mkdir -p artifacts -ARTIFACT_DIR="$PWD/artifacts" +# TODO(jtattermusch): it doesn't look like builds are actually running in parallel in the same dir +mkdir -p "${ARTIFACTS_OUT}" +ARTIFACT_DIR="$PWD/${ARTIFACTS_OUT}" BUILD_DIR=`mktemp -d "${TMPDIR:-/tmp}/pygrpc.XXXXXX"` trap "rm -rf $BUILD_DIR" EXIT cp -r * "$BUILD_DIR" diff --git a/tools/run_tests/artifacts/build_artifact_ruby.sh b/tools/run_tests/artifacts/build_artifact_ruby.sh index ca461ba561..c92d7a8a89 100755 --- a/tools/run_tests/artifacts/build_artifact_ruby.sh +++ b/tools/run_tests/artifacts/build_artifact_ruby.sh @@ -63,6 +63,6 @@ if [ "$SYSTEM" == "Darwin" ] ; then rm `ls pkg/*.gem | grep -v darwin` fi -mkdir -p artifacts +mkdir -p "${ARTIFACTS_OUT}" -cp pkg/*.gem artifacts +cp pkg/*.gem "${ARTIFACTS_OUT}"/ diff --git a/tools/run_tests/artifacts/build_package_node.sh b/tools/run_tests/artifacts/build_package_node.sh index 8b5e8c0bc1..d06f141869 100755 --- a/tools/run_tests/artifacts/build_package_node.sh +++ b/tools/run_tests/artifacts/build_package_node.sh @@ -40,7 +40,7 @@ base=$(pwd) artifacts=$base/artifacts mkdir -p $artifacts -cp -r $EXTERNAL_GIT_ROOT/architecture={x86,x64},language=node,platform={windows,linux,macos}/artifacts/* $artifacts/ || true +cp -r $EXTERNAL_GIT_ROOT/platform={windows,linux,macos}/artifacts/node_ext_*/* $artifacts/ || true npm update npm pack @@ -86,7 +86,7 @@ for arch in {x86,x64}; do ;; esac rm -r bin/* - input_dir="$EXTERNAL_GIT_ROOT/architecture=$arch,language=protoc,platform=$plat/artifacts" + input_dir="$EXTERNAL_GIT_ROOT/platform=${plat}/artifacts/protoc_${plat}_${arch}" cp $input_dir/protoc* bin/ cp $input_dir/grpc_node_plugin* bin/ mkdir -p bin/google/protobuf diff --git a/tools/run_tests/artifacts/build_package_php.sh b/tools/run_tests/artifacts/build_package_php.sh index 42a8d9f8df..16da58d065 100755 --- a/tools/run_tests/artifacts/build_package_php.sh +++ b/tools/run_tests/artifacts/build_package_php.sh @@ -33,4 +33,4 @@ set -ex cd $(dirname $0)/../../.. mkdir -p artifacts/ -cp -r $EXTERNAL_GIT_ROOT/architecture={x86,x64},language=php,platform={windows,linux,macos}/artifacts/* artifacts/ || true +cp -r $EXTERNAL_GIT_ROOT/platform={windows,linux,macos}/artifacts/php_*/* artifacts/ || true diff --git a/tools/run_tests/artifacts/build_package_python.sh b/tools/run_tests/artifacts/build_package_python.sh index 4a1c15ceee..9148bb0e59 100755 --- a/tools/run_tests/artifacts/build_package_python.sh +++ b/tools/run_tests/artifacts/build_package_python.sh @@ -36,7 +36,7 @@ mkdir -p artifacts/ # All the python packages have been built in the artifact phase already # and we only collect them here to deliver them to the distribtest phase. -cp -r $EXTERNAL_GIT_ROOT/architecture={x86,x64},language=python,platform={windows,linux,macos}/artifacts/* artifacts/ || true +cp -r $EXTERNAL_GIT_ROOT/platform={windows,linux,macos}/artifacts/python_*/* artifacts/ || true # TODO: all the artifact builder configurations generate a grpcio-VERSION.tar.gz # source distribution package, and only one of them will end up diff --git a/tools/run_tests/artifacts/build_package_ruby.sh b/tools/run_tests/artifacts/build_package_ruby.sh index b4d20d8a4c..8388b84cc5 100755 --- a/tools/run_tests/artifacts/build_package_ruby.sh +++ b/tools/run_tests/artifacts/build_package_ruby.sh @@ -38,7 +38,7 @@ mkdir -p artifacts/ # All the ruby packages have been built in the artifact phase already # and we only collect them here to deliver them to the distribtest phase. -cp -r $EXTERNAL_GIT_ROOT/architecture={x86,x64},language=ruby,platform={windows,linux,macos}/artifacts/* artifacts/ || true +cp -r $EXTERNAL_GIT_ROOT/platform={windows,linux,macos}/artifacts/ruby_native_gem_*/* artifacts/ || true well_known_protos=( any api compiler/plugin descriptor duration empty field_mask source_context struct timestamp type wrappers ) @@ -56,7 +56,7 @@ for arch in {x86,x64}; do ;; esac for plat in {windows,linux,macos}; do - input_dir="$EXTERNAL_GIT_ROOT/architecture=$arch,language=protoc,platform=$plat/artifacts" + input_dir="$EXTERNAL_GIT_ROOT/platform=${plat}/artifacts/protoc_${plat}_${arch}" output_dir="$base/src/ruby/tools/bin/${ruby_arch}-${plat}" mkdir -p $output_dir/google/protobuf mkdir -p $output_dir/google/protobuf/compiler # needed for plugin.proto diff --git a/tools/run_tests/artifacts/distribtest_targets.py b/tools/run_tests/artifacts/distribtest_targets.py index 097fd2d8b5..3cb7d97b6d 100644 --- a/tools/run_tests/artifacts/distribtest_targets.py +++ b/tools/run_tests/artifacts/distribtest_targets.py @@ -63,8 +63,14 @@ def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={}, def create_jobspec(name, cmdline, environ=None, shell=False, - flake_retries=0, timeout_retries=0): + flake_retries=0, timeout_retries=0, + use_workspace=False): """Creates jobspec.""" + environ = environ.copy() + if use_workspace: + environ['WORKSPACE_NAME'] = 'workspace_%s' % name + cmdline = ['bash', + 'tools/run_tests/artifacts/run_in_workspace.sh'] + cmdline jobspec = jobset.JobSpec( cmdline=cmdline, environ=environ, @@ -80,7 +86,7 @@ class CSharpDistribTest(object): """Tests C# NuGet package""" def __init__(self, platform, arch, docker_suffix=None, use_dotnet_cli=False): - self.name = 'csharp_nuget_%s_%s' % (platform, arch) + self.name = 'csharp_%s_%s' % (platform, arch) self.platform = platform self.arch = arch self.docker_suffix = docker_suffix @@ -110,16 +116,18 @@ class CSharpDistribTest(object): elif self.platform == 'macos': return create_jobspec(self.name, ['test/distrib/csharp/run_distrib_test%s.sh' % self.script_suffix], - environ={'EXTERNAL_GIT_ROOT': '../../..'}) + environ={'EXTERNAL_GIT_ROOT': '../../../..'}, + use_workspace=True) elif self.platform == 'windows': if self.arch == 'x64': environ={'MSBUILD_EXTRA_ARGS': '/p:Platform=x64', 'DISTRIBTEST_OUTPATH': 'DistribTest\\bin\\x64\\Debug'} else: - environ={'DISTRIBTEST_OUTPATH': 'DistribTest\\bin\\\Debug'} + environ={'DISTRIBTEST_OUTPATH': 'DistribTest\\bin\\Debug'} return create_jobspec(self.name, ['test\\distrib\\csharp\\run_distrib_test%s.bat' % self.script_suffix], - environ=environ) + environ=environ, + use_workspace=True) else: raise Exception("Not supported yet.") @@ -161,7 +169,8 @@ class NodeDistribTest(object): return create_jobspec(self.name, ['test/distrib/node/run_distrib_test.sh', str(self.node_version)], - environ={'EXTERNAL_GIT_ROOT': '../../..'}) + environ={'EXTERNAL_GIT_ROOT': '../../../..'}, + use_workspace=True) else: raise Exception("Not supported yet.") @@ -249,7 +258,8 @@ class PHPDistribTest(object): elif self.platform == 'macos': return create_jobspec(self.name, ['test/distrib/php/run_distrib_test.sh'], - environ={'EXTERNAL_GIT_ROOT': '../../..'}) + environ={'EXTERNAL_GIT_ROOT': '../../../..'}, + use_workspace=True) else: raise Exception("Not supported yet.") @@ -339,5 +349,5 @@ def targets(): NodeDistribTest('linux', 'x64', os, version) for os in ('wheezy', 'jessie', 'ubuntu1204', 'ubuntu1404', 'ubuntu1504', 'ubuntu1510', 'ubuntu1604') - for version in ('0.12', '3', '4', '5') + for version in ('4', '5') ] diff --git a/tools/run_tests/artifacts/run_in_workspace.sh b/tools/run_tests/artifacts/run_in_workspace.sh new file mode 100755 index 0000000000..ed3bfda8c0 --- /dev/null +++ b/tools/run_tests/artifacts/run_in_workspace.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Create a workspace in a subdirectory to allow running multiple builds in isolation. +# WORKSPACE_NAME env variable needs to contain name of the workspace to create. +# All cmdline args will be executed as a command. +set -ex + +cd $(dirname $0)/../../.. +export repo_root=$(pwd) + +rm -rf "${WORKSPACE_NAME}" +git clone . "${WORKSPACE_NAME}" +# clone gRPC submodules, use data from locally cloned submodules where possible +git submodule foreach 'cd "${repo_root}/${WORKSPACE_NAME}" \ + && git submodule update --init --reference ${repo_root}/${name} ${name}' + +echo "Running in workspace ${WORKSPACE_NAME}" +cd ${WORKSPACE_NAME} +$@ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 8a5a2887cc..ce65f1d48a 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -7620,7 +7620,6 @@ "include/grpc/support/tls_msvc.h", "include/grpc/support/tls_pthread.h", "include/grpc/support/useful.h", - "include/grpc/support/workaround_list.h", "src/core/lib/profiling/timers.h", "src/core/lib/support/arena.h", "src/core/lib/support/atomic.h", @@ -7670,7 +7669,6 @@ "include/grpc/support/tls_msvc.h", "include/grpc/support/tls_pthread.h", "include/grpc/support/useful.h", - "include/grpc/support/workaround_list.h", "src/core/lib/profiling/basic_timers.c", "src/core/lib/profiling/stap_timers.c", "src/core/lib/profiling/timers.h", @@ -7790,6 +7788,7 @@ "include/grpc/slice.h", "include/grpc/slice_buffer.h", "include/grpc/status.h", + "include/grpc/support/workaround_list.h", "src/core/lib/channel/channel_args.h", "src/core/lib/channel/channel_stack.h", "src/core/lib/channel/channel_stack_builder.h", @@ -7917,6 +7916,7 @@ "include/grpc/slice.h", "include/grpc/slice_buffer.h", "include/grpc/status.h", + "include/grpc/support/workaround_list.h", "src/core/lib/channel/channel_args.c", "src/core/lib/channel/channel_args.h", "src/core/lib/channel/channel_stack.c", diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 2c8e4ebae7..d1e3a99a08 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -2805,7 +2805,7 @@ }, { "args": [ - "--benchmark_min_time=0" + "--benchmark_min_time=4" ], "ci_platforms": [ "linux", diff --git a/tools/run_tests/helper_scripts/build_python.sh b/tools/run_tests/helper_scripts/build_python.sh index 28397be13e..6ad285cdb9 100755 --- a/tools/run_tests/helper_scripts/build_python.sh +++ b/tools/run_tests/helper_scripts/build_python.sh @@ -187,7 +187,8 @@ $VENV_PYTHON $ROOT/src/python/grpcio_reflection/setup.py build_package_protos pip_install_dir $ROOT/src/python/grpcio_reflection # Build/install tests -$VENV_PYTHON -m pip install coverage oauth2client +$VENV_PYTHON -m pip install coverage==4.4 oauth2client==4.1.0 \ + google-auth==1.0.0 requests==2.14.2 $VENV_PYTHON $ROOT/src/python/grpcio_tests/setup.py preprocess $VENV_PYTHON $ROOT/src/python/grpcio_tests/setup.py build_package_protos pip_install_dir $ROOT/src/python/grpcio_tests diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 867d9e6f7b..ae2da26e1f 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -581,7 +581,7 @@ def auth_options(language, test_case): env = {} # TODO(jtattermusch): this file path only works inside docker - key_filepath = '/root/service_account/stubbyCloudTestingTest-ee3fce360ac5.json' + key_filepath = '/root/service_account/GrpcTesting-726eb1347f15.json' oauth_scope_arg = '--oauth_scope=https://www.googleapis.com/auth/xapi.zoo' key_file_arg = '--service_account_key_file=%s' % key_filepath default_account_arg = '--default_service_account=830293263384-compute@developer.gserviceaccount.com' diff --git a/tools/run_tests/task_runner.py b/tools/run_tests/task_runner.py index 0ec7efbbee..7feac29673 100755 --- a/tools/run_tests/task_runner.py +++ b/tools/run_tests/task_runner.py @@ -40,6 +40,7 @@ import artifacts.artifact_targets as artifact_targets import artifacts.distribtest_targets as distribtest_targets import artifacts.package_targets as package_targets import python_utils.jobset as jobset +import python_utils.report_utils as report_utils _TARGETS = [] _TARGETS += artifact_targets.targets() @@ -116,8 +117,10 @@ if not build_jobs: sys.exit(1) jobset.message('START', 'Building targets.', do_newline=True) -num_failures, _ = jobset.run( +num_failures, resultset = jobset.run( build_jobs, newline_on_success=True, maxjobs=args.jobs) +report_utils.render_junit_xml_report(resultset, 'report_taskrunner_sponge_log.xml', + suite_name='tasks') if num_failures == 0: jobset.message('SUCCESS', 'All targets built successfully.', do_newline=True) |