diff options
author | Jan Tattermusch <jtattermusch@users.noreply.github.com> | 2016-12-22 10:49:21 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2016-12-22 10:49:21 +0100 |
commit | 276f390480a4ecb90909e9ebd7ef581dea3d480a (patch) | |
tree | 2a5cb418a5485dbd0677212fb45394387846ecff | |
parent | d7c923d40bc609d442130e57bf1177fcc3bbcf80 (diff) | |
parent | 5c79a3199c1b47d1a2d3ff4dc5636c9c90ca2a7f (diff) |
Merge pull request #9176 from jtattermusch/runtests_refactor
Clean up tools/run_tests directory
-rw-r--r-- | .gitignore | 2 | ||||
-rw-r--r-- | templates/tools/run_tests/generated/configs.json.template (renamed from templates/tools/run_tests/configs.json.template) | 0 | ||||
-rw-r--r-- | templates/tools/run_tests/generated/sources_and_headers.json.template (renamed from templates/tools/run_tests/sources_and_headers.json.template) | 0 | ||||
-rw-r--r-- | templates/tools/run_tests/generated/tests.json.template (renamed from templates/tools/run_tests/tests.json.template) | 0 | ||||
-rwxr-xr-x | tools/buildgen/generate_projects.py | 2 | ||||
-rw-r--r--[-rwxr-xr-x] | tools/run_tests/artifacts/__init__.py (renamed from tools/run_tests/prepare_travis.sh) | 41 | ||||
-rw-r--r-- | tools/run_tests/artifacts/artifact_targets.py (renamed from tools/run_tests/artifact_targets.py) | 37 | ||||
-rw-r--r-- | tools/run_tests/artifacts/build_artifact_csharp.bat (renamed from tools/run_tests/build_artifact_csharp.bat) | 0 | ||||
-rwxr-xr-x | tools/run_tests/artifacts/build_artifact_csharp.sh (renamed from tools/run_tests/build_artifact_csharp.sh) | 2 | ||||
-rw-r--r-- | tools/run_tests/artifacts/build_artifact_node.bat (renamed from tools/run_tests/build_artifact_node.bat) | 0 | ||||
-rwxr-xr-x | tools/run_tests/artifacts/build_artifact_node.sh (renamed from tools/run_tests/build_artifact_node.sh) | 2 | ||||
-rwxr-xr-x | tools/run_tests/artifacts/build_artifact_php.sh (renamed from tools/run_tests/build_artifact_php.sh) | 2 | ||||
-rw-r--r-- | tools/run_tests/artifacts/build_artifact_protoc.bat (renamed from tools/run_tests/build_artifact_protoc.bat) | 2 | ||||
-rwxr-xr-x | tools/run_tests/artifacts/build_artifact_protoc.sh (renamed from tools/run_tests/build_artifact_protoc.sh) | 2 | ||||
-rw-r--r-- | tools/run_tests/artifacts/build_artifact_python.bat (renamed from tools/run_tests/build_artifact_python.bat) | 0 | ||||
-rwxr-xr-x | tools/run_tests/artifacts/build_artifact_python.sh (renamed from tools/run_tests/build_artifact_python.sh) | 2 | ||||
-rwxr-xr-x | tools/run_tests/artifacts/build_artifact_ruby.sh (renamed from tools/run_tests/build_artifact_ruby.sh) | 2 | ||||
-rwxr-xr-x | tools/run_tests/artifacts/build_package_node.sh (renamed from tools/run_tests/build_package_node.sh) | 2 | ||||
-rwxr-xr-x | tools/run_tests/artifacts/build_package_php.sh (renamed from tools/run_tests/build_package_php.sh) | 2 | ||||
-rwxr-xr-x | tools/run_tests/artifacts/build_package_python.sh (renamed from tools/run_tests/build_package_python.sh) | 2 | ||||
-rwxr-xr-x | tools/run_tests/artifacts/build_package_ruby.sh (renamed from tools/run_tests/build_package_ruby.sh) | 2 | ||||
-rw-r--r-- | tools/run_tests/artifacts/distribtest_targets.py (renamed from tools/run_tests/distribtest_targets.py) | 6 | ||||
-rw-r--r-- | tools/run_tests/artifacts/package_targets.py (renamed from tools/run_tests/package_targets.py) | 15 | ||||
-rw-r--r-- | tools/run_tests/build_stats/build_stats_schema.json (renamed from tools/run_tests/build_stats_schema.json) | 0 | ||||
-rw-r--r-- | tools/run_tests/build_stats/build_stats_schema_no_matrix.json (renamed from tools/run_tests/build_stats_schema_no_matrix.json) | 0 | ||||
-rw-r--r-- | tools/run_tests/generated/configs.json (renamed from tools/run_tests/configs.json) | 0 | ||||
-rw-r--r-- | tools/run_tests/generated/sources_and_headers.json (renamed from tools/run_tests/sources_and_headers.json) | 0 | ||||
-rw-r--r-- | tools/run_tests/generated/tests.json (renamed from tools/run_tests/tests.json) | 0 | ||||
-rwxr-xr-x | tools/run_tests/helper_scripts/build_csharp.sh (renamed from tools/run_tests/build_csharp.sh) | 2 | ||||
-rw-r--r-- | tools/run_tests/helper_scripts/build_csharp_coreclr.bat (renamed from tools/run_tests/build_csharp_coreclr.bat) | 2 | ||||
-rwxr-xr-x | tools/run_tests/helper_scripts/build_csharp_coreclr.sh (renamed from tools/run_tests/build_csharp_coreclr.sh) | 2 | ||||
-rw-r--r-- | tools/run_tests/helper_scripts/build_node.bat (renamed from tools/run_tests/build_node.bat) | 0 | ||||
-rwxr-xr-x | tools/run_tests/helper_scripts/build_node.sh (renamed from tools/run_tests/build_node.sh) | 2 | ||||
-rwxr-xr-x | tools/run_tests/helper_scripts/build_php.sh (renamed from tools/run_tests/build_php.sh) | 2 | ||||
-rwxr-xr-x | tools/run_tests/helper_scripts/build_python.sh (renamed from tools/run_tests/build_python.sh) | 2 | ||||
-rw-r--r-- | tools/run_tests/helper_scripts/build_python_msys2.sh (renamed from tools/run_tests/build_python_msys2.sh) | 0 | ||||
-rwxr-xr-x | tools/run_tests/helper_scripts/build_ruby.sh (renamed from tools/run_tests/build_ruby.sh) | 2 | ||||
-rwxr-xr-x | tools/run_tests/helper_scripts/post_tests_c.sh (renamed from tools/run_tests/post_tests_c.sh) | 2 | ||||
-rw-r--r-- | tools/run_tests/helper_scripts/post_tests_csharp.bat (renamed from tools/run_tests/post_tests_csharp.bat) | 2 | ||||
-rwxr-xr-x | tools/run_tests/helper_scripts/post_tests_csharp.sh (renamed from tools/run_tests/post_tests_csharp.sh) | 2 | ||||
-rwxr-xr-x | tools/run_tests/helper_scripts/post_tests_php.sh (renamed from tools/run_tests/post_tests_php.sh) | 2 | ||||
-rwxr-xr-x | tools/run_tests/helper_scripts/post_tests_ruby.sh (renamed from tools/run_tests/post_tests_ruby.sh) | 2 | ||||
-rw-r--r-- | tools/run_tests/helper_scripts/pre_build_c.bat (renamed from tools/run_tests/pre_build_c.bat) | 2 | ||||
-rw-r--r-- | tools/run_tests/helper_scripts/pre_build_csharp.bat (renamed from tools/run_tests/pre_build_csharp.bat) | 2 | ||||
-rwxr-xr-x | tools/run_tests/helper_scripts/pre_build_csharp.sh (renamed from tools/run_tests/pre_build_csharp.sh) | 2 | ||||
-rw-r--r-- | tools/run_tests/helper_scripts/pre_build_node.bat (renamed from tools/run_tests/pre_build_node.bat) | 0 | ||||
-rwxr-xr-x | tools/run_tests/helper_scripts/pre_build_node.sh (renamed from tools/run_tests/pre_build_node.sh) | 0 | ||||
-rwxr-xr-x | tools/run_tests/helper_scripts/pre_build_ruby.sh (renamed from tools/run_tests/pre_build_ruby.sh) | 2 | ||||
-rwxr-xr-x | tools/run_tests/helper_scripts/run_lcov.sh (renamed from tools/run_tests/run_lcov.sh) | 2 | ||||
-rw-r--r-- | tools/run_tests/helper_scripts/run_node.bat (renamed from tools/run_tests/run_node.bat) | 0 | ||||
-rwxr-xr-x | tools/run_tests/helper_scripts/run_node.sh (renamed from tools/run_tests/run_node.sh) | 2 | ||||
-rwxr-xr-x | tools/run_tests/helper_scripts/run_python.sh (renamed from tools/run_tests/run_python.sh) | 2 | ||||
-rwxr-xr-x | tools/run_tests/helper_scripts/run_ruby.sh (renamed from tools/run_tests/run_ruby.sh) | 2 | ||||
-rwxr-xr-x | tools/run_tests/helper_scripts/run_tests_in_workspace.sh (renamed from tools/run_tests/run_tests_in_workspace.sh) | 2 | ||||
-rw-r--r-- | tools/run_tests/interop/interop_html_report.template (renamed from tools/run_tests/interop_html_report.template) | 0 | ||||
-rw-r--r-- | tools/run_tests/python_utils/__init__.py | 28 | ||||
-rwxr-xr-x | tools/run_tests/python_utils/antagonist.py (renamed from tools/run_tests/antagonist.py) | 0 | ||||
-rwxr-xr-x | tools/run_tests/python_utils/dockerjob.py (renamed from tools/run_tests/dockerjob.py) | 3 | ||||
-rw-r--r-- | tools/run_tests/python_utils/filter_pull_request_tests.py (renamed from tools/run_tests/filter_pull_request_tests.py) | 0 | ||||
-rwxr-xr-x | tools/run_tests/python_utils/jobset.py (renamed from tools/run_tests/jobset.py) | 0 | ||||
-rwxr-xr-x | tools/run_tests/python_utils/port_server.py (renamed from tools/run_tests/port_server.py) | 0 | ||||
-rw-r--r-- | tools/run_tests/python_utils/report_utils.py (renamed from tools/run_tests/report_utils.py) | 0 | ||||
-rwxr-xr-x | tools/run_tests/python_utils/watch_dirs.py (renamed from tools/run_tests/watch_dirs.py) | 0 | ||||
-rwxr-xr-x | tools/run_tests/run_interop_tests.py | 7 | ||||
-rwxr-xr-x | tools/run_tests/run_performance_tests.py | 8 | ||||
-rwxr-xr-x | tools/run_tests/run_stress_tests.py | 5 | ||||
-rwxr-xr-x | tools/run_tests/run_tests.py | 72 | ||||
-rwxr-xr-x | tools/run_tests/run_tests_matrix.py | 9 | ||||
-rwxr-xr-x | tools/run_tests/sanity/check_sources_and_headers.py | 2 | ||||
-rwxr-xr-x | tools/run_tests/sanity/check_test_filtering.py | 2 | ||||
-rwxr-xr-x | tools/run_tests/task_runner.py | 9 |
71 files changed, 157 insertions, 155 deletions
diff --git a/.gitignore b/.gitignore index 3cc35ff7cd..1610bd40cd 100644 --- a/.gitignore +++ b/.gitignore @@ -96,7 +96,7 @@ DerivedData Pods/ # Artifacts directory -artifacts/ +/artifacts/ # Git generated files for conflicting *.orig diff --git a/templates/tools/run_tests/configs.json.template b/templates/tools/run_tests/generated/configs.json.template index 5c82dfb347..5c82dfb347 100644 --- a/templates/tools/run_tests/configs.json.template +++ b/templates/tools/run_tests/generated/configs.json.template diff --git a/templates/tools/run_tests/sources_and_headers.json.template b/templates/tools/run_tests/generated/sources_and_headers.json.template index 1c5c9747d6..1c5c9747d6 100644 --- a/templates/tools/run_tests/sources_and_headers.json.template +++ b/templates/tools/run_tests/generated/sources_and_headers.json.template diff --git a/templates/tools/run_tests/tests.json.template b/templates/tools/run_tests/generated/tests.json.template index 1e21465dd2..1e21465dd2 100644 --- a/templates/tools/run_tests/tests.json.template +++ b/templates/tools/run_tests/generated/tests.json.template diff --git a/tools/buildgen/generate_projects.py b/tools/buildgen/generate_projects.py index 5e78ad52d6..f8ddaf4963 100755 --- a/tools/buildgen/generate_projects.py +++ b/tools/buildgen/generate_projects.py @@ -36,7 +36,7 @@ import shutil import sys import tempfile import multiprocessing -sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..', 'run_tests')) +sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..', 'run_tests', 'python_utils')) assert sys.argv[1:], 'run generate_projects.sh instead of this directly' diff --git a/tools/run_tests/prepare_travis.sh b/tools/run_tests/artifacts/__init__.py index 10546535e8..100a624dc9 100755..100644 --- a/tools/run_tests/prepare_travis.sh +++ b/tools/run_tests/artifacts/__init__.py @@ -1,5 +1,4 @@ -#!/bin/bash -# Copyright 2015, Google Inc. +# Copyright 2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -27,41 +26,3 @@ # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -cd `dirname $0`/../.. -grpc_dir=`pwd` - -distrib=`md5sum /etc/issue | cut -f1 -d\ ` -echo "Configuring for distribution $distrib" -git submodule | while read sha path extra ; do - cd /tmp - name=`basename $path` - file=$name-$sha-$CONFIG-prebuilt-$distrib.tar.gz - echo -n "Looking for $file ..." - url=http://storage.googleapis.com/grpc-prebuilt-packages/$file - wget -q $url && ( - echo " Found." - tar xfz $file - ) || echo " Not found." -done - -mkdir -p bins/$CONFIG/protobuf -mkdir -p libs/$CONFIG/protobuf -mkdir -p libs/$CONFIG/openssl - -function cpt { - cp /tmp/prebuilt/$1 $2/$CONFIG/$3 - touch $2/$CONFIG/$3/`basename $1` -} - -if [ -e /tmp/prebuilt/bin/protoc ] ; then - touch third_party/protobuf/configure - cpt bin/protoc bins protobuf - cpt lib/libprotoc.a libs protobuf - cpt lib/libprotobuf.a libs protobuf -fi - -if [ -e /tmp/prebuilt/lib/libssl.a ] ; then - cpt lib/libcrypto.a libs openssl - cpt lib/libssl.a libs openssl -fi diff --git a/tools/run_tests/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index 65d34e17e1..005d99790a 100644 --- a/tools/run_tests/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -35,7 +35,8 @@ import random import string import sys -import jobset +sys.path.insert(0, os.path.abspath('..')) +import python_utils.jobset as jobset def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={}, @@ -113,7 +114,7 @@ class PythonArtifact: environ['GRPC_BUILD_MANYLINUX_WHEEL'] = 'TRUE' return create_docker_jobspec(self.name, 'tools/dockerfile/grpc_artifact_python_manylinux_%s' % self.arch, - 'tools/run_tests/build_artifact_python.sh', + 'tools/run_tests/artifacts/build_artifact_python.sh', environ=environ, timeout_seconds=60*60) elif self.platform == 'windows': @@ -125,7 +126,7 @@ class PythonArtifact: # seed. We create a random temp-dir here dir = ''.join(random.choice(string.ascii_uppercase) for _ in range(10)) return create_jobspec(self.name, - ['tools\\run_tests\\build_artifact_python.bat', + ['tools\\run_tests\\artifacts\\build_artifact_python.bat', self.py_version, '32' if self.arch == 'x86' else '64', dir @@ -136,7 +137,7 @@ class PythonArtifact: environ['PYTHON'] = self.py_version environ['SKIP_PIP_INSTALL'] = 'TRUE' return create_jobspec(self.name, - ['tools/run_tests/build_artifact_python.sh'], + ['tools/run_tests/artifacts/build_artifact_python.sh'], environ=environ) def __str__(self): @@ -165,11 +166,11 @@ class RubyArtifact: environ['SETARCH_CMD'] = 'linux32' return create_docker_jobspec(self.name, 'tools/dockerfile/grpc_artifact_linux_%s' % self.arch, - 'tools/run_tests/build_artifact_ruby.sh', + 'tools/run_tests/artifacts/build_artifact_ruby.sh', environ=environ) else: return create_jobspec(self.name, - ['tools/run_tests/build_artifact_ruby.sh']) + ['tools/run_tests/artifacts/build_artifact_ruby.sh']) class CSharpExtArtifact: @@ -184,7 +185,7 @@ class CSharpExtArtifact: def pre_build_jobspecs(self): if self.platform == 'windows': return [create_jobspec('prebuild_%s' % self.name, - ['tools\\run_tests\\pre_build_c.bat'], + ['tools\\run_tests\\helper_scripts\\pre_build_c.bat'], shell=True, flake_retries=5, timeout_retries=2)] @@ -195,7 +196,7 @@ class CSharpExtArtifact: if self.platform == 'windows': msbuild_platform = 'Win32' if self.arch == 'x86' else self.arch return create_jobspec(self.name, - ['tools\\run_tests\\build_artifact_csharp.bat', + ['tools\\run_tests\\artifacts\\build_artifact_csharp.bat', 'vsprojects\\grpc_csharp_ext.sln', '/p:Configuration=Release', '/p:PlatformToolset=v120', @@ -210,14 +211,14 @@ class CSharpExtArtifact: if self.platform == 'linux': return create_docker_jobspec(self.name, 'tools/dockerfile/grpc_artifact_linux_%s' % self.arch, - 'tools/run_tests/build_artifact_csharp.sh', + 'tools/run_tests/artifacts/build_artifact_csharp.sh', environ=environ) else: archflag = _ARCH_FLAG_MAP[self.arch] environ['CFLAGS'] += ' %s %s' % (archflag, _MACOS_COMPAT_FLAG) environ['LDFLAGS'] += ' %s' % archflag return create_jobspec(self.name, - ['tools/run_tests/build_artifact_csharp.sh'], + ['tools/run_tests/artifacts/build_artifact_csharp.sh'], environ=environ) def __str__(self): @@ -245,7 +246,7 @@ class NodeExtArtifact: def build_jobspec(self): if self.platform == 'windows': return create_jobspec(self.name, - ['tools\\run_tests\\build_artifact_node.bat', + ['tools\\run_tests\\artifacts\\build_artifact_node.bat', self.gyp_arch], shell=True) else: @@ -253,10 +254,10 @@ class NodeExtArtifact: return create_docker_jobspec( self.name, 'tools/dockerfile/grpc_artifact_linux_{}'.format(self.arch), - 'tools/run_tests/build_artifact_node.sh {}'.format(self.gyp_arch)) + 'tools/run_tests/artifacts/build_artifact_node.sh {}'.format(self.gyp_arch)) else: return create_jobspec(self.name, - ['tools/run_tests/build_artifact_node.sh', + ['tools/run_tests/artifacts/build_artifact_node.sh', self.gyp_arch]) class PHPArtifact: @@ -276,10 +277,10 @@ class PHPArtifact: return create_docker_jobspec( self.name, 'tools/dockerfile/grpc_artifact_linux_{}'.format(self.arch), - 'tools/run_tests/build_artifact_php.sh') + 'tools/run_tests/artifacts/build_artifact_php.sh') else: return create_jobspec(self.name, - ['tools/run_tests/build_artifact_php.sh']) + ['tools/run_tests/artifacts/build_artifact_php.sh']) class ProtocArtifact: """Builds protoc and protoc-plugin artifacts""" @@ -306,18 +307,18 @@ class ProtocArtifact: if self.platform == 'linux': return create_docker_jobspec(self.name, 'tools/dockerfile/grpc_artifact_protoc', - 'tools/run_tests/build_artifact_protoc.sh', + 'tools/run_tests/artifacts/build_artifact_protoc.sh', environ=environ) else: environ['CXXFLAGS'] += ' -std=c++11 -stdlib=libc++ %s' % _MACOS_COMPAT_FLAG return create_jobspec(self.name, - ['tools/run_tests/build_artifact_protoc.sh'], + ['tools/run_tests/artifacts/build_artifact_protoc.sh'], environ=environ) else: generator = 'Visual Studio 12 Win64' if self.arch == 'x64' else 'Visual Studio 12' vcplatform = 'x64' if self.arch == 'x64' else 'Win32' return create_jobspec(self.name, - ['tools\\run_tests\\build_artifact_protoc.bat'], + ['tools\\run_tests\\artifacts\\build_artifact_protoc.bat'], environ={'generator': generator, 'Platform': vcplatform}) diff --git a/tools/run_tests/build_artifact_csharp.bat b/tools/run_tests/artifacts/build_artifact_csharp.bat index 24c8d485f9..24c8d485f9 100644 --- a/tools/run_tests/build_artifact_csharp.bat +++ b/tools/run_tests/artifacts/build_artifact_csharp.bat diff --git a/tools/run_tests/build_artifact_csharp.sh b/tools/run_tests/artifacts/build_artifact_csharp.sh index 7438713f5c..aed04b2745 100755 --- a/tools/run_tests/build_artifact_csharp.sh +++ b/tools/run_tests/artifacts/build_artifact_csharp.sh @@ -30,7 +30,7 @@ set -ex -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. make grpc_csharp_ext diff --git a/tools/run_tests/build_artifact_node.bat b/tools/run_tests/artifacts/build_artifact_node.bat index 2e0ecd21d0..2e0ecd21d0 100644 --- a/tools/run_tests/build_artifact_node.bat +++ b/tools/run_tests/artifacts/build_artifact_node.bat diff --git a/tools/run_tests/build_artifact_node.sh b/tools/run_tests/artifacts/build_artifact_node.sh index 778a5c95d4..1066ebde19 100755 --- a/tools/run_tests/build_artifact_node.sh +++ b/tools/run_tests/artifacts/build_artifact_node.sh @@ -34,7 +34,7 @@ source ~/.nvm/nvm.sh nvm use 4 set -ex -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. rm -rf build || true diff --git a/tools/run_tests/build_artifact_php.sh b/tools/run_tests/artifacts/build_artifact_php.sh index 669447fa9a..c8d55860c1 100755 --- a/tools/run_tests/build_artifact_php.sh +++ b/tools/run_tests/artifacts/build_artifact_php.sh @@ -31,7 +31,7 @@ PHP_TARGET_ARCH=$1 set -ex -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. mkdir -p artifacts diff --git a/tools/run_tests/build_artifact_protoc.bat b/tools/run_tests/artifacts/build_artifact_protoc.bat index b2bf86da40..fd93318833 100644 --- a/tools/run_tests/build_artifact_protoc.bat +++ b/tools/run_tests/artifacts/build_artifact_protoc.bat @@ -34,7 +34,7 @@ cd third_party/protobuf/cmake mkdir build & cd build mkdir solution & cd solution -cmake -G "%generator%" -Dprotobuf_BUILD_TESTS=OFF ../.. || goto :error +cmake -G "%generator%" -Dprotobuf_BUILD_TESTS=OFF ../../.. || goto :error endlocal call vsprojects/build_plugins.bat || goto :error diff --git a/tools/run_tests/build_artifact_protoc.sh b/tools/run_tests/artifacts/build_artifact_protoc.sh index 161d3a84d6..26c2280eff 100755 --- a/tools/run_tests/build_artifact_protoc.sh +++ b/tools/run_tests/artifacts/build_artifact_protoc.sh @@ -33,7 +33,7 @@ source scl_source enable devtoolset-1.1 set -ex -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. make plugins diff --git a/tools/run_tests/build_artifact_python.bat b/tools/run_tests/artifacts/build_artifact_python.bat index 246713a6ce..246713a6ce 100644 --- a/tools/run_tests/build_artifact_python.bat +++ b/tools/run_tests/artifacts/build_artifact_python.bat diff --git a/tools/run_tests/build_artifact_python.sh b/tools/run_tests/artifacts/build_artifact_python.sh index 2a1d41fd68..5a5506029a 100755 --- a/tools/run_tests/build_artifact_python.sh +++ b/tools/run_tests/artifacts/build_artifact_python.sh @@ -30,7 +30,7 @@ set -ex -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. export GRPC_PYTHON_USE_CUSTOM_BDIST=0 export GRPC_PYTHON_BUILD_WITH_CYTHON=1 diff --git a/tools/run_tests/build_artifact_ruby.sh b/tools/run_tests/artifacts/build_artifact_ruby.sh index 2d97b4068b..019efb01fd 100755 --- a/tools/run_tests/build_artifact_ruby.sh +++ b/tools/run_tests/artifacts/build_artifact_ruby.sh @@ -31,7 +31,7 @@ set -ex SYSTEM=`uname | cut -f 1 -d_` -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. set +ex [[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh [[ -s "$HOME/.rvm/scripts/rvm" ]] && source "$HOME/.rvm/scripts/rvm" diff --git a/tools/run_tests/build_package_node.sh b/tools/run_tests/artifacts/build_package_node.sh index a5636cf87a..8b5e8c0bc1 100755 --- a/tools/run_tests/build_package_node.sh +++ b/tools/run_tests/artifacts/build_package_node.sh @@ -33,7 +33,7 @@ source ~/.nvm/nvm.sh nvm use 4 set -ex -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. base=$(pwd) diff --git a/tools/run_tests/build_package_php.sh b/tools/run_tests/artifacts/build_package_php.sh index 56e3319ed9..42a8d9f8df 100755 --- a/tools/run_tests/build_package_php.sh +++ b/tools/run_tests/artifacts/build_package_php.sh @@ -30,7 +30,7 @@ set -ex -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. mkdir -p artifacts/ cp -r $EXTERNAL_GIT_ROOT/architecture={x86,x64},language=php,platform={windows,linux,macos}/artifacts/* artifacts/ || true diff --git a/tools/run_tests/build_package_python.sh b/tools/run_tests/artifacts/build_package_python.sh index 2511a6ae46..4a1c15ceee 100755 --- a/tools/run_tests/build_package_python.sh +++ b/tools/run_tests/artifacts/build_package_python.sh @@ -30,7 +30,7 @@ set -ex -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. mkdir -p artifacts/ diff --git a/tools/run_tests/build_package_ruby.sh b/tools/run_tests/artifacts/build_package_ruby.sh index 0a755bddb0..b4d20d8a4c 100755 --- a/tools/run_tests/build_package_ruby.sh +++ b/tools/run_tests/artifacts/build_package_ruby.sh @@ -30,7 +30,7 @@ set -ex -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. base=$(pwd) diff --git a/tools/run_tests/distribtest_targets.py b/tools/run_tests/artifacts/distribtest_targets.py index a16daac4fe..a7535b3852 100644 --- a/tools/run_tests/distribtest_targets.py +++ b/tools/run_tests/artifacts/distribtest_targets.py @@ -30,7 +30,11 @@ """Definition of targets run distribution package tests.""" -import jobset +import os.path +import sys + +sys.path.insert(0, os.path.abspath('..')) +import python_utils.jobset as jobset def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={}, diff --git a/tools/run_tests/package_targets.py b/tools/run_tests/artifacts/package_targets.py index 673affeac0..d490f571c3 100644 --- a/tools/run_tests/package_targets.py +++ b/tools/run_tests/artifacts/package_targets.py @@ -30,7 +30,12 @@ """Definition of targets to build distribution packages.""" -import jobset +import os.path +import sys + +sys.path.insert(0, os.path.abspath('..')) +import python_utils.jobset as jobset + def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={}, flake_retries=0, timeout_retries=0): @@ -114,7 +119,7 @@ class NodePackage: return create_docker_jobspec( self.name, 'tools/dockerfile/grpc_artifact_linux_x64', - 'tools/run_tests/build_package_node.sh') + 'tools/run_tests/artifacts/build_package_node.sh') class RubyPackage: @@ -131,7 +136,7 @@ class RubyPackage: return create_docker_jobspec( self.name, 'tools/dockerfile/grpc_artifact_linux_x64', - 'tools/run_tests/build_package_ruby.sh') + 'tools/run_tests/artifacts/build_package_ruby.sh') class PythonPackage: @@ -148,7 +153,7 @@ class PythonPackage: return create_docker_jobspec( self.name, 'tools/dockerfile/grpc_artifact_linux_x64', - 'tools/run_tests/build_package_python.sh') + 'tools/run_tests/artifacts/build_package_python.sh') class PHPPackage: @@ -165,7 +170,7 @@ class PHPPackage: return create_docker_jobspec( self.name, 'tools/dockerfile/grpc_artifact_linux_x64', - 'tools/run_tests/build_package_php.sh') + 'tools/run_tests/artifacts/build_package_php.sh') def targets(): diff --git a/tools/run_tests/build_stats_schema.json b/tools/run_tests/build_stats/build_stats_schema.json index 021a349545..021a349545 100644 --- a/tools/run_tests/build_stats_schema.json +++ b/tools/run_tests/build_stats/build_stats_schema.json diff --git a/tools/run_tests/build_stats_schema_no_matrix.json b/tools/run_tests/build_stats/build_stats_schema_no_matrix.json index eeb067d7a5..eeb067d7a5 100644 --- a/tools/run_tests/build_stats_schema_no_matrix.json +++ b/tools/run_tests/build_stats/build_stats_schema_no_matrix.json diff --git a/tools/run_tests/configs.json b/tools/run_tests/generated/configs.json index b0839ef026..b0839ef026 100644 --- a/tools/run_tests/configs.json +++ b/tools/run_tests/generated/configs.json diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 6ae269cc20..6ae269cc20 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json diff --git a/tools/run_tests/tests.json b/tools/run_tests/generated/tests.json index b76263b8b9..b76263b8b9 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/generated/tests.json diff --git a/tools/run_tests/build_csharp.sh b/tools/run_tests/helper_scripts/build_csharp.sh index 48ce11a10b..84c5b1c777 100755 --- a/tools/run_tests/build_csharp.sh +++ b/tools/run_tests/helper_scripts/build_csharp.sh @@ -30,7 +30,7 @@ set -ex -cd $(dirname $0)/../../src/csharp +cd $(dirname $0)/../../../src/csharp # overriding NativeDependenciesConfigurationUnix is needed to make gcov code coverage work. xbuild /p:Configuration=$MSBUILD_CONFIG /p:NativeDependenciesConfigurationUnix=$CONFIG Grpc.sln diff --git a/tools/run_tests/build_csharp_coreclr.bat b/tools/run_tests/helper_scripts/build_csharp_coreclr.bat index b6e3ccbd2b..78e5f5998b 100644 --- a/tools/run_tests/build_csharp_coreclr.bat +++ b/tools/run_tests/helper_scripts/build_csharp_coreclr.bat @@ -29,7 +29,7 @@ setlocal -cd /d %~dp0\..\..\src\csharp +cd /d %~dp0\..\..\..\src\csharp dotnet restore . || goto :error diff --git a/tools/run_tests/build_csharp_coreclr.sh b/tools/run_tests/helper_scripts/build_csharp_coreclr.sh index 02cf0d39cb..dd5fd31c75 100755 --- a/tools/run_tests/build_csharp_coreclr.sh +++ b/tools/run_tests/helper_scripts/build_csharp_coreclr.sh @@ -30,7 +30,7 @@ set -ex -cd $(dirname $0)/../../src/csharp +cd $(dirname $0)/../../../src/csharp # TODO(jtattermusch): introduce caching dotnet restore . diff --git a/tools/run_tests/build_node.bat b/tools/run_tests/helper_scripts/build_node.bat index 82e8208348..82e8208348 100644 --- a/tools/run_tests/build_node.bat +++ b/tools/run_tests/helper_scripts/build_node.bat diff --git a/tools/run_tests/build_node.sh b/tools/run_tests/helper_scripts/build_node.sh index d9292fd8aa..8a928bb762 100755 --- a/tools/run_tests/build_node.sh +++ b/tools/run_tests/helper_scripts/build_node.sh @@ -38,6 +38,6 @@ set -ex CONFIG=${CONFIG:-opt} # change to grpc repo root -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. npm install --unsafe-perm --build-from-source diff --git a/tools/run_tests/build_php.sh b/tools/run_tests/helper_scripts/build_php.sh index 77a8abcfe7..acaaa23adf 100755 --- a/tools/run_tests/build_php.sh +++ b/tools/run_tests/helper_scripts/build_php.sh @@ -33,7 +33,7 @@ set -ex CONFIG=${CONFIG:-opt} # change to grpc repo root -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. root=`pwd` export GRPC_LIB_SUBDIR=libs/$CONFIG diff --git a/tools/run_tests/build_python.sh b/tools/run_tests/helper_scripts/build_python.sh index 7cac394960..0e88e96765 100755 --- a/tools/run_tests/build_python.sh +++ b/tools/run_tests/helper_scripts/build_python.sh @@ -31,7 +31,7 @@ set -ex # change to grpc repo root -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. ########################## # Portability operations # diff --git a/tools/run_tests/build_python_msys2.sh b/tools/run_tests/helper_scripts/build_python_msys2.sh index 6e9d369018..6e9d369018 100644 --- a/tools/run_tests/build_python_msys2.sh +++ b/tools/run_tests/helper_scripts/build_python_msys2.sh diff --git a/tools/run_tests/build_ruby.sh b/tools/run_tests/helper_scripts/build_ruby.sh index 10343fce69..32638dede9 100755 --- a/tools/run_tests/build_ruby.sh +++ b/tools/run_tests/helper_scripts/build_ruby.sh @@ -34,7 +34,7 @@ set -ex export GRPC_CONFIG=${CONFIG:-opt} # change to grpc's ruby directory -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. rm -rf ./tmp rake compile diff --git a/tools/run_tests/post_tests_c.sh b/tools/run_tests/helper_scripts/post_tests_c.sh index 4409526dab..a83a59e23b 100755 --- a/tools/run_tests/post_tests_c.sh +++ b/tools/run_tests/helper_scripts/post_tests_c.sh @@ -32,7 +32,7 @@ set -ex if [ "$CONFIG" != "gcov" ] ; then exit ; fi -root=$(readlink -f $(dirname $0)/../..) +root=$(readlink -f $(dirname $0)/../../..) out=$root/reports/c_cxx_coverage tmp1=$(mktemp) tmp2=$(mktemp) diff --git a/tools/run_tests/post_tests_csharp.bat b/tools/run_tests/helper_scripts/post_tests_csharp.bat index 0d49a00b2a..2359f148ce 100644 --- a/tools/run_tests/post_tests_csharp.bat +++ b/tools/run_tests/helper_scripts/post_tests_csharp.bat @@ -36,7 +36,7 @@ if not "%CONFIG%" == "gcov" ( ) @rem enter src/csharp directory -cd /d %~dp0\..\..\src\csharp +cd /d %~dp0\..\..\..\src\csharp @rem Generate code coverage report @rem TODO(jtattermusch): currently the report list is hardcoded diff --git a/tools/run_tests/post_tests_csharp.sh b/tools/run_tests/helper_scripts/post_tests_csharp.sh index bb6f5c6e18..762c1f8827 100755 --- a/tools/run_tests/post_tests_csharp.sh +++ b/tools/run_tests/helper_scripts/post_tests_csharp.sh @@ -33,7 +33,7 @@ set -ex if [ "$CONFIG" != "gcov" ] ; then exit ; fi # change to gRPC repo root -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. # Generate the csharp extension coverage report gcov objs/gcov/src/csharp/ext/*.o diff --git a/tools/run_tests/post_tests_php.sh b/tools/run_tests/helper_scripts/post_tests_php.sh index b4098066ea..23dc202322 100755 --- a/tools/run_tests/post_tests_php.sh +++ b/tools/run_tests/helper_scripts/post_tests_php.sh @@ -32,7 +32,7 @@ set -ex if [ "$CONFIG" != "gcov" ] ; then exit ; fi -root=$(readlink -f $(dirname $0)/../..) +root=$(readlink -f $(dirname $0)/../../..) out=$root/reports/php_ext_coverage tmp1=$(mktemp) tmp2=$(mktemp) diff --git a/tools/run_tests/post_tests_ruby.sh b/tools/run_tests/helper_scripts/post_tests_ruby.sh index 0877e44805..300edfe8a3 100755 --- a/tools/run_tests/post_tests_ruby.sh +++ b/tools/run_tests/helper_scripts/post_tests_ruby.sh @@ -32,7 +32,7 @@ set -ex if [ "$CONFIG" != "gcov" ] ; then exit ; fi -root=$(readlink -f $(dirname $0)/../..) +root=$(readlink -f $(dirname $0)/../../..) out=$root/reports/ruby_ext_coverage tmp1=$(mktemp) tmp2=$(mktemp) diff --git a/tools/run_tests/pre_build_c.bat b/tools/run_tests/helper_scripts/pre_build_c.bat index e4ab69384c..75b90f85b2 100644 --- a/tools/run_tests/pre_build_c.bat +++ b/tools/run_tests/helper_scripts/pre_build_c.bat @@ -32,7 +32,7 @@ setlocal @rem enter repo root -cd /d %~dp0\..\.. +cd /d %~dp0\..\..\.. @rem Location of nuget.exe set NUGET=C:\nuget\nuget.exe diff --git a/tools/run_tests/pre_build_csharp.bat b/tools/run_tests/helper_scripts/pre_build_csharp.bat index f15979a96b..139955d4da 100644 --- a/tools/run_tests/pre_build_csharp.bat +++ b/tools/run_tests/helper_scripts/pre_build_csharp.bat @@ -32,7 +32,7 @@ setlocal @rem enter repo root -cd /d %~dp0\..\.. +cd /d %~dp0\..\..\.. @rem Location of nuget.exe set NUGET=C:\nuget\nuget.exe diff --git a/tools/run_tests/pre_build_csharp.sh b/tools/run_tests/helper_scripts/pre_build_csharp.sh index ee678ddce5..1f808556f4 100755 --- a/tools/run_tests/pre_build_csharp.sh +++ b/tools/run_tests/helper_scripts/pre_build_csharp.sh @@ -31,7 +31,7 @@ set -ex # cd to gRPC csharp directory -cd $(dirname $0)/../../src/csharp +cd $(dirname $0)/../../../src/csharp root=`pwd` diff --git a/tools/run_tests/pre_build_node.bat b/tools/run_tests/helper_scripts/pre_build_node.bat index addb01a2a4..addb01a2a4 100644 --- a/tools/run_tests/pre_build_node.bat +++ b/tools/run_tests/helper_scripts/pre_build_node.bat diff --git a/tools/run_tests/pre_build_node.sh b/tools/run_tests/helper_scripts/pre_build_node.sh index e63be9da52..e63be9da52 100755 --- a/tools/run_tests/pre_build_node.sh +++ b/tools/run_tests/helper_scripts/pre_build_node.sh diff --git a/tools/run_tests/pre_build_ruby.sh b/tools/run_tests/helper_scripts/pre_build_ruby.sh index e7074c45c2..56b58df544 100755 --- a/tools/run_tests/pre_build_ruby.sh +++ b/tools/run_tests/helper_scripts/pre_build_ruby.sh @@ -34,6 +34,6 @@ set -ex export GRPC_CONFIG=${CONFIG:-opt} # change to grpc repo root -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. bundle install diff --git a/tools/run_tests/run_lcov.sh b/tools/run_tests/helper_scripts/run_lcov.sh index 796a0b5ceb..bc7b44cd3e 100755 --- a/tools/run_tests/run_lcov.sh +++ b/tools/run_tests/helper_scripts/run_lcov.sh @@ -32,7 +32,7 @@ set -ex out=$(readlink -f ${1:-coverage}) -root=$(readlink -f $(dirname $0)/../..) +root=$(readlink -f $(dirname $0)/../../..) shift || true tmp=$(mktemp) cd $root diff --git a/tools/run_tests/run_node.bat b/tools/run_tests/helper_scripts/run_node.bat index 0987fbee55..0987fbee55 100644 --- a/tools/run_tests/run_node.bat +++ b/tools/run_tests/helper_scripts/run_node.bat diff --git a/tools/run_tests/run_node.sh b/tools/run_tests/helper_scripts/run_node.sh index 44f75645f5..0fafe9481a 100755 --- a/tools/run_tests/run_node.sh +++ b/tools/run_tests/helper_scripts/run_node.sh @@ -37,7 +37,7 @@ set -ex CONFIG=${CONFIG:-opt} # change to grpc repo root -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. root=`pwd` diff --git a/tools/run_tests/run_python.sh b/tools/run_tests/helper_scripts/run_python.sh index 17e0186f2a..7be473428f 100755 --- a/tools/run_tests/run_python.sh +++ b/tools/run_tests/helper_scripts/run_python.sh @@ -31,7 +31,7 @@ set -ex # change to grpc repo root -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. PYTHON=`realpath -s "${1:-py27/bin/python}"` diff --git a/tools/run_tests/run_ruby.sh b/tools/run_tests/helper_scripts/run_ruby.sh index 73a84ac361..ab153b7e25 100755 --- a/tools/run_tests/run_ruby.sh +++ b/tools/run_tests/helper_scripts/run_ruby.sh @@ -31,6 +31,6 @@ set -ex # change to grpc repo root -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. rake diff --git a/tools/run_tests/run_tests_in_workspace.sh b/tools/run_tests/helper_scripts/run_tests_in_workspace.sh index 9c6c5b76e0..002c8d6de2 100755 --- a/tools/run_tests/run_tests_in_workspace.sh +++ b/tools/run_tests/helper_scripts/run_tests_in_workspace.sh @@ -34,7 +34,7 @@ # newly created workspace) set -ex -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. export repo_root=$(pwd) rm -rf "${WORKSPACE_NAME}" diff --git a/tools/run_tests/interop_html_report.template b/tools/run_tests/interop/interop_html_report.template index 46cce426b7..46cce426b7 100644 --- a/tools/run_tests/interop_html_report.template +++ b/tools/run_tests/interop/interop_html_report.template diff --git a/tools/run_tests/python_utils/__init__.py b/tools/run_tests/python_utils/__init__.py new file mode 100644 index 0000000000..100a624dc9 --- /dev/null +++ b/tools/run_tests/python_utils/__init__.py @@ -0,0 +1,28 @@ +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/tools/run_tests/antagonist.py b/tools/run_tests/python_utils/antagonist.py index 857addfb38..857addfb38 100755 --- a/tools/run_tests/antagonist.py +++ b/tools/run_tests/python_utils/antagonist.py diff --git a/tools/run_tests/dockerjob.py b/tools/run_tests/python_utils/dockerjob.py index 4a7e61b3c4..0869c5cee9 100755 --- a/tools/run_tests/dockerjob.py +++ b/tools/run_tests/python_utils/dockerjob.py @@ -31,13 +31,14 @@ from __future__ import print_function -import jobset import tempfile import time import uuid import os import subprocess +import jobset + _DEVNULL = open(os.devnull, 'w') diff --git a/tools/run_tests/filter_pull_request_tests.py b/tools/run_tests/python_utils/filter_pull_request_tests.py index ca1d6d4eb5..ca1d6d4eb5 100644 --- a/tools/run_tests/filter_pull_request_tests.py +++ b/tools/run_tests/python_utils/filter_pull_request_tests.py diff --git a/tools/run_tests/jobset.py b/tools/run_tests/python_utils/jobset.py index 7b2c62d1a2..7b2c62d1a2 100755 --- a/tools/run_tests/jobset.py +++ b/tools/run_tests/python_utils/jobset.py diff --git a/tools/run_tests/port_server.py b/tools/run_tests/python_utils/port_server.py index e9b3f7ff79..e9b3f7ff79 100755 --- a/tools/run_tests/port_server.py +++ b/tools/run_tests/python_utils/port_server.py diff --git a/tools/run_tests/report_utils.py b/tools/run_tests/python_utils/report_utils.py index 5ce2a87cfa..5ce2a87cfa 100644 --- a/tools/run_tests/report_utils.py +++ b/tools/run_tests/python_utils/report_utils.py diff --git a/tools/run_tests/watch_dirs.py b/tools/run_tests/python_utils/watch_dirs.py index 21ef23e158..21ef23e158 100755 --- a/tools/run_tests/watch_dirs.py +++ b/tools/run_tests/python_utils/watch_dirs.py diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 83cfc429f9..c14f18af81 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -34,20 +34,21 @@ from __future__ import print_function import argparse import atexit -import dockerjob import itertools -import jobset import json import multiprocessing import os import re -import report_utils import subprocess import sys import tempfile import time import uuid +import python_utils.dockerjob as dockerjob +import python_utils.jobset as jobset +import python_utils.report_utils as report_utils + # Docker doesn't clean up after itself, so we do it on exit. atexit.register(lambda: subprocess.call(['stty', 'echo'])) diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index 69ccff85cf..b7b742d7af 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -35,21 +35,21 @@ from __future__ import print_function import argparse import collections import itertools -import jobset import json import multiprocessing import os -import performance.scenario_config as scenario_config import pipes import re -import report_utils import subprocess import sys import tempfile import time import traceback import uuid -import report_utils + +import performance.scenario_config as scenario_config +import python_utils.jobset as jobset +import python_utils.report_utils as report_utils _ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) diff --git a/tools/run_tests/run_stress_tests.py b/tools/run_tests/run_stress_tests.py index de4a22877c..a94a615b88 100755 --- a/tools/run_tests/run_stress_tests.py +++ b/tools/run_tests/run_stress_tests.py @@ -33,9 +33,7 @@ from __future__ import print_function import argparse import atexit -import dockerjob import itertools -import jobset import json import multiprocessing import os @@ -46,6 +44,9 @@ import tempfile import time import uuid +import python_utils.dockerjob as dockerjob +import python_utils.jobset as jobset + # Docker doesn't clean up after itself, so we do it on exit. atexit.register(lambda: subprocess.call(['stty', 'echo'])) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index fe56f4a175..1008c6b6cf 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -54,9 +54,9 @@ import time from six.moves import urllib import uuid -import jobset -import report_utils -import watch_dirs +import python_utils.jobset as jobset +import python_utils.report_utils as report_utils +import python_utils.watch_dirs as watch_dirs _ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) @@ -116,7 +116,7 @@ class Config(object): def get_c_tests(travis, test_lang) : out = [] platforms_str = 'ci_platforms' if travis else 'platforms' - with open('tools/run_tests/tests.json') as f: + with open('tools/run_tests/generated/tests.json') as f: js = json.load(f) return [tgt for tgt in js @@ -300,7 +300,7 @@ class CLanguage(object): def pre_build_steps(self): if self.platform == 'windows': - return [['tools\\run_tests\\pre_build_c.bat']] + return [['tools\\run_tests\\helper_scripts\\pre_build_c.bat']] else: return [] @@ -311,7 +311,7 @@ class CLanguage(object): if self.platform == 'windows': return [] else: - return [['tools/run_tests/post_tests_c.sh']] + return [['tools/run_tests/helper_scripts/post_tests_c.sh']] def makefile_name(self): return 'Makefile' @@ -382,16 +382,16 @@ class NodeLanguage(object): def test_specs(self): if self.platform == 'windows': - return [self.config.job_spec(['tools\\run_tests\\run_node.bat'])] + return [self.config.job_spec(['tools\\run_tests\\helper_scripts\\run_node.bat'])] else: - return [self.config.job_spec(['tools/run_tests/run_node.sh', self.node_version], + return [self.config.job_spec(['tools/run_tests/helper_scripts/run_node.sh', self.node_version], environ=_FORCE_ENVIRON_FOR_WRAPPERS)] def pre_build_steps(self): if self.platform == 'windows': - return [['tools\\run_tests\\pre_build_node.bat']] + return [['tools\\run_tests\\helper_scripts\\pre_build_node.bat']] else: - return [['tools/run_tests/pre_build_node.sh', self.node_version]] + return [['tools/run_tests/helper_scripts/pre_build_node.sh', self.node_version]] def make_targets(self): return [] @@ -401,9 +401,9 @@ class NodeLanguage(object): def build_steps(self): if self.platform == 'windows': - return [['tools\\run_tests\\build_node.bat']] + return [['tools\\run_tests\\helper_scripts\\build_node.bat']] else: - return [['tools/run_tests/build_node.sh', self.node_version]] + return [['tools/run_tests/helper_scripts/build_node.sh', self.node_version]] def post_tests_steps(self): return [] @@ -439,10 +439,10 @@ class PhpLanguage(object): return [] def build_steps(self): - return [['tools/run_tests/build_php.sh']] + return [['tools/run_tests/helper_scripts/build_php.sh']] def post_tests_steps(self): - return [['tools/run_tests/post_tests_php.sh']] + return [['tools/run_tests/helper_scripts/post_tests_php.sh']] def makefile_name(self): return 'Makefile' @@ -475,10 +475,10 @@ class Php7Language(object): return [] def build_steps(self): - return [['tools/run_tests/build_php.sh']] + return [['tools/run_tests/helper_scripts/build_php.sh']] def post_tests_steps(self): - return [['tools/run_tests/post_tests_php.sh']] + return [['tools/run_tests/helper_scripts/post_tests_php.sh']] def makefile_name(self): return 'Makefile' @@ -547,18 +547,18 @@ class PythonLanguage(object): if os.name == 'nt': shell = ['bash'] - builder = [os.path.abspath('tools/run_tests/build_python_msys2.sh')] + builder = [os.path.abspath('tools/run_tests/helper_scripts/build_python_msys2.sh')] builder_prefix_arguments = ['MINGW{}'.format(bits)] venv_relative_python = ['Scripts/python.exe'] toolchain = ['mingw32'] else: shell = [] - builder = [os.path.abspath('tools/run_tests/build_python.sh')] + builder = [os.path.abspath('tools/run_tests/helper_scripts/build_python.sh')] builder_prefix_arguments = [] venv_relative_python = ['bin/python'] toolchain = ['unix'] - runner = [os.path.abspath('tools/run_tests/run_python.sh')] + runner = [os.path.abspath('tools/run_tests/helper_scripts/run_python.sh')] config_vars = _PythonConfigVars(shell, builder, builder_prefix_arguments, venv_relative_python, toolchain, runner) python27_config = _python_config_generator(name='py27', major='2', @@ -610,12 +610,12 @@ class RubyLanguage(object): _check_compiler(self.args.compiler, ['default']) def test_specs(self): - return [self.config.job_spec(['tools/run_tests/run_ruby.sh'], + return [self.config.job_spec(['tools/run_tests/helper_scripts/run_ruby.sh'], timeout_seconds=10*60, environ=_FORCE_ENVIRON_FOR_WRAPPERS)] def pre_build_steps(self): - return [['tools/run_tests/pre_build_ruby.sh']] + return [['tools/run_tests/helper_scripts/pre_build_ruby.sh']] def make_targets(self): return [] @@ -624,10 +624,10 @@ class RubyLanguage(object): return [] def build_steps(self): - return [['tools/run_tests/build_ruby.sh']] + return [['tools/run_tests/helper_scripts/build_ruby.sh']] def post_tests_steps(self): - return [['tools/run_tests/post_tests_ruby.sh']] + return [['tools/run_tests/helper_scripts/post_tests_ruby.sh']] def makefile_name(self): return 'Makefile' @@ -725,9 +725,9 @@ class CSharpLanguage(object): def pre_build_steps(self): if self.platform == 'windows': - return [['tools\\run_tests\\pre_build_csharp.bat']] + return [['tools\\run_tests\\helper_scripts\\pre_build_csharp.bat']] else: - return [['tools/run_tests/pre_build_csharp.sh']] + return [['tools/run_tests/helper_scripts/pre_build_csharp.sh']] def make_targets(self): return ['grpc_csharp_ext'] @@ -738,22 +738,22 @@ class CSharpLanguage(object): def build_steps(self): if self.args.compiler == 'coreclr': if self.platform == 'windows': - return [['tools\\run_tests\\build_csharp_coreclr.bat']] + return [['tools\\run_tests\\helper_scripts\\build_csharp_coreclr.bat']] else: - return [['tools/run_tests/build_csharp_coreclr.sh']] + return [['tools/run_tests/helper_scripts/build_csharp_coreclr.sh']] else: if self.platform == 'windows': return [[_windows_build_bat(self.args.compiler), 'src/csharp/Grpc.sln', '/p:Configuration=%s' % _MSBUILD_CONFIG[self.config.build_config]]] else: - return [['tools/run_tests/build_csharp.sh']] + return [['tools/run_tests/helper_scripts/build_csharp.sh']] def post_tests_steps(self): if self.platform == 'windows': - return [['tools\\run_tests\\post_tests_csharp.bat']] + return [['tools\\run_tests\\helper_scripts\\post_tests_csharp.bat']] else: - return [['tools/run_tests/post_tests_csharp.sh']] + return [['tools/run_tests/helper_scripts/post_tests_csharp.sh']] def makefile_name(self): return 'Makefile' @@ -872,9 +872,9 @@ class NodeExpressLanguage(object): def pre_build_steps(self): if self.platform == 'windows': - return [['tools\\run_tests\\pre_build_node.bat']] + return [['tools\\run_tests\\helper_scripts\\pre_build_node.bat']] else: - return [['tools/run_tests/pre_build_node.sh', self.node_version]] + return [['tools/run_tests/helper_scripts/pre_build_node.sh', self.node_version]] def make_targets(self): return [] @@ -898,7 +898,7 @@ class NodeExpressLanguage(object): return 'node_express' # different configurations we can run under -with open('tools/run_tests/configs.json') as f: +with open('tools/run_tests/generated/configs.json') as f: _CONFIGS = dict((cfg['config'], Config(**cfg)) for cfg in ast.literal_eval(f.read())) @@ -1299,7 +1299,7 @@ def _start_port_server(port_server_port): running = False if running: current_version = int(subprocess.check_output( - [sys.executable, os.path.abspath('tools/run_tests/port_server.py'), + [sys.executable, os.path.abspath('tools/run_tests/python_utils/port_server.py'), 'dump_version'])) print('my port server is version %d' % current_version) running = (version >= current_version) @@ -1311,7 +1311,7 @@ def _start_port_server(port_server_port): fd, logfile = tempfile.mkstemp() os.close(fd) print('starting port_server, with log file %s' % logfile) - args = [sys.executable, os.path.abspath('tools/run_tests/port_server.py'), + args = [sys.executable, os.path.abspath('tools/run_tests/python_utils/port_server.py'), '-p', '%d' % port_server_port, '-l', logfile] env = dict(os.environ) env['BUILD_ID'] = 'pleaseDontKillMeJenkins' @@ -1417,7 +1417,7 @@ def _build_and_run( return [] # start antagonists - antagonists = [subprocess.Popen(['tools/run_tests/antagonist.py']) + antagonists = [subprocess.Popen(['tools/run_tests/python_utils/antagonist.py']) for _ in range(0, args.antagonists)] port_server_port = 32766 _start_port_server(port_server_port) diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py index df48099971..6e83180c66 100755 --- a/tools/run_tests/run_tests_matrix.py +++ b/tools/run_tests/run_tests_matrix.py @@ -31,12 +31,13 @@ """Run test matrix.""" import argparse -import jobset import multiprocessing import os -import report_utils import sys -from filter_pull_request_tests import filter_tests + +import python_utils.jobset as jobset +import python_utils.report_utils as report_utils +from python_utils.filter_pull_request_tests import filter_tests _ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) os.chdir(_ROOT) @@ -69,7 +70,7 @@ def _workspace_jobspec(name, runtests_args=[], workspace_name=None, inner_jobs=_ workspace_name = 'workspace_%s' % name env = {'WORKSPACE_NAME': workspace_name} test_job = jobset.JobSpec( - cmdline=['tools/run_tests/run_tests_in_workspace.sh', + cmdline=['tools/run_tests/helper_scripts/run_tests_in_workspace.sh', '-t', '-j', str(inner_jobs), '-x', '../report_%s.xml' % name, diff --git a/tools/run_tests/sanity/check_sources_and_headers.py b/tools/run_tests/sanity/check_sources_and_headers.py index b733ba173f..a86db02b80 100755 --- a/tools/run_tests/sanity/check_sources_and_headers.py +++ b/tools/run_tests/sanity/check_sources_and_headers.py @@ -34,7 +34,7 @@ import re import sys root = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../../..')) -with open(os.path.join(root, 'tools', 'run_tests', 'sources_and_headers.json')) as f: +with open(os.path.join(root, 'tools', 'run_tests', 'generated', 'sources_and_headers.json')) as f: js = json.loads(f.read()) re_inc1 = re.compile(r'^#\s*include\s*"([^"]*)"') diff --git a/tools/run_tests/sanity/check_test_filtering.py b/tools/run_tests/sanity/check_test_filtering.py index b522cdeb49..290a6e2ddf 100755 --- a/tools/run_tests/sanity/check_test_filtering.py +++ b/tools/run_tests/sanity/check_test_filtering.py @@ -38,7 +38,7 @@ import re # hack import paths to pick up extra code sys.path.insert(0, os.path.abspath('tools/run_tests/')) from run_tests_matrix import _create_test_jobs, _create_portability_test_jobs -import filter_pull_request_tests +import python_utils.filter_pull_request_tests as filter_pull_request_tests _LIST_OF_LANGUAGE_LABELS = ['c', 'c++', 'csharp', 'node', 'objc', 'php', 'php7', 'python', 'ruby'] _LIST_OF_PLATFORM_LABELS = ['linux', 'macos', 'windows'] diff --git a/tools/run_tests/task_runner.py b/tools/run_tests/task_runner.py index 2e3fa443b9..fdc4668222 100755 --- a/tools/run_tests/task_runner.py +++ b/tools/run_tests/task_runner.py @@ -33,14 +33,13 @@ from __future__ import print_function import argparse -import atexit -import jobset import multiprocessing import sys -import artifact_targets -import distribtest_targets -import package_targets +import artifacts.artifact_targets as artifact_targets +import artifacts.distribtest_targets as distribtest_targets +import artifacts.package_targets as package_targets +import python_utils.jobset as jobset _TARGETS = [] _TARGETS += artifact_targets.targets() |