aboutsummaryrefslogtreecommitdiffhomepage
path: root/tools/run_tests
diff options
context:
space:
mode:
Diffstat (limited to 'tools/run_tests')
-rw-r--r--tools/run_tests/artifact_targets.py62
-rw-r--r--tools/run_tests/build_artifact_python.bat12
-rwxr-xr-xtools/run_tests/build_artifact_python.sh29
-rwxr-xr-xtools/run_tests/build_package_node.sh4
-rwxr-xr-xtools/run_tests/build_python.sh3
-rwxr-xr-xtools/run_tests/run_tests.py7
-rwxr-xr-xtools/run_tests/sanity/check_sources_and_headers.py3
-rw-r--r--tools/run_tests/sources_and_headers.json44
8 files changed, 113 insertions, 51 deletions
diff --git a/tools/run_tests/artifact_targets.py b/tools/run_tests/artifact_targets.py
index e9267be58b..8550ee7b84 100644
--- a/tools/run_tests/artifact_targets.py
+++ b/tools/run_tests/artifact_targets.py
@@ -30,11 +30,14 @@
"""Definition of targets to build artifacts."""
+import os.path
+import sys
+
import jobset
def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={},
- flake_retries=0, timeout_retries=0):
+ flake_retries=0, timeout_retries=0, timeout_seconds=30*60):
"""Creates jobspec for a task running under docker."""
environ = environ.copy()
environ['RUN_COMMAND'] = shell_command
@@ -49,20 +52,20 @@ def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={},
cmdline=['tools/run_tests/dockerize/build_and_run_docker.sh'] + docker_args,
environ=docker_env,
shortname='build_artifact.%s' % (name),
- timeout_seconds=30*60,
+ timeout_seconds=timeout_seconds,
flake_retries=flake_retries,
timeout_retries=timeout_retries)
return jobspec
def create_jobspec(name, cmdline, environ=None, shell=False,
- flake_retries=0, timeout_retries=0):
+ flake_retries=0, timeout_retries=0, timeout_seconds=30*60):
"""Creates jobspec."""
jobspec = jobset.JobSpec(
cmdline=cmdline,
environ=environ,
shortname='build_artifact.%s' % (name),
- timeout_seconds=30*60,
+ timeout_seconds=timeout_seconds,
flake_retries=flake_retries,
timeout_retries=timeout_retries,
shell=shell)
@@ -76,27 +79,30 @@ _ARCH_FLAG_MAP = {
'x64': '-m64'
}
-python_version_arch_map = {
- 'x86': 'Python27_32bits',
- 'x64': 'Python27'
+python_windows_version_arch_map = {
+ ('x86', '2.7'): 'Python27_32bits',
+ ('x64', '2.7'): 'Python27',
+ ('x86', '3.4'): 'Python34_32bits',
+ ('x64', '3.4'): 'Python34',
}
class PythonArtifact:
"""Builds Python artifacts."""
- def __init__(self, platform, arch, manylinux_build=None):
+ def __init__(self, platform, arch, python_version, manylinux_build=None):
if manylinux_build:
- self.name = 'python_%s_%s_%s' % (platform, arch, manylinux_build)
+ self.name = 'python%s_%s_%s_%s' % (python_version, platform, arch, manylinux_build)
else:
- self.name = 'python_%s_%s' % (platform, arch)
+ self.name = 'python%s_%s_%s' % (python_version, platform, arch)
self.platform = platform
self.arch = arch
- self.labels = ['artifact', 'python', platform, arch]
- self.python_version = python_version_arch_map[arch]
+ self.labels = ['artifact', 'python', python_version, platform, arch]
+ self.python_version = python_version
+ self.python_windows_prefix = python_windows_version_arch_map[arch, python_version]
self.manylinux_build = manylinux_build
def pre_build_jobspecs(self):
- return []
+ return []
def build_jobspec(self):
environ = {}
@@ -107,26 +113,27 @@ class PythonArtifact:
# special places...
environ['PYTHON'] = '/opt/python/{}/bin/python'.format(self.manylinux_build)
environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.manylinux_build)
- # Our docker image has all the prerequisites pip-installed already.
- environ['SKIP_PIP_INSTALL'] = '1'
# Platform autodetection for the manylinux1 image breaks so we set the
# defines ourselves.
# TODO(atash) get better platform-detection support in core so we don't
# need to do this manually...
environ['CFLAGS'] = '-DGPR_MANYLINUX1=1'
+ environ['BUILD_HEALTH_CHECKING'] = 'TRUE'
+ environ['BUILD_MANYLINUX_WHEEL'] = 'TRUE'
return create_docker_jobspec(self.name,
'tools/dockerfile/grpc_artifact_python_manylinux_%s' % self.arch,
'tools/run_tests/build_artifact_python.sh',
- environ=environ)
+ environ=environ,
+ timeout_seconds=60*60)
elif self.platform == 'windows':
return create_jobspec(self.name,
['tools\\run_tests\\build_artifact_python.bat',
- self.python_version,
+ self.python_windows_prefix,
'32' if self.arch == 'x86' else '64'
],
shell=True)
else:
- environ['SKIP_PIP_INSTALL'] = 'TRUE'
+ environ['PYTHON'] = 'python{}'.format(self.python_version)
return create_jobspec(self.name,
['tools/run_tests/build_artifact_python.sh'],
environ=environ)
@@ -323,13 +330,18 @@ def targets():
for Cls in (CSharpExtArtifact, NodeExtArtifact, ProtocArtifact)
for platform in ('linux', 'macos', 'windows')
for arch in ('x86', 'x64')] +
- [PythonArtifact('linux', 'x86', 'cp27-cp27m'),
- PythonArtifact('linux', 'x86', 'cp27-cp27mu'),
- PythonArtifact('linux', 'x64', 'cp27-cp27m'),
- PythonArtifact('linux', 'x64', 'cp27-cp27mu'),
- PythonArtifact('macos', 'x64'),
- PythonArtifact('windows', 'x86'),
- PythonArtifact('windows', 'x64'),
+ [PythonArtifact('linux', 'x86', '2.7', 'cp27-cp27m'),
+ PythonArtifact('linux', 'x86', '2.7', 'cp27-cp27mu'),
+ PythonArtifact('linux', 'x64', '2.7', 'cp27-cp27m'),
+ PythonArtifact('linux', 'x64', '2.7', 'cp27-cp27mu'),
+ PythonArtifact('macos', 'x64', '2.7'),
+ PythonArtifact('windows', 'x86', '2.7'),
+ PythonArtifact('windows', 'x64', '2.7'),
+ PythonArtifact('linux', 'x86', '3.4', 'cp34-cp34m'),
+ PythonArtifact('linux', 'x64', '3.4', 'cp34-cp34m'),
+ PythonArtifact('macos', 'x64', '3.4'),
+ PythonArtifact('windows', 'x86', '3.4'),
+ PythonArtifact('windows', 'x64', '3.4'),
RubyArtifact('linux', 'x86'),
RubyArtifact('linux', 'x64'),
RubyArtifact('macos', 'x64'),
diff --git a/tools/run_tests/build_artifact_python.bat b/tools/run_tests/build_artifact_python.bat
index a7b1a58284..074a3c6781 100644
--- a/tools/run_tests/build_artifact_python.bat
+++ b/tools/run_tests/build_artifact_python.bat
@@ -42,11 +42,19 @@ python tools\distrib\python\make_grpcio_tools.py
@rem Build gRPC Python extensions
python setup.py build_ext -c mingw32
-python tools\distrib\python\grpcio_tools\setup.py build_ext -c mingw32
+
+pushd tools\distrib\python\grpcio_tools
+python setup.py build_ext -c mingw32
+popd
+
@rem Build gRPC Python distributions
python setup.py bdist_wheel
-python tools\distrib\python\grpcio_tools\setup.py bdist_wheel
+
+pushd tools\distrib\python\grpcio_tools
+python setup.py bdist_wheel
+popd
+
mkdir artifacts
xcopy /Y /I /S dist\* artifacts\ || goto :error
diff --git a/tools/run_tests/build_artifact_python.sh b/tools/run_tests/build_artifact_python.sh
index 55f8eb634b..8f8330ef24 100755
--- a/tools/run_tests/build_artifact_python.sh
+++ b/tools/run_tests/build_artifact_python.sh
@@ -39,15 +39,6 @@ export PIP=${PIP:-pip}
export AUDITWHEEL=${AUDITWHEEL:-auditwheel}
-if [ "$SKIP_PIP_INSTALL" == "" ]
-then
- ${PIP} install --upgrade six
- # There's a bug in newer versions of setuptools (see
- # https://bitbucket.org/pypa/setuptools/issues/503/pkg_resources_vendorpackagingrequirementsi)
- ${PIP} pip install --upgrade 'setuptools==18'
- ${PIP} install -rrequirements.txt
-fi
-
# Build the source distribution first because MANIFEST.in cannot override
# exclusion of built shared objects among package resources (for some
# inexplicable reason).
@@ -71,15 +62,33 @@ CFLAGS="$CFLAGS -fno-wrapv" ${SETARCH_CMD} \
${PYTHON} tools/distrib/python/grpcio_tools/setup.py bdist_wheel
mkdir -p artifacts
-if command -v ${AUDITWHEEL}
+if [ "$BUILD_MANYLINUX_WHEEL" != "" ]
then
for wheel in dist/*.whl; do
${AUDITWHEEL} repair $wheel -w artifacts/
+ rm $wheel
done
for wheel in tools/distrib/python/grpcio_tools/dist/*.whl; do
${AUDITWHEEL} repair $wheel -w artifacts/
+ rm $wheel
done
fi
+# We need to use the built grpcio-tools/grpcio to compile the health proto
+# Wheels are not supported by setup_requires/dependency_links, so we
+# manually install the dependency. Note we should only do this if we
+# are in a docker image or in a virtualenv.
+if [ "$BUILD_HEALTH_CHECKING" != "" ]
+then
+ ${PIP} install -rrequirements.txt
+ ${PIP} install grpcio --no-index --find-links "file://${PWD}/artifacts/"
+ ${PIP} install grpcio-tools --no-index --find-links "file://${PWD}/artifacts/"
+
+ # Build gRPC health check source distribution
+ ${SETARCH_CMD} ${PYTHON} src/python/grpcio_health_checking/setup.py \
+ preprocess build_package_protos sdist
+ cp -r src/python/grpcio_health_checking/dist/* artifacts
+fi
+
cp -r dist/* artifacts
cp -r tools/distrib/python/grpcio_tools/dist/* artifacts
diff --git a/tools/run_tests/build_package_node.sh b/tools/run_tests/build_package_node.sh
index ff4cfdb8bf..ef4a10cca7 100755
--- a/tools/run_tests/build_package_node.sh
+++ b/tools/run_tests/build_package_node.sh
@@ -61,7 +61,7 @@ mkdir -p $output_dir
well_known_protos=( any api compiler/plugin descriptor duration empty field_mask source_context struct timestamp type wrappers )
for arch in {x86,x64}; do
- case arch in
+ case $arch in
x86)
node_arch=ia32
;;
@@ -70,7 +70,7 @@ for arch in {x86,x64}; do
;;
esac
for plat in {windows,linux,macos}; do
- case plat in
+ case $plat in
windows)
node_plat=win32
;;
diff --git a/tools/run_tests/build_python.sh b/tools/run_tests/build_python.sh
index 9cb3cb12a9..c4b5d98a53 100755
--- a/tools/run_tests/build_python.sh
+++ b/tools/run_tests/build_python.sh
@@ -164,7 +164,8 @@ pip_install_dir $ROOT/tools/distrib/python/grpcio_tools
# etc...
pip_install_dir $ROOT
$VENV_PYTHON $ROOT/src/python/grpcio_health_checking/setup.py preprocess
+$VENV_PYTHON $ROOT/src/python/grpcio_health_checking/setup.py build_package_protos
pip_install_dir $ROOT/src/python/grpcio_health_checking
$VENV_PYTHON $ROOT/src/python/grpcio_tests/setup.py preprocess
-$VENV_PYTHON $ROOT/src/python/grpcio_tests/setup.py build_proto_modules
+$VENV_PYTHON $ROOT/src/python/grpcio_tests/setup.py build_package_protos
pip_install_dir $ROOT/src/python/grpcio_tests
diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py
index 57fff2ec9c..bb3d090ae1 100755
--- a/tools/run_tests/run_tests.py
+++ b/tools/run_tests/run_tests.py
@@ -63,7 +63,9 @@ _ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
os.chdir(_ROOT)
-_FORCE_ENVIRON_FOR_WRAPPERS = {}
+_FORCE_ENVIRON_FOR_WRAPPERS = {
+ 'GRPC_VERBOSITY': 'DEBUG',
+}
_POLLING_STRATEGIES = {
@@ -164,7 +166,8 @@ class CLanguage(object):
for polling_strategy in polling_strategies:
env={'GRPC_DEFAULT_SSL_ROOTS_FILE_PATH':
_ROOT + '/src/core/lib/tsi/test_creds/ca.pem',
- 'GRPC_POLL_STRATEGY': polling_strategy}
+ 'GRPC_POLL_STRATEGY': polling_strategy,
+ 'GRPC_VERBOSITY': 'DEBUG'}
shortname_ext = '' if polling_strategy=='all' else ' GRPC_POLL_STRATEGY=%s' % polling_strategy
if self.config.build_config in target['exclude_configs']:
continue
diff --git a/tools/run_tests/sanity/check_sources_and_headers.py b/tools/run_tests/sanity/check_sources_and_headers.py
index c028499ca6..28c1dc46d7 100755
--- a/tools/run_tests/sanity/check_sources_and_headers.py
+++ b/tools/run_tests/sanity/check_sources_and_headers.py
@@ -55,7 +55,8 @@ def target_has_header(target, name):
for dep in target['deps']:
if target_has_header(get_target(dep), name):
return True
- if name == 'src/core/lib/profiling/stap_probes.h':
+ if name in ['src/core/lib/profiling/stap_probes.h',
+ 'src/proto/grpc/reflection/v1alpha/reflection.grpc.pb.h']:
return True
return False
diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json
index e2d2796184..21a2da9051 100644
--- a/tools/run_tests/sources_and_headers.json
+++ b/tools/run_tests/sources_and_headers.json
@@ -4387,25 +4387,19 @@
{
"deps": [
"grpc++",
- "grpc++_codegen_proto"
+ "grpc++_reflection_proto"
],
"headers": [
"include/grpc++/ext/proto_server_reflection_plugin.h",
- "include/grpc++/ext/reflection.grpc.pb.h",
- "include/grpc++/ext/reflection.pb.h",
"src/cpp/ext/proto_server_reflection.h"
],
"language": "c++",
"name": "grpc++_reflection",
"src": [
"include/grpc++/ext/proto_server_reflection_plugin.h",
- "include/grpc++/ext/reflection.grpc.pb.h",
- "include/grpc++/ext/reflection.pb.h",
"src/cpp/ext/proto_server_reflection.cc",
"src/cpp/ext/proto_server_reflection.h",
- "src/cpp/ext/proto_server_reflection_plugin.cc",
- "src/cpp/ext/reflection.grpc.pb.cc",
- "src/cpp/ext/reflection.pb.cc"
+ "src/cpp/ext/proto_server_reflection_plugin.cc"
],
"third_party": false,
"type": "lib"
@@ -4413,6 +4407,18 @@
{
"deps": [],
"headers": [
+ "src/proto/grpc/reflection/v1alpha/reflection.grpc.pb.h",
+ "src/proto/grpc/reflection/v1alpha/reflection.pb.h"
+ ],
+ "language": "c++",
+ "name": "grpc++_reflection_codegen",
+ "src": [],
+ "third_party": false,
+ "type": "lib"
+ },
+ {
+ "deps": [],
+ "headers": [
"test/cpp/util/test_config.h"
],
"language": "c++",
@@ -5734,6 +5740,7 @@
"src/core/lib/channel/compress_filter.h",
"src/core/lib/channel/connected_channel.h",
"src/core/lib/channel/context.h",
+ "src/core/lib/channel/handshaker.h",
"src/core/lib/channel/http_client_filter.h",
"src/core/lib/channel/http_server_filter.h",
"src/core/lib/compression/algorithm_metadata.h",
@@ -5827,6 +5834,8 @@
"src/core/lib/channel/connected_channel.c",
"src/core/lib/channel/connected_channel.h",
"src/core/lib/channel/context.h",
+ "src/core/lib/channel/handshaker.c",
+ "src/core/lib/channel/handshaker.h",
"src/core/lib/channel/http_client_filter.c",
"src/core/lib/channel/http_client_filter.h",
"src/core/lib/channel/http_server_filter.c",
@@ -6789,5 +6798,24 @@
],
"third_party": false,
"type": "filegroup"
+ },
+ {
+ "deps": [
+ "grpc++_codegen_proto"
+ ],
+ "headers": [
+ "include/grpc++/ext/reflection.grpc.pb.h",
+ "include/grpc++/ext/reflection.pb.h"
+ ],
+ "language": "c++",
+ "name": "grpc++_reflection_proto",
+ "src": [
+ "include/grpc++/ext/reflection.grpc.pb.h",
+ "include/grpc++/ext/reflection.pb.h",
+ "src/cpp/ext/reflection.grpc.pb.cc",
+ "src/cpp/ext/reflection.pb.cc"
+ ],
+ "third_party": false,
+ "type": "filegroup"
}
]
generated by cgit v1.2.3 (git 2.39.1) at 2025-01-19 03:02:35 +0000