aboutsummaryrefslogtreecommitdiffhomepage
path: root/tools
diff options
context:
space:
mode:
Diffstat (limited to 'tools')
-rw-r--r--tools/distrib/python/grpcio_tools/_parallel_compile_patch.py63
-rw-r--r--tools/distrib/python/grpcio_tools/setup.py3
-rw-r--r--tools/doxygen/Doxyfile.c++.internal2
-rw-r--r--tools/internal_ci/helper_scripts/prepare_build_macos_rc18
-rw-r--r--tools/run_tests/artifacts/build_artifact_python.bat4
-rwxr-xr-xtools/run_tests/artifacts/build_artifact_python.sh4
-rw-r--r--tools/run_tests/generated/sources_and_headers.json12
-rwxr-xr-xtools/run_tests/helper_scripts/build_python.sh6
-rwxr-xr-xtools/run_tests/run_interop_tests.py24
9 files changed, 111 insertions, 25 deletions
diff --git a/tools/distrib/python/grpcio_tools/_parallel_compile_patch.py b/tools/distrib/python/grpcio_tools/_parallel_compile_patch.py
new file mode 100644
index 0000000000..4d03ef49ba
--- /dev/null
+++ b/tools/distrib/python/grpcio_tools/_parallel_compile_patch.py
@@ -0,0 +1,63 @@
+# Copyright 2018 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Patches the compile() to allow enable parallel compilation of C/C++.
+
+build_ext has lots of C/C++ files and normally them one by one.
+Enabling parallel build helps a lot.
+"""
+
+import distutils.ccompiler
+import os
+
+try:
+ BUILD_EXT_COMPILER_JOBS = int(
+ os.environ.get('GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS', '1'))
+except ValueError:
+ BUILD_EXT_COMPILER_JOBS = 1
+
+
+# monkey-patch for parallel compilation
+def _parallel_compile(self,
+ sources,
+ output_dir=None,
+ macros=None,
+ include_dirs=None,
+ debug=0,
+ extra_preargs=None,
+ extra_postargs=None,
+ depends=None):
+ # setup the same way as distutils.ccompiler.CCompiler
+ # https://github.com/python/cpython/blob/31368a4f0e531c19affe2a1becd25fc316bc7501/Lib/distutils/ccompiler.py#L564
+ macros, objects, extra_postargs, pp_opts, build = self._setup_compile(
+ output_dir, macros, include_dirs, sources, depends, extra_postargs)
+ cc_args = self._get_cc_args(pp_opts, debug, extra_preargs)
+
+ def _compile_single_file(obj):
+ try:
+ src, ext = build[obj]
+ except KeyError:
+ return
+ self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
+
+ # run compilation of individual files in parallel
+ import multiprocessing.pool
+ multiprocessing.pool.ThreadPool(BUILD_EXT_COMPILER_JOBS).map(
+ _compile_single_file, objects)
+ return objects
+
+
+def monkeypatch_compile_maybe():
+ """Monkeypatching is dumb, but the build speed gain is worth it."""
+ if BUILD_EXT_COMPILER_JOBS > 1:
+ distutils.ccompiler.CCompiler.compile = _parallel_compile
diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py
index c13dfe9ade..64c468cbf7 100644
--- a/tools/distrib/python/grpcio_tools/setup.py
+++ b/tools/distrib/python/grpcio_tools/setup.py
@@ -34,9 +34,12 @@ from setuptools.command import build_ext
os.chdir(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, os.path.abspath('.'))
+import _parallel_compile_patch
import protoc_lib_deps
import grpc_version
+_parallel_compile_patch.monkeypatch_compile_maybe()
+
CLASSIFIERS = [
'Development Status :: 5 - Production/Stable',
'Programming Language :: Python',
diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal
index dc27a6ae76..a96683883c 100644
--- a/tools/doxygen/Doxyfile.c++.internal
+++ b/tools/doxygen/Doxyfile.c++.internal
@@ -1187,6 +1187,7 @@ src/core/lib/uri/uri_parser.h \
src/cpp/README.md \
src/cpp/client/channel_cc.cc \
src/cpp/client/client_context.cc \
+src/cpp/client/client_interceptor.cc \
src/cpp/client/create_channel.cc \
src/cpp/client/create_channel_internal.cc \
src/cpp/client/create_channel_internal.h \
@@ -1196,7 +1197,6 @@ src/cpp/client/generic_stub.cc \
src/cpp/client/insecure_credentials.cc \
src/cpp/client/secure_credentials.cc \
src/cpp/client/secure_credentials.h \
-src/cpp/codegen/client_interceptor.cc \
src/cpp/codegen/codegen_init.cc \
src/cpp/common/alarm.cc \
src/cpp/common/auth_property_iterator.cc \
diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc
index 4b7477db14..2362b370d5 100644
--- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc
+++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc
@@ -57,25 +57,25 @@ source $HOME/.rvm/scripts/rvm
set -e # rvm commands are very verbose
time rvm install 2.5.0
rvm use 2.5.0 --default
-gem install bundler --no-ri --no-doc
-gem install cocoapods --version 1.3.1 --no-ri --no-doc
-gem install rake-compiler --no-ri --no-doc
+time gem install bundler --no-ri --no-doc
+time gem install cocoapods --version 1.3.1 --no-ri --no-doc
+time gem install rake-compiler --no-ri --no-doc
rvm osx-ssl-certs status all
rvm osx-ssl-certs update all
set -ex
# cocoapods
export LANG=en_US.UTF-8
-pod repo update # needed by python
+time pod repo update # needed by python
# python
-pip install virtualenv --user python
-pip install -U Mako six tox setuptools twisted pyyaml --user python
+time pip install virtualenv --user python
+time pip install -U Mako six tox setuptools twisted pyyaml --user python
export PYTHONPATH=/Library/Python/3.4/site-packages
# Install Python 3.7
-curl -O https://www.python.org/ftp/python/3.7.0/python-3.7.0-macosx10.9.pkg
-sudo installer -pkg ./python-3.7.0-macosx10.9.pkg -target /
+time curl -O https://www.python.org/ftp/python/3.7.0/python-3.7.0-macosx10.9.pkg
+time sudo installer -pkg ./python-3.7.0-macosx10.9.pkg -target /
# set xcode version for Obj-C tests
sudo xcode-select -switch /Applications/Xcode_9.2.app/Contents/Developer/
@@ -88,7 +88,7 @@ export DOTNET_CLI_TELEMETRY_OPTOUT=true
# TODO(jtattermusch): better debugging of clock skew, remove once not needed
date
-git submodule update --init
+time git submodule update --init
# Store intermediate build files of ObjC tests into /tmpfs
mkdir /tmpfs/Build-ios-binary-size
diff --git a/tools/run_tests/artifacts/build_artifact_python.bat b/tools/run_tests/artifacts/build_artifact_python.bat
index d277668c94..795e80dc40 100644
--- a/tools/run_tests/artifacts/build_artifact_python.bat
+++ b/tools/run_tests/artifacts/build_artifact_python.bat
@@ -22,6 +22,10 @@ pip install -rrequirements.txt
set GRPC_PYTHON_BUILD_WITH_CYTHON=1
+@rem Allow build_ext to build C/C++ files in parallel
+@rem by enabling a monkeypatch. It speeds up the build a lot.
+set GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS=2
+
mkdir -p %ARTIFACTS_OUT%
set ARTIFACT_DIR=%cd%\%ARTIFACTS_OUT%
diff --git a/tools/run_tests/artifacts/build_artifact_python.sh b/tools/run_tests/artifacts/build_artifact_python.sh
index 2878005bb2..9a2e0f739f 100755
--- a/tools/run_tests/artifacts/build_artifact_python.sh
+++ b/tools/run_tests/artifacts/build_artifact_python.sh
@@ -22,6 +22,10 @@ export PYTHON=${PYTHON:-python}
export PIP=${PIP:-pip}
export AUDITWHEEL=${AUDITWHEEL:-auditwheel}
+# Allow build_ext to build C/C++ files in parallel
+# by enabling a monkeypatch. It speeds up the build a lot.
+export GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS=2
+
mkdir -p "${ARTIFACTS_OUT}"
ARTIFACT_DIR="$PWD/${ARTIFACTS_OUT}"
diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json
index b94fc77d76..1bbacc41f0 100644
--- a/tools/run_tests/generated/sources_and_headers.json
+++ b/tools/run_tests/generated/sources_and_headers.json
@@ -3402,6 +3402,7 @@
"name": "client_interceptors_end2end_test",
"src": [
"test/cpp/end2end/client_interceptors_end2end_test.cc",
+ "test/cpp/end2end/interceptors_util.cc",
"test/cpp/end2end/interceptors_util.h"
],
"third_party": false,
@@ -3600,12 +3601,16 @@
"grpc++_test_util",
"grpc_test_util"
],
- "headers": [],
+ "headers": [
+ "test/cpp/end2end/interceptors_util.h"
+ ],
"is_filegroup": false,
"language": "c++",
"name": "end2end_test",
"src": [
- "test/cpp/end2end/end2end_test.cc"
+ "test/cpp/end2end/end2end_test.cc",
+ "test/cpp/end2end/interceptors_util.cc",
+ "test/cpp/end2end/interceptors_util.h"
],
"third_party": false,
"type": "target"
@@ -4736,6 +4741,7 @@
"language": "c++",
"name": "server_interceptors_end2end_test",
"src": [
+ "test/cpp/end2end/interceptors_util.cc",
"test/cpp/end2end/interceptors_util.h",
"test/cpp/end2end/server_interceptors_end2end_test.cc"
],
@@ -11322,7 +11328,6 @@
"language": "c++",
"name": "grpc++_codegen_base_src",
"src": [
- "src/cpp/codegen/client_interceptor.cc",
"src/cpp/codegen/codegen_init.cc"
],
"third_party": false,
@@ -11563,6 +11568,7 @@
"include/grpcpp/support/time.h",
"src/cpp/client/channel_cc.cc",
"src/cpp/client/client_context.cc",
+ "src/cpp/client/client_interceptor.cc",
"src/cpp/client/create_channel.cc",
"src/cpp/client/create_channel_internal.cc",
"src/cpp/client/create_channel_internal.h",
diff --git a/tools/run_tests/helper_scripts/build_python.sh b/tools/run_tests/helper_scripts/build_python.sh
index 6990244e51..eb3ea9e1f5 100755
--- a/tools/run_tests/helper_scripts/build_python.sh
+++ b/tools/run_tests/helper_scripts/build_python.sh
@@ -80,6 +80,8 @@ function toolchain() {
fi
}
+# TODO(jtattermusch): this adds dependency on grealpath on mac
+# (brew install coreutils) for little reason.
# Command to invoke the linux command `realpath` or equivalent.
function script_realpath() {
# Find `realpath`
@@ -112,6 +114,10 @@ export CFLAGS="-I$ROOT/include -std=gnu99 -fno-wrapv $CFLAGS"
export GRPC_PYTHON_BUILD_WITH_CYTHON=1
export LANG=en_US.UTF-8
+# Allow build_ext to build C/C++ files in parallel
+# by enabling a monkeypatch. It speeds up the build a lot.
+export GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS=4
+
# If ccache is available on Linux, use it.
if [ "$(is_linux)" ]; then
# We're not on Darwin (Mac OS X)
diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py
index 11f483e083..ff6682c3cf 100755
--- a/tools/run_tests/run_interop_tests.py
+++ b/tools/run_tests/run_interop_tests.py
@@ -71,11 +71,9 @@ _TEST_TIMEOUT = 3 * 60
# see https://github.com/grpc/grpc/issues/9779
_SKIP_DATA_FRAME_PADDING = ['data_frame_padding']
-# report suffix is important for reports to get picked up by internal CI
-_INTERNAL_CL_XML_REPORT = 'sponge_log.xml'
-
-# report suffix is important for reports to get picked up by internal CI
-_XML_REPORT = 'report.xml'
+# report suffix "sponge_log.xml" is important for reports to get picked up by internal CI
+_DOCKER_BUILD_XML_REPORT = 'interop_docker_build/sponge_log.xml'
+_TESTS_XML_REPORT = 'interop_test/sponge_log.xml'
class CXXLanguage:
@@ -1163,8 +1161,9 @@ argp.add_argument(
default=False,
action='store_const',
const=True,
- help=('Put reports into subdirectories to improve '
- 'presentation of results by Internal CI.'))
+ help=(
+ '(Deprecated, has no effect) Put reports into subdirectories to improve '
+ 'presentation of results by Internal CI.'))
argp.add_argument(
'--bq_result_table',
default='',
@@ -1253,8 +1252,12 @@ if args.use_docker:
if args.verbose:
print('Jobs to run: \n%s\n' % '\n'.join(str(j) for j in build_jobs))
- num_failures, _ = jobset.run(
+ num_failures, build_resultset = jobset.run(
build_jobs, newline_on_success=True, maxjobs=args.jobs)
+
+ report_utils.render_junit_xml_report(build_resultset,
+ _DOCKER_BUILD_XML_REPORT)
+
if num_failures == 0:
jobset.message(
'SUCCESS',
@@ -1519,10 +1522,7 @@ try:
write_cmdlog_maybe(server_manual_cmd_log, 'interop_server_cmds.sh')
write_cmdlog_maybe(client_manual_cmd_log, 'interop_client_cmds.sh')
- xml_report_name = _XML_REPORT
- if args.internal_ci:
- xml_report_name = _INTERNAL_CL_XML_REPORT
- report_utils.render_junit_xml_report(resultset, xml_report_name)
+ report_utils.render_junit_xml_report(resultset, _TESTS_XML_REPORT)
for name, job in resultset.items():
if "http2" in name:
generated by cgit v1.2.3 (git 2.39.1) at 2024-07-08 07:03:54 +0000