From 185a0a7028b4bb01b749d80054d4de6f09a22d90 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 30 Oct 2018 12:59:59 +0100 Subject: add python monkey-patch for parallel build_ext compilation --- setup.py | 2 + src/python/grpcio/_parallel_compile_patch.py | 63 ++++++++++++++++++++++ .../python/grpcio_tools/_parallel_compile_patch.py | 63 ++++++++++++++++++++++ tools/distrib/python/grpcio_tools/setup.py | 3 ++ .../run_tests/artifacts/build_artifact_python.bat | 4 ++ tools/run_tests/artifacts/build_artifact_python.sh | 4 ++ tools/run_tests/helper_scripts/build_python.sh | 4 ++ 7 files changed, 143 insertions(+) create mode 100644 src/python/grpcio/_parallel_compile_patch.py create mode 100644 tools/distrib/python/grpcio_tools/_parallel_compile_patch.py diff --git a/setup.py b/setup.py index 388e629ec2..b7a3fda33a 100644 --- a/setup.py +++ b/setup.py @@ -57,11 +57,13 @@ os.chdir(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.abspath(PYTHON_STEM)) # Break import-style to ensure we can actually find our in-repo dependencies. +import _parallel_compile_patch import _spawn_patch import commands import grpc_core_dependencies import grpc_version +_parallel_compile_patch.monkeypatch_compile_maybe() _spawn_patch.monkeypatch_spawn() LICENSE = 'Apache License 2.0' diff --git a/src/python/grpcio/_parallel_compile_patch.py b/src/python/grpcio/_parallel_compile_patch.py new file mode 100644 index 0000000000..4d03ef49ba --- /dev/null +++ b/src/python/grpcio/_parallel_compile_patch.py @@ -0,0 +1,63 @@ +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Patches the compile() to allow enable parallel compilation of C/C++. + +build_ext has lots of C/C++ files and normally them one by one. +Enabling parallel build helps a lot. +""" + +import distutils.ccompiler +import os + +try: + BUILD_EXT_COMPILER_JOBS = int( + os.environ.get('GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS', '1')) +except ValueError: + BUILD_EXT_COMPILER_JOBS = 1 + + +# monkey-patch for parallel compilation +def _parallel_compile(self, + sources, + output_dir=None, + macros=None, + include_dirs=None, + debug=0, + extra_preargs=None, + extra_postargs=None, + depends=None): + # setup the same way as distutils.ccompiler.CCompiler + # https://github.com/python/cpython/blob/31368a4f0e531c19affe2a1becd25fc316bc7501/Lib/distutils/ccompiler.py#L564 + macros, objects, extra_postargs, pp_opts, build = self._setup_compile( + output_dir, macros, include_dirs, sources, depends, extra_postargs) + cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) + + def _compile_single_file(obj): + try: + src, ext = build[obj] + except KeyError: + return + self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) + + # run compilation of individual files in parallel + import multiprocessing.pool + multiprocessing.pool.ThreadPool(BUILD_EXT_COMPILER_JOBS).map( + _compile_single_file, objects) + return objects + + +def monkeypatch_compile_maybe(): + """Monkeypatching is dumb, but the build speed gain is worth it.""" + if BUILD_EXT_COMPILER_JOBS > 1: + distutils.ccompiler.CCompiler.compile = _parallel_compile diff --git a/tools/distrib/python/grpcio_tools/_parallel_compile_patch.py b/tools/distrib/python/grpcio_tools/_parallel_compile_patch.py new file mode 100644 index 0000000000..4d03ef49ba --- /dev/null +++ b/tools/distrib/python/grpcio_tools/_parallel_compile_patch.py @@ -0,0 +1,63 @@ +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Patches the compile() to allow enable parallel compilation of C/C++. + +build_ext has lots of C/C++ files and normally them one by one. +Enabling parallel build helps a lot. +""" + +import distutils.ccompiler +import os + +try: + BUILD_EXT_COMPILER_JOBS = int( + os.environ.get('GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS', '1')) +except ValueError: + BUILD_EXT_COMPILER_JOBS = 1 + + +# monkey-patch for parallel compilation +def _parallel_compile(self, + sources, + output_dir=None, + macros=None, + include_dirs=None, + debug=0, + extra_preargs=None, + extra_postargs=None, + depends=None): + # setup the same way as distutils.ccompiler.CCompiler + # https://github.com/python/cpython/blob/31368a4f0e531c19affe2a1becd25fc316bc7501/Lib/distutils/ccompiler.py#L564 + macros, objects, extra_postargs, pp_opts, build = self._setup_compile( + output_dir, macros, include_dirs, sources, depends, extra_postargs) + cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) + + def _compile_single_file(obj): + try: + src, ext = build[obj] + except KeyError: + return + self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) + + # run compilation of individual files in parallel + import multiprocessing.pool + multiprocessing.pool.ThreadPool(BUILD_EXT_COMPILER_JOBS).map( + _compile_single_file, objects) + return objects + + +def monkeypatch_compile_maybe(): + """Monkeypatching is dumb, but the build speed gain is worth it.""" + if BUILD_EXT_COMPILER_JOBS > 1: + distutils.ccompiler.CCompiler.compile = _parallel_compile diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py index c13dfe9ade..64c468cbf7 100644 --- a/tools/distrib/python/grpcio_tools/setup.py +++ b/tools/distrib/python/grpcio_tools/setup.py @@ -34,9 +34,12 @@ from setuptools.command import build_ext os.chdir(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.abspath('.')) +import _parallel_compile_patch import protoc_lib_deps import grpc_version +_parallel_compile_patch.monkeypatch_compile_maybe() + CLASSIFIERS = [ 'Development Status :: 5 - Production/Stable', 'Programming Language :: Python', diff --git a/tools/run_tests/artifacts/build_artifact_python.bat b/tools/run_tests/artifacts/build_artifact_python.bat index d277668c94..795e80dc40 100644 --- a/tools/run_tests/artifacts/build_artifact_python.bat +++ b/tools/run_tests/artifacts/build_artifact_python.bat @@ -22,6 +22,10 @@ pip install -rrequirements.txt set GRPC_PYTHON_BUILD_WITH_CYTHON=1 +@rem Allow build_ext to build C/C++ files in parallel +@rem by enabling a monkeypatch. It speeds up the build a lot. +set GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS=2 + mkdir -p %ARTIFACTS_OUT% set ARTIFACT_DIR=%cd%\%ARTIFACTS_OUT% diff --git a/tools/run_tests/artifacts/build_artifact_python.sh b/tools/run_tests/artifacts/build_artifact_python.sh index 2878005bb2..9a2e0f739f 100755 --- a/tools/run_tests/artifacts/build_artifact_python.sh +++ b/tools/run_tests/artifacts/build_artifact_python.sh @@ -22,6 +22,10 @@ export PYTHON=${PYTHON:-python} export PIP=${PIP:-pip} export AUDITWHEEL=${AUDITWHEEL:-auditwheel} +# Allow build_ext to build C/C++ files in parallel +# by enabling a monkeypatch. It speeds up the build a lot. +export GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS=2 + mkdir -p "${ARTIFACTS_OUT}" ARTIFACT_DIR="$PWD/${ARTIFACTS_OUT}" diff --git a/tools/run_tests/helper_scripts/build_python.sh b/tools/run_tests/helper_scripts/build_python.sh index b0751300f8..eb3ea9e1f5 100755 --- a/tools/run_tests/helper_scripts/build_python.sh +++ b/tools/run_tests/helper_scripts/build_python.sh @@ -114,6 +114,10 @@ export CFLAGS="-I$ROOT/include -std=gnu99 -fno-wrapv $CFLAGS" export GRPC_PYTHON_BUILD_WITH_CYTHON=1 export LANG=en_US.UTF-8 +# Allow build_ext to build C/C++ files in parallel +# by enabling a monkeypatch. It speeds up the build a lot. +export GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS=4 + # If ccache is available on Linux, use it. if [ "$(is_linux)" ]; then # We're not on Darwin (Mac OS X) -- cgit v1.2.3