diff options
Diffstat (limited to 'tools/distrib/python')
4 files changed, 68 insertions, 1 deletions
diff --git a/tools/distrib/python/grpcio_tools/MANIFEST.in b/tools/distrib/python/grpcio_tools/MANIFEST.in index 11ce367747..4943751879 100644 --- a/tools/distrib/python/grpcio_tools/MANIFEST.in +++ b/tools/distrib/python/grpcio_tools/MANIFEST.in @@ -1,3 +1,4 @@ +include _parallel_compile_patch.py include grpc_version.py include protoc_deps.py include protoc_lib_deps.py diff --git a/tools/distrib/python/grpcio_tools/_parallel_compile_patch.py b/tools/distrib/python/grpcio_tools/_parallel_compile_patch.py new file mode 100644 index 0000000000..4d03ef49ba --- /dev/null +++ b/tools/distrib/python/grpcio_tools/_parallel_compile_patch.py @@ -0,0 +1,63 @@ +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Patches the compile() to allow enable parallel compilation of C/C++. + +build_ext has lots of C/C++ files and normally them one by one. +Enabling parallel build helps a lot. +""" + +import distutils.ccompiler +import os + +try: + BUILD_EXT_COMPILER_JOBS = int( + os.environ.get('GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS', '1')) +except ValueError: + BUILD_EXT_COMPILER_JOBS = 1 + + +# monkey-patch for parallel compilation +def _parallel_compile(self, + sources, + output_dir=None, + macros=None, + include_dirs=None, + debug=0, + extra_preargs=None, + extra_postargs=None, + depends=None): + # setup the same way as distutils.ccompiler.CCompiler + # https://github.com/python/cpython/blob/31368a4f0e531c19affe2a1becd25fc316bc7501/Lib/distutils/ccompiler.py#L564 + macros, objects, extra_postargs, pp_opts, build = self._setup_compile( + output_dir, macros, include_dirs, sources, depends, extra_postargs) + cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) + + def _compile_single_file(obj): + try: + src, ext = build[obj] + except KeyError: + return + self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) + + # run compilation of individual files in parallel + import multiprocessing.pool + multiprocessing.pool.ThreadPool(BUILD_EXT_COMPILER_JOBS).map( + _compile_single_file, objects) + return objects + + +def monkeypatch_compile_maybe(): + """Monkeypatching is dumb, but the build speed gain is worth it.""" + if BUILD_EXT_COMPILER_JOBS > 1: + distutils.ccompiler.CCompiler.compile = _parallel_compile diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py index ccb69a8ebc..4b775e667e 100644 --- a/tools/distrib/python/grpcio_tools/grpc_version.py +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION = '1.15.0.dev0' +VERSION = '1.17.0.dev0' diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py index c13dfe9ade..64c468cbf7 100644 --- a/tools/distrib/python/grpcio_tools/setup.py +++ b/tools/distrib/python/grpcio_tools/setup.py @@ -34,9 +34,12 @@ from setuptools.command import build_ext os.chdir(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.abspath('.')) +import _parallel_compile_patch import protoc_lib_deps import grpc_version +_parallel_compile_patch.monkeypatch_compile_maybe() + CLASSIFIERS = [ 'Development Status :: 5 - Production/Stable', 'Programming Language :: Python', |