add python monkey-patch for parallel build_ext compilation

7 files changed, 143 insertions, 0 deletions

diff --git a/setup.py b/setup.py
index 388e629ec2..b7a3fda33a 100644
--- a/setup.py
+++ b/setup.py
@@ -57,11 +57,13 @@ os.chdir(os.path.dirname(os.path.abspath(__file__)))
 sys.path.insert(0, os.path.abspath(PYTHON_STEM))
 
 # Break import-style to ensure we can actually find our in-repo dependencies.
+import _parallel_compile_patch
 import _spawn_patch
 import commands
 import grpc_core_dependencies
 import grpc_version
 
+_parallel_compile_patch.monkeypatch_compile_maybe()
 _spawn_patch.monkeypatch_spawn()
 
 LICENSE = 'Apache License 2.0'
diff --git a/src/python/grpcio/_parallel_compile_patch.py b/src/python/grpcio/_parallel_compile_patch.py
new file mode 100644
index 0000000000..4d03ef49ba
--- /dev/null
+++ b/src/python/grpcio/_parallel_compile_patch.py
@@ -0,0 +1,63 @@
+# Copyright 2018 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Patches the compile() to allow enable parallel compilation of C/C++.
+
+build_ext has lots of C/C++ files and normally them one by one.
+Enabling parallel build helps a lot.
+"""
+
+import distutils.ccompiler
+import os
+
+try:
+    BUILD_EXT_COMPILER_JOBS = int(
+        os.environ.get('GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS', '1'))
+except ValueError:
+    BUILD_EXT_COMPILER_JOBS = 1
+
+
+# monkey-patch for parallel compilation
+def _parallel_compile(self,
+                      sources,
+                      output_dir=None,
+                      macros=None,
+                      include_dirs=None,
+                      debug=0,
+                      extra_preargs=None,
+                      extra_postargs=None,
+                      depends=None):
+    # setup the same way as distutils.ccompiler.CCompiler
+    # https://github.com/python/cpython/blob/31368a4f0e531c19affe2a1becd25fc316bc7501/Lib/distutils/ccompiler.py#L564
+    macros, objects, extra_postargs, pp_opts, build = self._setup_compile(
+        output_dir, macros, include_dirs, sources, depends, extra_postargs)
+    cc_args = self._get_cc_args(pp_opts, debug, extra_preargs)
+
+    def _compile_single_file(obj):
+        try:
+            src, ext = build[obj]
+        except KeyError:
+            return
+        self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
+
+    # run compilation of individual files in parallel
+    import multiprocessing.pool
+    multiprocessing.pool.ThreadPool(BUILD_EXT_COMPILER_JOBS).map(
+        _compile_single_file, objects)
+    return objects
+
+
+def monkeypatch_compile_maybe():
+    """Monkeypatching is dumb, but the build speed gain is worth it."""
+    if BUILD_EXT_COMPILER_JOBS > 1:
+        distutils.ccompiler.CCompiler.compile = _parallel_compile
diff --git a/tools/distrib/python/grpcio_tools/_parallel_compile_patch.py b/tools/distrib/python/grpcio_tools/_parallel_compile_patch.py
new file mode 100644
index 0000000000..4d03ef49ba
--- /dev/null
+++ b/tools/distrib/python/grpcio_tools/_parallel_compile_patch.py
@@ -0,0 +1,63 @@
+# Copyright 2018 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Patches the compile() to allow enable parallel compilation of C/C++.
+
+build_ext has lots of C/C++ files and normally them one by one.
+Enabling parallel build helps a lot.
+"""
+
+import distutils.ccompiler
+import os
+
+try:
+    BUILD_EXT_COMPILER_JOBS = int(
+        os.environ.get('GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS', '1'))
+except ValueError:
+    BUILD_EXT_COMPILER_JOBS = 1
+
+
+# monkey-patch for parallel compilation
+def _parallel_compile(self,
+                      sources,
+                      output_dir=None,
+                      macros=None,
+                      include_dirs=None,
+                      debug=0,
+                      extra_preargs=None,
+                      extra_postargs=None,
+                      depends=None):
+    # setup the same way as distutils.ccompiler.CCompiler
+    # https://github.com/python/cpython/blob/31368a4f0e531c19affe2a1becd25fc316bc7501/Lib/distutils/ccompiler.py#L564
+    macros, objects, extra_postargs, pp_opts, build = self._setup_compile(
+        output_dir, macros, include_dirs, sources, depends, extra_postargs)
+    cc_args = self._get_cc_args(pp_opts, debug, extra_preargs)
+
+    def _compile_single_file(obj):
+        try:
+            src, ext = build[obj]
+        except KeyError:
+            return
+        self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
+
+    # run compilation of individual files in parallel
+    import multiprocessing.pool
+    multiprocessing.pool.ThreadPool(BUILD_EXT_COMPILER_JOBS).map(
+        _compile_single_file, objects)
+    return objects
+
+
+def monkeypatch_compile_maybe():
+    """Monkeypatching is dumb, but the build speed gain is worth it."""
+    if BUILD_EXT_COMPILER_JOBS > 1:
+        distutils.ccompiler.CCompiler.compile = _parallel_compile
diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py
index c13dfe9ade..64c468cbf7 100644
--- a/tools/distrib/python/grpcio_tools/setup.py
+++ b/tools/distrib/python/grpcio_tools/setup.py
@@ -34,9 +34,12 @@ from setuptools.command import build_ext
 os.chdir(os.path.dirname(os.path.abspath(__file__)))
 sys.path.insert(0, os.path.abspath('.'))
 
+import _parallel_compile_patch
 import protoc_lib_deps
 import grpc_version
 
+_parallel_compile_patch.monkeypatch_compile_maybe()
+
 CLASSIFIERS = [
     'Development Status :: 5 - Production/Stable',
     'Programming Language :: Python',
diff --git a/tools/run_tests/artifacts/build_artifact_python.bat b/tools/run_tests/artifacts/build_artifact_python.bat
index d277668c94..795e80dc40 100644
--- a/tools/run_tests/artifacts/build_artifact_python.bat
+++ b/tools/run_tests/artifacts/build_artifact_python.bat
@@ -22,6 +22,10 @@ pip install -rrequirements.txt
 
 set GRPC_PYTHON_BUILD_WITH_CYTHON=1
 
+@rem Allow build_ext to build C/C++ files in parallel
+@rem by enabling a monkeypatch. It speeds up the build a lot.
+set GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS=2
+
 mkdir -p %ARTIFACTS_OUT%
 set ARTIFACT_DIR=%cd%\%ARTIFACTS_OUT%
 
diff --git a/tools/run_tests/artifacts/build_artifact_python.sh b/tools/run_tests/artifacts/build_artifact_python.sh
index 2878005bb2..9a2e0f739f 100755
--- a/tools/run_tests/artifacts/build_artifact_python.sh
+++ b/tools/run_tests/artifacts/build_artifact_python.sh
@@ -22,6 +22,10 @@ export PYTHON=${PYTHON:-python}
 export PIP=${PIP:-pip}
 export AUDITWHEEL=${AUDITWHEEL:-auditwheel}
 
+# Allow build_ext to build C/C++ files in parallel
+# by enabling a monkeypatch. It speeds up the build a lot.
+export GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS=2
+
 mkdir -p "${ARTIFACTS_OUT}"
 ARTIFACT_DIR="$PWD/${ARTIFACTS_OUT}"
 
diff --git a/tools/run_tests/helper_scripts/build_python.sh b/tools/run_tests/helper_scripts/build_python.sh
index b0751300f8..eb3ea9e1f5 100755
--- a/tools/run_tests/helper_scripts/build_python.sh
+++ b/tools/run_tests/helper_scripts/build_python.sh
@@ -114,6 +114,10 @@ export CFLAGS="-I$ROOT/include -std=gnu99 -fno-wrapv $CFLAGS"
 export GRPC_PYTHON_BUILD_WITH_CYTHON=1
 export LANG=en_US.UTF-8
 
+# Allow build_ext to build C/C++ files in parallel
+# by enabling a monkeypatch. It speeds up the build a lot.
+export GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS=4
+
 # If ccache is available on Linux, use it.
 if [ "$(is_linux)" ]; then
   # We're not on Darwin (Mac OS X)