Fix grpcio_{health_checking,reflection} packaging

The previous packaging structure exhibited strange behavior of
slowness when trying to use pip to install grpcio-reflection
or grpcio-health-checking in a single line with grpcio-tools.

The root cause seems to be the complicated interaction between
pip and setuptools and the fact that we ship a single .tar.gz
"source" archive for `grpcio_reflection` and
`grpcio_health_checking` packages.  `pip` tries to build this
"source" package, and our build process wants to generate
code for the `.proto` files in the package.  However, we have
already processed the `.proto` files into `_pb2.py` files in
our artifact build process, and installing `grpcio_tools`
to get `grpcio_{reflection,health_checking}` seems excessive.
The behavior gets worse since `setuptools`, while building
the package from source, tries to fetch `grpcio_tools` from
source and build that too.  This takes a while, since it
involves compiling a bunch of native code from `protobuf` and
`grpc` and requires a C compiler to boot.

This commit modifies the Python artifact for the two packages
so that they will not include the raw `.proto` files in the
distribution uploaded to PyPI, nor would they contain the
Python module that does the preprocessing code generation
from the respective .proto files.  Instead, a specific code
path is taken when the generated `_pb2_grpc` Python module is
not present in the package to provide such functionality
when built from the gRPC git repository (and hence when built
from our CI infrastructure.)

4 files changed, 76 insertions, 24 deletions

diff --git a/src/python/grpcio_health_checking/MANIFEST.in b/src/python/grpcio_health_checking/MANIFEST.in
index 5255e4c403..996c74a9d4 100644
--- a/src/python/grpcio_health_checking/MANIFEST.in
+++ b/src/python/grpcio_health_checking/MANIFEST.in
@@ -1,4 +1,3 @@
 include grpc_version.py
-include health_commands.py
-graft grpc_health
+recursive-include grpc_health *.py
 global-exclude *.pyc
diff --git a/src/python/grpcio_health_checking/setup.py b/src/python/grpcio_health_checking/setup.py
index 1f5e9c5130..01d796f4e6 100644
--- a/src/python/grpcio_health_checking/setup.py
+++ b/src/python/grpcio_health_checking/setup.py
@@ -20,10 +20,26 @@ import setuptools
 # Ensure we're in the proper directory whether or not we're being used by pip.
 os.chdir(os.path.dirname(os.path.abspath(__file__)))
 
-# Break import-style to ensure we can actually find our commands module.
-import health_commands
+# Break import-style to ensure we can actually find our local modules.
 import grpc_version
 
+
+class _NoOpCommand(setuptools.Command):
+    """No-op command."""
+
+    description = ''
+    user_options = []
+
+    def initialize_options(self):
+        pass
+
+    def finalize_options(self):
+        pass
+
+    def run(self):
+        pass
+
+
 CLASSIFIERS = [
     'Development Status :: 5 - Production/Stable',
     'Programming Language :: Python',
@@ -40,17 +56,28 @@ PACKAGE_DIRECTORIES = {
     '': '.',
 }
 
-SETUP_REQUIRES = (
-    'grpcio-tools>={version}'.format(version=grpc_version.VERSION),)
-
 INSTALL_REQUIRES = ('protobuf>=3.3.0',
                     'grpcio>={version}'.format(version=grpc_version.VERSION),)
 
-COMMAND_CLASS = {
-    # Run preprocess from the repository *before* doing any packaging!
-    'preprocess': health_commands.CopyProtoModules,
-    'build_package_protos': health_commands.BuildPackageProtos,
-}
+try:
+    # ensure we can load the _pb2_grpc module:
+    from grpc_health.v1 import health_pb2_grpc as _pb2_grpc
+    # if we can find the _pb2_grpc module, the package has already been built.
+    SETUP_REQUIRES = ()
+    COMMAND_CLASS = {
+        # wire up commands to no-op not to break the external dependencies
+        'preprocess': _NoOpCommand,
+        'build_package_protos': _NoOpCommand,
+    }
+except ImportError:  # we are in the build environment
+    import health_commands as _health_commands
+    SETUP_REQUIRES = (
+        'grpcio-tools=={version}'.format(version=grpc_version.VERSION),)
+    COMMAND_CLASS = {
+        # Run preprocess from the repository *before* doing any packaging!
+        'preprocess': _health_commands.CopyProtoModules,
+        'build_package_protos': _health_commands.BuildPackageProtos,
+    }
 
 setuptools.setup(
     name='grpcio-health-checking',
diff --git a/src/python/grpcio_reflection/MANIFEST.in b/src/python/grpcio_reflection/MANIFEST.in
index 0f2130c0b5..d6fb6ce73a 100644
--- a/src/python/grpcio_reflection/MANIFEST.in
+++ b/src/python/grpcio_reflection/MANIFEST.in
@@ -1,4 +1,3 @@
 include grpc_version.py
-include reflection_commands.py
-graft grpc_reflection
+recursive-include grpc_reflection *.py
 global-exclude *.pyc
diff --git a/src/python/grpcio_reflection/setup.py b/src/python/grpcio_reflection/setup.py
index 9360550afb..ad9e86990f 100644
--- a/src/python/grpcio_reflection/setup.py
+++ b/src/python/grpcio_reflection/setup.py
@@ -21,10 +21,26 @@ import setuptools
 # Ensure we're in the proper directory whether or not we're being used by pip.
 os.chdir(os.path.dirname(os.path.abspath(__file__)))
 
-# Break import-style to ensure we can actually find our commands module.
-import reflection_commands
+# Break import-style to ensure we can actually find our local modules.
 import grpc_version
 
+
+class _NoOpCommand(setuptools.Command):
+    """No-op command."""
+
+    description = ''
+    user_options = []
+
+    def initialize_options(self):
+        pass
+
+    def finalize_options(self):
+        pass
+
+    def run(self):
+        pass
+
+
 CLASSIFIERS = [
     'Development Status :: 5 - Production/Stable',
     'Programming Language :: Python',
@@ -41,17 +57,28 @@ PACKAGE_DIRECTORIES = {
     '': '.',
 }
 
-SETUP_REQUIRES = (
-    'grpcio-tools>={version}'.format(version=grpc_version.VERSION),)
-
 INSTALL_REQUIRES = ('protobuf>=3.3.0',
                     'grpcio>={version}'.format(version=grpc_version.VERSION),)
 
-COMMAND_CLASS = {
-    # Run preprocess from the repository *before* doing any packaging!
-    'preprocess': reflection_commands.CopyProtoModules,
-    'build_package_protos': reflection_commands.BuildPackageProtos,
-}
+try:
+    # ensure we can load the _pb2_grpc module:
+    from grpc_reflection.v1alpha import reflection_pb2_grpc as _pb2_grpc
+    # if we can find the _pb2_grpc module, the package has already been built.
+    SETUP_REQUIRES = ()
+    COMMAND_CLASS = {
+        # wire up commands to no-op not to break the external dependencies
+        'preprocess': _NoOpCommand,
+        'build_package_protos': _NoOpCommand,
+    }
+except ImportError:  # we are in the build environment
+    import reflection_commands as _reflection_commands
+    SETUP_REQUIRES = (
+        'grpcio-tools=={version}'.format(version=grpc_version.VERSION),)
+    COMMAND_CLASS = {
+        # Run preprocess from the repository *before* doing any packaging!
+        'preprocess': _reflection_commands.CopyProtoModules,
+        'build_package_protos': _reflection_commands.BuildPackageProtos,
+    }
 
 setuptools.setup(
     name='grpcio-reflection',