Cython refactor of Python C wrapping layer

Also set up environment-related details needed for a smooth Cython
experience: have the test script avoid rebuilding all dependencies if the
virtualenv directory already exists, have the PyPI distribution script
distribute the Cython generated C code rather than the .pyx files.

3 files changed, 29 insertions, 5 deletions

diff --git a/tools/distrib/python/submit.py b/tools/distrib/python/submit.py
index dd48f440ba..a3615b3640 100755
--- a/tools/distrib/python/submit.py
+++ b/tools/distrib/python/submit.py
@@ -66,6 +66,12 @@ try:
 except:
   pass
 
+# Build the Cython C files
+build_env = os.environ.copy()
+build_env['GRPC_PYTHON_BUILD_WITH_CYTHON'] = "1"
+cmd = ['python', 'setup.py', 'build_ext', '--inplace']
+subprocess.call(cmd, cwd=pkgdir, env=build_env)
+
 # Make the push.
 cmd = ['python', 'setup.py', 'sdist']
 subprocess.call(cmd, cwd=pkgdir)
diff --git a/tools/run_tests/build_python.sh b/tools/run_tests/build_python.sh
index d9b7644f44..108dd6aeff 100755
--- a/tools/run_tests/build_python.sh
+++ b/tools/run_tests/build_python.sh
@@ -34,9 +34,21 @@ set -ex
 cd $(dirname $0)/../..
 
 root=`pwd`
-rm -rf python2.7_virtual_environment
-virtualenv -p /usr/bin/python2.7 python2.7_virtual_environment
-source python2.7_virtual_environment/bin/activate
-pip install -r src/python/requirements.txt
-CFLAGS="-I$root/include -std=c89 -Werror" LDFLAGS=-L$root/libs/$CONFIG pip install src/python/src
+
+if [ ! -d 'python2.7_virtual_environment' ]
+then
+  # Build the entire virtual environment
+  virtualenv -p /usr/bin/python2.7 python2.7_virtual_environment
+  source python2.7_virtual_environment/bin/activate
+  pip install -r src/python/requirements.txt
+else
+  source python2.7_virtual_environment/bin/activate
+  # Uninstall and re-install the packages we care about. Don't use
+  # --force-reinstall or --ignore-installed to avoid propagating this
+  # unnecessarily to dependencies. Don't use --no-deps to avoid missing
+  # dependency upgrades.
+  (yes | pip uninstall grpcio) || true
+  (yes | pip uninstall interop) || true
+fi
+CFLAGS="-I$root/include -std=c89" LDFLAGS=-L$root/libs/$CONFIG GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install src/python/src
 pip install src/python/interop
diff --git a/tools/run_tests/python_tests.json b/tools/run_tests/python_tests.json
index 6c969d765f..3bef345029 100755
--- a/tools/run_tests/python_tests.json
+++ b/tools/run_tests/python_tests.json
@@ -1,5 +1,11 @@
 [
   {
+    "module": "grpc._cython.cygrpc_test"
+  },
+  {
+    "module": "grpc._cython.adapter_low_test"
+  },
+  {
     "module": "grpc._adapter._c_test"
   },
   {