Merge pull request #6791 from soltanmm/pain

Make running individual Python tests less painful

5 files changed, 119 insertions, 80 deletions

diff --git a/tools/run_tests/build_python.sh b/tools/run_tests/build_python.sh
index b1c90df824..687b04e954 100755
--- a/tools/run_tests/build_python.sh
+++ b/tools/run_tests/build_python.sh
@@ -33,44 +33,78 @@ set -ex
 # change to grpc repo root
 cd $(dirname $0)/../..
 
-TOX_PYTHON_ENV="$1"
-PY_VERSION="${TOX_PYTHON_ENV: -2}"
+# Arguments
+PYTHON=${1:-python2.7}
+VENV=${2:-py27}
+VENV_RELATIVE_PYTHON=${3:-bin/python}
+TOOLCHAIN=${4:-unix}
 
 ROOT=`pwd`
-export LD_LIBRARY_PATH=$ROOT/libs/$CONFIG
-export DYLD_LIBRARY_PATH=$ROOT/libs/$CONFIG
-export PATH=$ROOT/bins/$CONFIG:$ROOT/bins/$CONFIG/protobuf:$PATH
-export CFLAGS="-I$ROOT/include -std=gnu99"
-export LDFLAGS="-L$ROOT/libs/$CONFIG"
+export CFLAGS="-I$ROOT/include -std=gnu99 -fno-wrapv"
 export GRPC_PYTHON_BUILD_WITH_CYTHON=1
-export GRPC_PYTHON_USE_PRECOMPILED_BINARIES=0
 
-if [ "$CONFIG" = "gcov" ]
-then
-  export GRPC_PYTHON_ENABLE_CYTHON_TRACING=1
-fi
+# Default python on the host to fall back to when instantiating e.g. the
+# virtualenv.
+HOST_PYTHON=${HOST_PYTHON:-python}
 
-tox -e ${TOX_PYTHON_ENV} --notest
+# If ccache is available, use it... unless we're on Mac, then all hell breaks
+# loose because Python does hacky things to support other hacky things done to
+# hacky things on Mac OS X
+PLATFORM=`uname -s`
+if [ "${PLATFORM/Darwin}" = "$PLATFORM" ]; then
+  # We're not on Darwin (Mac OS X)
+  if [ -x "$(command -v ccache)" ]; then
+    if [ -x "$(command -v gcc)" ]; then
+      export CC='ccache gcc'
+    elif [ -x "$(command -v clang)" ]; then
+      export CC='ccache clang'
+    fi
+  fi
+fi
 
-# We force the .so naming convention in PEP 3149 for side by side installation support
-# Note this is the default in Python3, but explicitly disabled for Darwin, so we only
-# use this hack for our testing environment.
-if [ "$PY_VERSION" -gt "27" ]
-then
-  mv $ROOT/src/python/grpcio/grpc/_cython/cygrpc.so $ROOT/src/python/grpcio/grpc/_cython/cygrpc.so.backup || true
+# Find `realpath`
+if [ -x "$(command -v realpath)" ]; then
+  export REALPATH=realpath
+elif [ -x "$(command -v grealpath)" ]; then
+  export REALPATH=grealpath
+else
+  echo 'Couldn'"'"'t find `realpath` or `grealpath`'
+  exit 1
 fi
 
-$ROOT/.tox/${TOX_PYTHON_ENV}/bin/python $ROOT/setup.py build
-$ROOT/.tox/${TOX_PYTHON_ENV}/bin/python $ROOT/setup.py build_py
-$ROOT/.tox/${TOX_PYTHON_ENV}/bin/python $ROOT/setup.py build_ext --inplace
-$ROOT/.tox/${TOX_PYTHON_ENV}/bin/python $ROOT/setup.py gather --test
+# Instnatiate the virtualenv, preferring to do so from the relevant python
+# version. Even if these commands fail (e.g. on Windows due to name conflicts)
+# it's possible that the virtualenv is still usable and we trust the tester to
+# be able to 'figure it out' instead of us e.g. doing potentially expensive and
+# unnecessary error recovery by `rm -rf`ing the virtualenv.
+($PYTHON -m virtualenv $VENV ||
+ $HOST_PYTHON -m virtualenv -p $PYTHON $VENV ||
+ true)
+VENV_PYTHON=`$REALPATH -s "$VENV/$VENV_RELATIVE_PYTHON"`
 
-if [ "$PY_VERSION" -gt "27" ]
-then
-  mv $ROOT/src/python/grpcio/grpc/_cython/cygrpc.so $ROOT/src/python/grpcio/grpc/_cython/cygrpc.cpython-${PY_VERSION}m.so || true
-  mv $ROOT/src/python/grpcio/grpc/_cython/cygrpc.so.backup $ROOT/src/python/grpcio/grpc/_cython/cygrpc.so || true
-fi
+# pip-installs the directory specified. Used because on MSYS the vanilla Windows
+# Python gets confused when parsing paths.
+pip_install_dir() {
+  PWD=`pwd`
+  cd $1
+  ($VENV_PYTHON setup.py build_ext -c $TOOLCHAIN || true)
+  # install the dependencies
+  $VENV_PYTHON -m pip install --upgrade .
+  # ensure that we've reinstalled the test packages
+  $VENV_PYTHON -m pip install --upgrade --force-reinstall --no-deps .
+  cd $PWD
+}
 
-# Build the health checker
-$ROOT/.tox/${TOX_PYTHON_ENV}/bin/python $ROOT/src/python/grpcio_health_checking/setup.py build
-$ROOT/.tox/${TOX_PYTHON_ENV}/bin/python $ROOT/src/python/grpcio_health_checking/setup.py build_py
+$VENV_PYTHON -m pip install --upgrade pip setuptools
+$VENV_PYTHON -m pip install cython
+pip_install_dir $ROOT
+$VENV_PYTHON $ROOT/tools/distrib/python/make_grpcio_tools.py
+pip_install_dir $ROOT/tools/distrib/python/grpcio_tools
+# TODO(atash) figure out namespace packages and grpcio-tools and auditwheel
+# etc...
+pip_install_dir $ROOT
+$VENV_PYTHON $ROOT/src/python/grpcio_health_checking/setup.py preprocess
+pip_install_dir $ROOT/src/python/grpcio_health_checking
+$VENV_PYTHON $ROOT/src/python/grpcio_tests/setup.py preprocess
+$VENV_PYTHON $ROOT/src/python/grpcio_tests/setup.py build_proto_modules
+pip_install_dir $ROOT/src/python/grpcio_tests
diff --git a/tools/run_tests/performance/run_worker_python.sh b/tools/run_tests/performance/run_worker_python.sh
index 0da8deda58..3b8ba6f4e4 100755
--- a/tools/run_tests/performance/run_worker_python.sh
+++ b/tools/run_tests/performance/run_worker_python.sh
@@ -32,4 +32,4 @@ set -ex
 
 cd $(dirname $0)/../../..
 
-PYTHONPATH=src/python/grpcio:src/python/gens .tox/py27/bin/python src/python/grpcio/tests/qps/qps_worker.py $@
+PYTHONPATH=src/python/grpcio_tests:src/python/grpcio:src/python/gens py27/bin/python src/python/grpcio_tests/tests/qps/qps_worker.py $@
diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py
index adf9adaf13..13a4a49325 100755
--- a/tools/run_tests/run_interop_tests.py
+++ b/tools/run_tests/run_interop_tests.py
@@ -304,8 +304,11 @@ class PythonLanguage:
 
   def client_cmd(self, args):
     return [
-        'tox -einterop_client --',
-        ' '.join(args)
+        'py27/bin/python',
+        'src/python/grpcio_tests/setup.py',
+        'run_interop',
+        '--client',
+        '--args="{}"'.format(' '.join(args))
     ]
 
   def cloud_to_prod_env(self):
@@ -313,8 +316,11 @@ class PythonLanguage:
 
   def server_cmd(self, args):
     return [
-        'tox -einterop_server --',
-        ' '.join(args) + ' --use_tls=true'
+        'py27/bin/python',
+        'src/python/grpcio_tests/setup.py',
+        'run_interop',
+        '--server',
+        '--args="{}"'.format(' '.join(args) + ' --use_tls=true')
     ]
 
   def global_env(self):
diff --git a/tools/run_tests/run_python.sh b/tools/run_tests/run_python.sh
index 8059059d41..17e0186f2a 100755
--- a/tools/run_tests/run_python.sh
+++ b/tools/run_tests/run_python.sh
@@ -33,24 +33,11 @@ set -ex
 # change to grpc repo root
 cd $(dirname $0)/../..
 
-TOX_PYTHON_ENV="$1"
+PYTHON=`realpath -s "${1:-py27/bin/python}"`
 
 ROOT=`pwd`
-export LD_LIBRARY_PATH=$ROOT/libs/$CONFIG
-export DYLD_LIBRARY_PATH=$ROOT/libs/$CONFIG
-export PATH=$ROOT/bins/$CONFIG:$ROOT/bins/$CONFIG/protobuf:$PATH
-export CFLAGS="-I$ROOT/include -std=c89"
-export LDFLAGS="-L$ROOT/libs/$CONFIG"
-export GRPC_PYTHON_BUILD_WITH_CYTHON=1
-export GRPC_PYTHON_USE_PRECOMPILED_BINARIES=0
 
-if [ "$CONFIG" = "gcov" ]
-then
-  export GRPC_PYTHON_ENABLE_CYTHON_TRACING=1
-  tox -e ${TOX_PYTHON_ENV}
-else
-  $ROOT/.tox/${TOX_PYTHON_ENV}/bin/python $ROOT/setup.py test_lite
-fi
+$PYTHON $ROOT/src/python/grpcio_tests/setup.py test_lite
 
 mkdir -p $ROOT/reports
 rm -rf $ROOT/reports/python-coverage
diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py
index 413d414560..e1b7cf550f 100755
--- a/tools/run_tests/run_tests.py
+++ b/tools/run_tests/run_tests.py
@@ -32,11 +32,13 @@
 
 import argparse
 import ast
+import collections
 import glob
 import itertools
 import json
 import multiprocessing
 import os
+import os.path
 import platform
 import random
 import re
@@ -372,50 +374,53 @@ class PhpLanguage(object):
     return 'php'
 
 
+class PythonConfig(collections.namedtuple('PythonConfig', [
+    'python', 'venv', 'venv_relative_python', 'toolchain',])):
+
+  @property
+  def venv_python(self):
+    return os.path.abspath('{}/{}'.format(self.venv, self.venv_relative_python))
+
+
 class PythonLanguage(object):
 
   def configure(self, config, args):
     self.config = config
     self.args = args
-    self._tox_envs = self._get_tox_envs(self.args.compiler)
+    self.pythons = self._get_pythons(self.args.compiler)
 
   def test_specs(self):
     # load list of known test suites
-    with open('src/python/grpcio/tests/tests.json') as tests_json_file:
+    with open('src/python/grpcio_tests/tests/tests.json') as tests_json_file:
       tests_json = json.load(tests_json_file)
     environment = dict(_FORCE_ENVIRON_FOR_WRAPPERS)
-    environment['PYTHONPATH'] = '{}:{}'.format(
-      os.path.abspath('src/python/gens'),
-      os.path.abspath('src/python/grpcio_health_checking'))
-    if self.config.build_config != 'gcov':
-      return [self.config.job_spec(
-          ['tools/run_tests/run_python.sh', tox_env],
-          environ=dict(environment.items() +
-                       [('GRPC_PYTHON_TESTRUNNER_FILTER', suite_name)]),
-          shortname='%s.test.%s' % (tox_env, suite_name),
-          timeout_seconds=5*60)
-          for suite_name in tests_json
-          for tox_env in self._tox_envs]
-    else:
-      return [self.config.job_spec(['tools/run_tests/run_python.sh', tox_env],
-                                   environ=environment,
-                                   shortname='%s.test.coverage' % tox_env,
-                                   timeout_seconds=15*60)
-                                   for tox_env in self._tox_envs]
-
+    return [self.config.job_spec(
+        ['tools/run_tests/run_python.sh', config.venv_python],
+        timeout_seconds=5*60,
+        environ=dict(environment.items() +
+                     [('GRPC_PYTHON_TESTRUNNER_FILTER', suite_name)]),
+        shortname='%s.test.%s' % (config.venv, suite_name),)
+        for suite_name in tests_json
+        for config in self.pythons]
 
   def pre_build_steps(self):
     return []
 
   def make_targets(self):
-    return ['static_c', 'grpc_python_plugin', 'shared_c']
+    return []
 
   def make_options(self):
     return []
 
   def build_steps(self):
-    return [['tools/run_tests/build_python.sh', tox_env]
-            for tox_env in self._tox_envs]
+    return [
+        [
+            'tools/run_tests/build_python.sh',
+            config.python, config.venv,
+            config.venv_relative_python, config.toolchain
+        ]
+        for config in self.pythons
+    ]
 
   def post_tests_steps(self):
     return []
@@ -426,14 +431,21 @@ class PythonLanguage(object):
   def dockerfile_dir(self):
     return 'tools/dockerfile/test/python_jessie_%s' % _docker_arch_suffix(self.args.arch)
 
-  def _get_tox_envs(self, compiler):
-    """Returns name of tox environment based on selected compiler."""
+  def _get_pythons(self, compiler):
+    if os.name == 'nt':
+      venv_relative_python = 'Scripts/python.exe'
+      toolchain = 'mingw32'
+    else:
+      venv_relative_python = 'bin/python'
+      toolchain = 'unix'
+    python27_config = PythonConfig('python2.7', 'py27', venv_relative_python, toolchain)
+    python34_config = PythonConfig('python3.4', 'py34', venv_relative_python, toolchain)
     if compiler == 'default':
-      return ('py27', 'py34')
+      return (python27_config, python34_config,)
     elif compiler == 'python2.7':
-      return ('py27',)
+      return (python27_config,)
     elif compiler == 'python3.4':
-      return ('py34',)
+      return (python34_config,)
     else:
       raise Exception('Compiler %s not supported.' % compiler)