Make Python testing predictable again

This reorganizes the Python code, scraps the current testing
infrastructure, and implements a simple test discovery and run script
based on the standard Python unittest library so we can trust that our
tests are running.

3 files changed, 97 insertions, 97 deletions

diff --git a/tools/run_tests/build_python.sh b/tools/run_tests/build_python.sh
index 24cf6ba7c8..c6396c85b1 100755
--- a/tools/run_tests/build_python.sh
+++ b/tools/run_tests/build_python.sh
@@ -34,36 +34,7 @@ set -ex
 cd $(dirname $0)/../..
 
 ROOT=`pwd`
-PATH=$ROOT/bins/$CONFIG:$ROOT/bins/$CONFIG/protobuf:$PATH
 GRPCIO=$ROOT/src/python/grpcio
-GRPCIO_TEST=$ROOT/src/python/grpcio_test
-GRPCIO_HEALTH_CHECKING=$ROOT/src/python/grpcio_health_checking
-
-install_grpcio_deps() {
-  cd $GRPCIO
-  pip install -r requirements.txt
-}
-install_grpcio_test_deps() {
-  cd $GRPCIO_TEST
-  pip install -r requirements.txt
-}
-
-install_grpcio() {
-  CFLAGS="-I$ROOT/include -std=c89" LDFLAGS=-L$ROOT/libs/$CONFIG GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install $GRPCIO
-}
-install_grpcio_test() {
-  pip install $GRPCIO_TEST
-}
-install_grpcio_health_checking() {
-  pip install $GRPCIO_HEALTH_CHECKING
-}
-
-# Cleans the environment of previous installations
-clean_grpcio_all() {
-  (yes | pip uninstall grpcio) || true
-  (yes | pip uninstall grpcio_test) || true
-  (yes | pip uninstall grpcio_health_checking) || true
-}
 
 # Builds the testing environment.
 make_virtualenv() {
@@ -74,31 +45,25 @@ make_virtualenv() {
     virtualenv -p `which "python"$1` $virtualenv_name
     source $virtualenv_name/bin/activate
 
-    # Install grpcio
-    install_grpcio_deps
-    install_grpcio
-
-    # Install grpcio_test
-    install_grpcio_test_deps
-    install_grpcio_test
-
-    # Install grpcio_health_checking
-    install_grpcio_health_checking
+    cd $GRPCIO
+    pip install -r requirements.txt
   else
     source $virtualenv_name/bin/activate
-    # Uninstall and re-install the packages we care about. Don't use
-    # --force-reinstall or --ignore-installed to avoid propagating this
-    # unnecessarily to dependencies. Don't use --no-deps to avoid missing
-    # dependency upgrades.
-    clean_grpcio_all
-    install_grpcio || (
-      # Fall back to rebuilding the entire environment
-      rm -rf $virtualenv_name
-      make_virtualenv $1
-    )
-    install_grpcio_test
-    install_grpcio_health_checking
+
+    cd $GRPCIO
+    pip install -U -r requirements.txt
   fi
+
+  export LD_LIBRARY_PATH=$ROOT/libs/$CONFIG
+  export DYLD_LIBRARY_PATH=$ROOT/libs/$CONFIG
+  export PATH=$ROOT/bins/$CONFIG:$ROOT/bins/$CONFIG/protobuf:$PATH
+  export CFLAGS="-I$ROOT/include -std=c89"
+  export LDFLAGS="-L$ROOT/libs/$CONFIG"
+  export GRPC_PYTHON_BUILD_WITH_CYTHON=1
+  export GRPC_PYTHON_ENABLE_CYTHON_TRACING=1
+  python $GRPCIO/setup.py gather --test --install
+  python $GRPCIO/setup.py build_ext --inplace
+  python $GRPCIO/setup.py build_py
 }
 
 make_virtualenv $1
diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py
index 2634164a21..ef8e29acf1 100755
--- a/tools/run_tests/run_interop_tests.py
+++ b/tools/run_tests/run_interop_tests.py
@@ -61,14 +61,14 @@ class CXXLanguage:
     self.server_cwd = None
     self.safename = 'cxx'
 
-  def client_args(self):
-    return ['bins/opt/interop_client']
+  def client_cmd(self, args):
+    return ['bins/opt/interop_client'] + args
 
   def cloud_to_prod_env(self):
     return {}
 
-  def server_args(self):
-    return ['bins/opt/interop_server', '--use_tls=true']
+  def server_cmd(self, args):
+    return ['bins/opt/interop_server', '--use_tls=true'] + args
 
   def global_env(self):
     return {}
@@ -87,14 +87,14 @@ class CSharpLanguage:
     self.server_cwd = 'src/csharp/Grpc.IntegrationTesting.Server/bin/Debug'
     self.safename = str(self)
 
-  def client_args(self):
-    return ['mono', 'Grpc.IntegrationTesting.Client.exe']
+  def client_cmd(self, args):
+    return ['mono', 'Grpc.IntegrationTesting.Client.exe'] + args
 
   def cloud_to_prod_env(self):
     return _SSL_CERT_ENV
 
-  def server_args(self):
-    return ['mono', 'Grpc.IntegrationTesting.Server.exe', '--use_tls=true']
+  def server_cmd(self, args):
+    return ['mono', 'Grpc.IntegrationTesting.Server.exe', '--use_tls=true'] + args
 
   def global_env(self):
     return {}
@@ -113,14 +113,14 @@ class JavaLanguage:
     self.server_cwd = '../grpc-java'
     self.safename = str(self)
 
-  def client_args(self):
-    return ['./run-test-client.sh']
+  def client_cmd(self, args):
+    return ['./run-test-client.sh'] + args
 
   def cloud_to_prod_env(self):
     return {}
 
-  def server_args(self):
-    return ['./run-test-server.sh', '--use_tls=true']
+  def server_cmd(self, args):
+    return ['./run-test-server.sh', '--use_tls=true'] + args
 
   def global_env(self):
     return {}
@@ -140,14 +140,14 @@ class GoLanguage:
     self.server_cwd = '/go/src/google.golang.org/grpc/interop/server'
     self.safename = str(self)
 
-  def client_args(self):
-    return ['go', 'run', 'client.go']
+  def client_cmd(self, args):
+    return ['go', 'run', 'client.go'] + args
 
   def cloud_to_prod_env(self):
     return {}
 
-  def server_args(self):
-    return ['go', 'run', 'server.go', '--use_tls=true']
+  def server_cmd(self, args):
+    return ['go', 'run', 'server.go', '--use_tls=true'] + args
 
   def global_env(self):
     return {}
@@ -191,14 +191,14 @@ class NodeLanguage:
     self.server_cwd = None
     self.safename = str(self)
 
-  def client_args(self):
-    return ['node', 'src/node/interop/interop_client.js']
+  def client_cmd(self, args):
+    return ['node', 'src/node/interop/interop_client.js'] + args
 
   def cloud_to_prod_env(self):
     return _SSL_CERT_ENV
 
-  def server_args(self):
-    return ['node', 'src/node/interop/interop_server.js', '--use_tls=true']
+  def server_cmd(self, args):
+    return ['node', 'src/node/interop/interop_server.js', '--use_tls=true'] + args
 
   def global_env(self):
     return {}
@@ -216,8 +216,8 @@ class PHPLanguage:
     self.client_cwd = None
     self.safename = str(self)
 
-  def client_args(self):
-    return ['src/php/bin/interop_client.sh']
+  def client_cmd(self, args):
+    return ['src/php/bin/interop_client.sh'] + args
 
   def cloud_to_prod_env(self):
     return _SSL_CERT_ENV
@@ -239,14 +239,14 @@ class RubyLanguage:
     self.server_cwd = None
     self.safename = str(self)
 
-  def client_args(self):
-    return ['ruby', 'src/ruby/bin/interop/interop_client.rb']
+  def client_cmd(self, args):
+    return ['ruby', 'src/ruby/bin/interop/interop_client.rb'] + args
 
   def cloud_to_prod_env(self):
     return _SSL_CERT_ENV
 
-  def server_args(self):
-    return ['ruby', 'src/ruby/bin/interop/interop_server.rb', '--use_tls=true']
+  def server_cmd(self, args):
+    return ['ruby', 'src/ruby/bin/interop/interop_server.rb', '--use_tls=true'] + args
 
   def global_env(self):
     return {}
@@ -265,17 +265,29 @@ class PythonLanguage:
     self.server_cwd = None
     self.safename = str(self)
 
-  def client_args(self):
-    return ['python2.7_virtual_environment/bin/python', '-m', 'grpc_interop.client']
+  def client_cmd(self, args):
+    return [
+        'python2.7_virtual_environment/bin/python',
+        'src/python/grpcio/setup.py',
+        'run_interop',
+        '--client',
+        '--args=\'{}\''.format(' '.join(args))
+    ]
 
   def cloud_to_prod_env(self):
     return _SSL_CERT_ENV
 
-  def server_args(self):
-    return ['python2.7_virtual_environment/bin/python', '-m', 'grpc_interop.server', '--use_tls=true']
+  def server_cmd(self, args):
+    return [
+        'python2.7_virtual_environment/bin/python',
+        'src/python/grpcio/setup.py',
+        'run_interop',
+        '--server',
+        '--args=\'{}\''.format(' '.join(args) + ' --use_tls=true')
+    ]
 
   def global_env(self):
-    return {'LD_LIBRARY_PATH': 'libs/opt'}
+    return {'LD_LIBRARY_PATH': '{}/libs/opt'.format(DOCKER_WORKDIR_ROOT)}
 
   def unimplemented_test_cases(self):
     return ['jwt_token_creds', 'per_rpc_creds']
@@ -308,6 +320,8 @@ _AUTH_TEST_CASES = ['compute_engine_creds', 'jwt_token_creds',
 
 _HTTP2_TEST_CASES = ["tls"]
 
+DOCKER_WORKDIR_ROOT = '/var/local/git/grpc'
+
 def docker_run_cmdline(cmdline, image, docker_args=[], cwd=None, environ=None):
   """Wraps given cmdline array to create 'docker run' cmdline from it."""
   docker_cmdline = ['docker', 'run', '-i', '--rm=true']
@@ -318,7 +332,7 @@ def docker_run_cmdline(cmdline, image, docker_args=[], cwd=None, environ=None):
       docker_cmdline += ['-e', '%s=%s' % (k,v)]
 
   # set working directory
-  workdir = '/var/local/git/grpc'
+  workdir = DOCKER_WORKDIR_ROOT
   if cwd:
     workdir = os.path.join(workdir, cwd)
   docker_cmdline += ['-w', workdir]
@@ -379,12 +393,12 @@ def _job_kill_handler(job):
 
 def cloud_to_prod_jobspec(language, test_case, docker_image=None, auth=False):
   """Creates jobspec for cloud-to-prod interop test"""
-  cmdline = language.client_args() + [
+  cmdline = language.client_cmd([
       '--server_host_override=grpc-test.sandbox.google.com',
       '--server_host=grpc-test.sandbox.google.com',
       '--server_port=443',
       '--use_tls=true',
-      '--test_case=%s' % test_case]
+      '--test_case=%s' % test_case])
   cwd = language.client_cwd
   environ = dict(language.cloud_to_prod_env(), **language.global_env())
   container_name = None
@@ -420,13 +434,13 @@ def cloud_to_prod_jobspec(language, test_case, docker_image=None, auth=False):
 def cloud_to_cloud_jobspec(language, test_case, server_name, server_host,
                            server_port, docker_image=None):
   """Creates jobspec for cloud-to-cloud interop test"""
-  cmdline = bash_login_cmdline(language.client_args() +
-                               ['--server_host_override=foo.test.google.fr',
-                                '--use_tls=true',
-                                '--use_test_ca=true',
-                                '--test_case=%s' % test_case,
-                                '--server_host=%s' % server_host,
-                                '--server_port=%s' % server_port])
+  cmdline = bash_login_cmdline(language.client_cmd([
+      '--server_host_override=foo.test.google.fr',
+      '--use_tls=true',
+      '--use_test_ca=true',
+      '--test_case=%s' % test_case,
+      '--server_host=%s' % server_host,
+      '--server_port=%s' % server_port]))
   cwd = language.client_cwd
   environ = language.global_env()
   if docker_image:
@@ -456,8 +470,8 @@ def cloud_to_cloud_jobspec(language, test_case, server_name, server_host,
 def server_jobspec(language, docker_image):
   """Create jobspec for running a server"""
   container_name = dockerjob.random_name('interop_server_%s' % language.safename)
-  cmdline = bash_login_cmdline(language.server_args() +
-                               ['--port=%s' % _DEFAULT_SERVER_PORT])
+  cmdline = bash_login_cmdline(
+      language.server_cmd(['--port=%s' % _DEFAULT_SERVER_PORT]))
   environ = language.global_env()
   docker_cmdline = docker_run_cmdline(cmdline,
                                       image=docker_image,
diff --git a/tools/run_tests/run_python.sh b/tools/run_tests/run_python.sh
index 848775e9b1..9bdfbfc60c 100755
--- a/tools/run_tests/run_python.sh
+++ b/tools/run_tests/run_python.sh
@@ -34,10 +34,31 @@ set -ex
 cd $(dirname $0)/../..
 
 ROOT=`pwd`
-GRPCIO_TEST=$ROOT/src/python/grpcio_test
+GRPCIO=$ROOT/src/python/grpcio
 export LD_LIBRARY_PATH=$ROOT/libs/$CONFIG
 export DYLD_LIBRARY_PATH=$ROOT/libs/$CONFIG
 export PATH=$ROOT/bins/$CONFIG:$ROOT/bins/$CONFIG/protobuf:$PATH
-source "python"$PYVER"_virtual_environment"/bin/activate
+export CFLAGS="-I$ROOT/include -std=c89"
+export LDFLAGS="-L$ROOT/libs/$CONFIG"
+export GRPC_PYTHON_BUILD_WITH_CYTHON=1
+export GRPC_PYTHON_ENABLE_CYTHON_TRACING=1
 
-"python"$PYVER $GRPCIO_TEST/setup.py test -a "-n8 --cov=grpc --junitxml=./report.xml --timeout=300 -v --boxed --timeout_method=thread"
+VIRTUALENV=python"$PYVER"_virtual_environment
+source $VIRTUALENV/bin/activate
+
+(rm $GRPCIO/.coverage)   || true
+(rm $GRPCIO/.coverage.*) || true
+
+if python -u $GRPCIO/setup.py test; then
+  EXIT_CODE=0
+else
+  EXIT_CODE=$?
+fi
+
+cp $GRPCIO/report.xml $ROOT
+
+cd $GRPCIO
+(coverage combine) || true
+(coverage report --include='grpc/*' --omit='grpc/framework/alpha/*','grpc/early_adopter/*','grpc/framework/base/*''grpc/framework/face/*') || true
+
+exit $EXIT_CODE