diff options
| -rw-r--r-- | PYTHON-MANIFEST.in | 1 | ||||
| -rwxr-xr-x | tools/run_tests/build_python.sh | 71 | ||||
| -rwxr-xr-x | tools/run_tests/performance/run_worker_python.sh | 2 | ||||
| -rwxr-xr-x | tools/run_tests/run_interop_tests.py | 14 | ||||
| -rwxr-xr-x | tools/run_tests/run_python.sh | 17 | ||||
| -rwxr-xr-x | tools/run_tests/run_tests.py | 60 |
6 files changed, 114 insertions, 51 deletions
diff --git a/PYTHON-MANIFEST.in b/PYTHON-MANIFEST.in index 3ebba6ec3f..635e77b875 100644 --- a/PYTHON-MANIFEST.in +++ b/PYTHON-MANIFEST.in @@ -1,6 +1,5 @@ recursive-include src/python/grpcio/grpc *.c *.h *.py *.pyx *.pxd *.pxi *.python *.pem recursive-exclude src/python/grpcio/grpc/_cython *.so *.pyd -graft src/python/grpcio/tests graft src/python/grpcio/grpcio.egg-info graft src/core graft src/boringssl diff --git a/tools/run_tests/build_python.sh b/tools/run_tests/build_python.sh index 3b1c7d2d40..9d2813fa19 100755 --- a/tools/run_tests/build_python.sh +++ b/tools/run_tests/build_python.sh @@ -33,25 +33,70 @@ set -ex # change to grpc repo root cd $(dirname $0)/../.. -TOX_PYTHON_ENV="$1" -PY_VERSION="${TOX_PYTHON_ENV: -2}" +PYTHON=${1:-python2.7} +VENV=${2:-py27} +VENV_RELATIVE_PYTHON=${3:-bin/python} +TOOLCHAIN=${4:-unix} ROOT=`pwd` export CFLAGS="-I$ROOT/include -std=gnu99 -fno-wrapv" export GRPC_PYTHON_BUILD_WITH_CYTHON=1 -if [ "$CONFIG" = "gcov" ] -then +# If ccache is available, use it... unless we're on Mac, then all hell breaks +# loose because Python does hacky things to support other hacky things done to +# hacky things on Mac OS X +PLATFORM=`uname -s` +if [ "${PLATFORM/Darwin}" = "$PLATFORM" ]; then + # We're not on Darwin (Mac OS X) + if [ -x "$(command -v ccache)" ]; then + if [ -x "$(command -v gcc)" ]; then + export CC='ccache gcc' + elif [ -x "$(command -v clang)" ]; then + export CC='ccache clang' + fi + fi +fi + +# Find `realpath` +if [ -x "$(command -v realpath)" ]; then + export REALPATH=realpath +elif [ -x "$(command -v grealpath)" ]; then + export REALPATH=grealpath +else + echo 'Couldn'"'"'t find `realpath` or `grealpath`' + exit 1 +fi + +if [ "$CONFIG" = "gcov" ]; then export GRPC_PYTHON_ENABLE_CYTHON_TRACING=1 fi -tox -e ${TOX_PYTHON_ENV} --notest +($PYTHON -m virtualenv $VENV || true) +VENV_PYTHON=`$REALPATH -s "$VENV/$VENV_RELATIVE_PYTHON"` + +# pip-installs the directory specified. Used because on MSYS the vanilla Windows +# Python gets confused when parsing paths. +pip_install_dir() { + PWD=`pwd` + cd $1 + ($VENV_PYTHON setup.py build_ext -c $TOOLCHAIN || true) + # install the dependencies + $VENV_PYTHON -m pip install --upgrade . + # ensure that we've reinstalled the test packages + $VENV_PYTHON -m pip install --upgrade --force-reinstall --no-deps . + cd $PWD +} -$ROOT/.tox/${TOX_PYTHON_ENV}/bin/pip install cython -$ROOT/.tox/${TOX_PYTHON_ENV}/bin/pip install $ROOT -$ROOT/.tox/${TOX_PYTHON_ENV}/bin/python $ROOT/tools/distrib/python/make_grpcio_tools.py -$ROOT/.tox/${TOX_PYTHON_ENV}/bin/pip install $ROOT/tools/distrib/python/grpcio_tools -$ROOT/.tox/${TOX_PYTHON_ENV}/bin/python $ROOT/src/python/grpcio_health_checking/setup.py preprocess -$ROOT/.tox/${TOX_PYTHON_ENV}/bin/pip install $ROOT/src/python/grpcio_health_checking -$ROOT/.tox/${TOX_PYTHON_ENV}/bin/python $ROOT/src/python/grpcio_tests/setup.py preprocess -$ROOT/.tox/${TOX_PYTHON_ENV}/bin/python $ROOT/src/python/grpcio_tests/setup.py build_proto_modules +$VENV_PYTHON -m pip install --upgrade pip setuptools +$VENV_PYTHON -m pip install cython +pip_install_dir $ROOT +$VENV_PYTHON $ROOT/tools/distrib/python/make_grpcio_tools.py +pip_install_dir $ROOT/tools/distrib/python/grpcio_tools +# TODO(atash) figure out namespace packages and grpcio-tools and auditwheel +# etc... +pip_install_dir $ROOT +$VENV_PYTHON $ROOT/src/python/grpcio_health_checking/setup.py preprocess +pip_install_dir $ROOT/src/python/grpcio_health_checking +$VENV_PYTHON $ROOT/src/python/grpcio_tests/setup.py preprocess +$VENV_PYTHON $ROOT/src/python/grpcio_tests/setup.py build_proto_modules +pip_install_dir $ROOT/src/python/grpcio_tests diff --git a/tools/run_tests/performance/run_worker_python.sh b/tools/run_tests/performance/run_worker_python.sh index 2bca5e1695..3b8ba6f4e4 100755 --- a/tools/run_tests/performance/run_worker_python.sh +++ b/tools/run_tests/performance/run_worker_python.sh @@ -32,4 +32,4 @@ set -ex cd $(dirname $0)/../../.. -PYTHONPATH=src/python/grpcio_tests:src/python/grpcio:src/python/gens .tox/py27/bin/python src/python/grpcio_tests/tests/qps/qps_worker.py $@ +PYTHONPATH=src/python/grpcio_tests:src/python/grpcio:src/python/gens py27/bin/python src/python/grpcio_tests/tests/qps/qps_worker.py $@ diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index adf9adaf13..13a4a49325 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -304,8 +304,11 @@ class PythonLanguage: def client_cmd(self, args): return [ - 'tox -einterop_client --', - ' '.join(args) + 'py27/bin/python', + 'src/python/grpcio_tests/setup.py', + 'run_interop', + '--client', + '--args="{}"'.format(' '.join(args)) ] def cloud_to_prod_env(self): @@ -313,8 +316,11 @@ class PythonLanguage: def server_cmd(self, args): return [ - 'tox -einterop_server --', - ' '.join(args) + ' --use_tls=true' + 'py27/bin/python', + 'src/python/grpcio_tests/setup.py', + 'run_interop', + '--server', + '--args="{}"'.format(' '.join(args) + ' --use_tls=true') ] def global_env(self): diff --git a/tools/run_tests/run_python.sh b/tools/run_tests/run_python.sh index b05efd1c76..17e0186f2a 100755 --- a/tools/run_tests/run_python.sh +++ b/tools/run_tests/run_python.sh @@ -33,24 +33,11 @@ set -ex # change to grpc repo root cd $(dirname $0)/../.. -TOX_PYTHON_ENV="$1" +PYTHON=`realpath -s "${1:-py27/bin/python}"` ROOT=`pwd` -export LD_LIBRARY_PATH=$ROOT/libs/$CONFIG -export DYLD_LIBRARY_PATH=$ROOT/libs/$CONFIG -export PATH=$ROOT/bins/$CONFIG:$ROOT/bins/$CONFIG/protobuf:$PATH -export CFLAGS="-I$ROOT/include -std=c89" -export LDFLAGS="-L$ROOT/libs/$CONFIG" -export GRPC_PYTHON_BUILD_WITH_CYTHON=1 -export GRPC_PYTHON_USE_PRECOMPILED_BINARIES=0 -if [ "$CONFIG" = "gcov" ] -then - export GRPC_PYTHON_ENABLE_CYTHON_TRACING=1 - tox -e ${TOX_PYTHON_ENV} -else - $ROOT/.tox/${TOX_PYTHON_ENV}/bin/python $ROOT/src/python/grpcio_tests/setup.py test_lite -fi +$PYTHON $ROOT/src/python/grpcio_tests/setup.py test_lite mkdir -p $ROOT/reports rm -rf $ROOT/reports/python-coverage diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index f32a621ee4..fbc5729c75 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -32,11 +32,13 @@ import argparse import ast +import collections import glob import itertools import json import multiprocessing import os +import os.path import platform import random import re @@ -372,12 +374,20 @@ class PhpLanguage(object): return 'php' +class PythonConfig(collections.namedtuple('PythonConfig', [ + 'python', 'venv', 'venv_relative_python', 'toolchain',])): + + @property + def venv_python(self): + return os.path.abspath('{}/{}'.format(self.venv, self.venv_relative_python)) + + class PythonLanguage(object): def configure(self, config, args): self.config = config self.args = args - self._tox_envs = self._get_tox_envs(self.args.compiler) + self.pythons = self._get_pythons(self.args.compiler) def test_specs(self): # load list of known test suites @@ -386,33 +396,42 @@ class PythonLanguage(object): environment = dict(_FORCE_ENVIRON_FOR_WRAPPERS) if self.config.build_config != 'gcov': return [self.config.job_spec( - ['tools/run_tests/run_python.sh', tox_env], + ['tools/run_tests/run_python.sh', config.venv_python], + None, environ=dict(environment.items() + [('GRPC_PYTHON_TESTRUNNER_FILTER', suite_name)]), - shortname='%s.test.%s' % (tox_env, suite_name), + shortname='%s.test.%s' % (config.venv, suite_name), timeout_seconds=5*60) for suite_name in tests_json - for tox_env in self._tox_envs] + for config in self.pythons] else: - return [self.config.job_spec(['tools/run_tests/run_python.sh', tox_env], - environ=environment, - shortname='%s.test.coverage' % tox_env, - timeout_seconds=15*60) - for tox_env in self._tox_envs] + return [self.config.job_spec( + ['tools/run_tests/run_python.sh', config.venv_python], + None, + environ=environment, + shortname='%s.test.coverage' % config.venv, + timeout_seconds=15*60) + for config in self.pythons] def pre_build_steps(self): return [] def make_targets(self): - return ['static_c', 'grpc_python_plugin', 'shared_c'] + return [] def make_options(self): return [] def build_steps(self): - return [['tools/run_tests/build_python.sh', tox_env] - for tox_env in self._tox_envs] + return [ + [ + 'tools/run_tests/build_python.sh', + config.python, config.venv, + config.venv_relative_python, config.toolchain + ] + for config in self.pythons + ] def post_tests_steps(self): return [] @@ -423,14 +442,21 @@ class PythonLanguage(object): def dockerfile_dir(self): return 'tools/dockerfile/test/python_jessie_%s' % _docker_arch_suffix(self.args.arch) - def _get_tox_envs(self, compiler): - """Returns name of tox environment based on selected compiler.""" + def _get_pythons(self, compiler): + if os.name == 'nt': + venv_relative_python = 'Scripts/python.exe' + toolchain = 'mingw32' + else: + venv_relative_python = 'bin/python' + toolchain = 'unix' + python27_config = PythonConfig('python2.7', 'py27', venv_relative_python, toolchain) + python34_config = PythonConfig('python3.4', 'py34', venv_relative_python, toolchain) if compiler == 'default': - return ('py27', 'py34') + return (python27_config, python34_config,) elif compiler == 'python2.7': - return ('py27',) + return (python27_config,) elif compiler == 'python3.4': - return ('py34',) + return (python34_config,) else: raise Exception('Compiler %s not supported.' % compiler) |