diff options
author | 2015-07-10 10:23:10 -0700 | |
---|---|---|
committer | 2015-07-10 10:23:10 -0700 | |
commit | 080d6c50bbed3dde739b2de610023d8bd8d7966f (patch) | |
tree | 647488fcbb5e1be2ef9d0f36f75b6d3382700623 /tools | |
parent | 75105868f667443ac059b7c4f779697932aca509 (diff) | |
parent | a9a8eb70604b5da61f8e768bc52192910ba60251 (diff) |
Merge github.com:grpc/grpc into footprints-on-the-sands-of-time
Diffstat (limited to 'tools')
-rwxr-xr-x | tools/buildgen/generate_projects.py | 77 | ||||
-rwxr-xr-x | tools/buildgen/generate_projects.sh | 28 | ||||
-rwxr-xr-x | tools/distrib/python/submit.py | 6 | ||||
-rw-r--r-- | tools/doxygen/Doxyfile.c++ | 1 | ||||
-rw-r--r-- | tools/doxygen/Doxyfile.c++.internal | 5 | ||||
-rwxr-xr-x | tools/run_tests/build_python.sh | 35 | ||||
-rwxr-xr-x | tools/run_tests/jobset.py | 1 | ||||
-rwxr-xr-x | tools/run_tests/python_tests.json | 103 | ||||
-rw-r--r-- | tools/run_tests/run_csharp.bat | 5 | ||||
-rwxr-xr-x | tools/run_tests/run_python.sh | 4 | ||||
-rwxr-xr-x | tools/run_tests/run_tests.py | 56 | ||||
-rw-r--r-- | tools/run_tests/sources_and_headers.json | 26 | ||||
-rw-r--r-- | tools/run_tests/tests.json | 9 |
13 files changed, 288 insertions, 68 deletions
diff --git a/tools/buildgen/generate_projects.py b/tools/buildgen/generate_projects.py new file mode 100755 index 0000000000..1964ceb665 --- /dev/null +++ b/tools/buildgen/generate_projects.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python2.7 + +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import glob +import os +import sys +import tempfile +sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..', 'run_tests')) + +assert sys.argv[1:], 'run generate_projects.sh instead of this directly' + +import jobset + +os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '..', '..')) +json = sys.argv[1:] + +test = {} if 'TEST' in os.environ else None + +plugins = sorted(glob.glob('tools/buildgen/plugins/*.py')) + +jobs = [] +for root, dirs, files in os.walk('templates'): + for f in files: + if os.path.splitext(f)[1] == '.template': + out = '.' + root[len('templates'):] + '/' + os.path.splitext(f)[0] + cmd = ['tools/buildgen/mako_renderer.py'] + for plugin in plugins: + cmd.append('-p') + cmd.append(plugin) + for js in json: + cmd.append('-d') + cmd.append(js) + cmd.append('-o') + if test is None: + cmd.append(out) + else: + tf = tempfile.mkstemp() + test[out] = tf[1] + os.close(tf[0]) + cmd.append(test[out]) + cmd.append(root + '/' + f) + jobs.append(jobset.JobSpec(cmd, shortname=out)) + +jobset.run(jobs) + +if test is not None: + for s, g in test.iteritems(): + assert(0 == os.system('diff %s %s' % (s, g))) + os.unlink(g) diff --git a/tools/buildgen/generate_projects.sh b/tools/buildgen/generate_projects.sh index 5399867746..32fc90fef5 100755 --- a/tools/buildgen/generate_projects.sh +++ b/tools/buildgen/generate_projects.sh @@ -45,32 +45,6 @@ fi . tools/buildgen/generate_build_additions.sh -global_plugins=`find ./tools/buildgen/plugins -name '*.py' | - sort | grep -v __init__ | awk ' { printf "-p %s ", $0 } '` - -for dir in . ; do - local_plugins=`find $dir/templates -name '*.py' | - sort | grep -v __init__ | awk ' { printf "-p %s ", $0 } '` - - plugins="$global_plugins $local_plugins" - - find -L $dir/templates -type f -and -name *.template | while read file ; do - out=${dir}/${file#$dir/templates/} # strip templates dir prefix - out=${out%.*} # strip template extension - echo "generating file: $out" - json_files="build.json $gen_build_files" - data=`for i in $json_files ; do echo $i ; done | awk ' { printf "-d %s ", $0 } '` - if [ "x$TEST" = "xtrue" ] ; then - actual_out=$out - out=`mktemp /tmp/gentXXXXXX` - fi - mkdir -p `dirname $out` # make sure dest directory exist - $mako_renderer $plugins $data -o $out $file - if [ "x$TEST" = "xtrue" ] ; then - diff -q $out $actual_out - rm $out - fi - done -done +tools/buildgen/generate_projects.py build.json $gen_build_files rm $gen_build_files diff --git a/tools/distrib/python/submit.py b/tools/distrib/python/submit.py index dd48f440ba..a3615b3640 100755 --- a/tools/distrib/python/submit.py +++ b/tools/distrib/python/submit.py @@ -66,6 +66,12 @@ try: except: pass +# Build the Cython C files +build_env = os.environ.copy() +build_env['GRPC_PYTHON_BUILD_WITH_CYTHON'] = "1" +cmd = ['python', 'setup.py', 'build_ext', '--inplace'] +subprocess.call(cmd, cwd=pkgdir, env=build_env) + # Make the push. cmd = ['python', 'setup.py', 'sdist'] subprocess.call(cmd, cwd=pkgdir) diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index d782dc18f7..feb7ad8bb9 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -762,6 +762,7 @@ WARN_LOGFILE = INPUT = include/grpc++/async_generic_service.h \ include/grpc++/async_unary_call.h \ +include/grpc++/auth_context.h \ include/grpc++/byte_buffer.h \ include/grpc++/channel_arguments.h \ include/grpc++/channel_interface.h \ diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 575cd95753..67718d8976 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -762,6 +762,7 @@ WARN_LOGFILE = INPUT = include/grpc++/async_generic_service.h \ include/grpc++/async_unary_call.h \ +include/grpc++/auth_context.h \ include/grpc++/byte_buffer.h \ include/grpc++/channel_arguments.h \ include/grpc++/channel_interface.h \ @@ -798,11 +799,15 @@ include/grpc++/stream.h \ include/grpc++/thread_pool_interface.h \ include/grpc++/time.h \ src/cpp/client/secure_credentials.h \ +src/cpp/common/secure_auth_context.h \ src/cpp/server/secure_server_credentials.h \ src/cpp/client/channel.h \ +src/cpp/common/create_auth_context.h \ src/cpp/server/thread_pool.h \ src/cpp/client/secure_channel_arguments.cc \ src/cpp/client/secure_credentials.cc \ +src/cpp/common/secure_auth_context.cc \ +src/cpp/common/secure_create_auth_context.cc \ src/cpp/server/secure_server_credentials.cc \ src/cpp/client/channel.cc \ src/cpp/client/channel_arguments.cc \ diff --git a/tools/run_tests/build_python.sh b/tools/run_tests/build_python.sh index d9b7644f44..ae0fb42241 100755 --- a/tools/run_tests/build_python.sh +++ b/tools/run_tests/build_python.sh @@ -34,9 +34,32 @@ set -ex cd $(dirname $0)/../.. root=`pwd` -rm -rf python2.7_virtual_environment -virtualenv -p /usr/bin/python2.7 python2.7_virtual_environment -source python2.7_virtual_environment/bin/activate -pip install -r src/python/requirements.txt -CFLAGS="-I$root/include -std=c89 -Werror" LDFLAGS=-L$root/libs/$CONFIG pip install src/python/src -pip install src/python/interop + +make_virtualenv() { + virtualenv_name="python"$1"_virtual_environment" + if [ ! -d $virtualenv_name ] + then + # Build the entire virtual environment + virtualenv -p `which "python"$1` $virtualenv_name + source $virtualenv_name/bin/activate + pip install -r src/python/requirements.txt + CFLAGS="-I$root/include -std=c89" LDFLAGS=-L$root/libs/$CONFIG GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install src/python/src + pip install src/python/interop + else + source $virtualenv_name/bin/activate + # Uninstall and re-install the packages we care about. Don't use + # --force-reinstall or --ignore-installed to avoid propagating this + # unnecessarily to dependencies. Don't use --no-deps to avoid missing + # dependency upgrades. + (yes | pip uninstall grpcio) || true + (yes | pip uninstall interop) || true + (CFLAGS="-I$root/include -std=c89" LDFLAGS=-L$root/libs/$CONFIG GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install src/python/src) || ( + # Fall back to rebuilding the entire environment + rm -rf $virtualenv_name + make_virtualenv $1 + ) + pip install src/python/interop + fi +} + +make_virtualenv $1 diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py index baa126ba5f..e5138147d3 100755 --- a/tools/run_tests/jobset.py +++ b/tools/run_tests/jobset.py @@ -81,6 +81,7 @@ _CLEAR_LINE = '\x1b[2K' _TAG_COLOR = { 'FAILED': 'red', + 'WARNING': 'yellow', 'TIMEOUT': 'red', 'PASSED': 'green', 'START': 'gray', diff --git a/tools/run_tests/python_tests.json b/tools/run_tests/python_tests.json index 6c969d765f..4da4c1b68b 100755 --- a/tools/run_tests/python_tests.json +++ b/tools/run_tests/python_tests.json @@ -1,56 +1,123 @@ [ { - "module": "grpc._adapter._c_test" + "module": "grpc._cython.cygrpc_test", + "pythonVersions": [ + "2.7", + "3.4" + ] }, { - "module": "grpc._adapter._low_test" + "module": "grpc._cython.adapter_low_test", + "pythonVersions": [ + "2.7" + ] }, { - "module": "grpc._adapter._intermediary_low_test" + "module": "grpc._adapter._c_test", + "pythonVersions": [ + "2.7" + ] }, { - "module": "grpc._adapter._links_test" + "module": "grpc._adapter._low_test", + "pythonVersions": [ + "2.7" + ] }, { - "module": "grpc._adapter._lonely_rear_link_test" + "module": "grpc._adapter._intermediary_low_test", + "pythonVersions": [ + "2.7" + ] }, { - "module": "grpc._adapter._blocking_invocation_inline_service_test" + "module": "grpc._adapter._links_test", + "pythonVersions": [ + "2.7" + ] }, { - "module": "grpc._adapter._event_invocation_synchronous_event_service_test" + "module": "grpc._adapter._lonely_rear_link_test", + "pythonVersions": [ + "2.7" + ] }, { - "module": "grpc._adapter._future_invocation_asynchronous_event_service_test" + "module": "grpc._adapter._blocking_invocation_inline_service_test", + "pythonVersions": [ + "2.7" + ] }, { - "module": "grpc.early_adopter.implementations_test" + "module": "grpc._adapter._event_invocation_synchronous_event_service_test", + "pythonVersions": [ + "2.7" + ] }, { - "module": "grpc.framework.base.implementations_test" + "module": "grpc._adapter._future_invocation_asynchronous_event_service_test", + "pythonVersions": [ + "2.7" + ] }, { - "module": "grpc.framework.face.blocking_invocation_inline_service_test" + "module": "grpc.early_adopter.implementations_test", + "pythonVersions": [ + "2.7" + ] }, { - "module": "grpc.framework.face.event_invocation_synchronous_event_service_test" + "module": "grpc.framework.base.implementations_test", + "pythonVersions": [ + "2.7" + ] }, { - "module": "grpc.framework.face.future_invocation_asynchronous_event_service_test" + "module": "grpc.framework.face.blocking_invocation_inline_service_test", + "pythonVersions": [ + "2.7" + ] }, { - "module": "grpc.framework.foundation._later_test" + "module": "grpc.framework.face.event_invocation_synchronous_event_service_test", + "pythonVersions": [ + "2.7" + ] }, { - "module": "grpc.framework.foundation._logging_pool_test" + "module": "grpc.framework.face.future_invocation_asynchronous_event_service_test", + "pythonVersions": [ + "2.7" + ] }, { - "module": "interop._insecure_interop_test" + "module": "grpc.framework.foundation._later_test", + "pythonVersions": [ + "2.7" + ] }, { - "module": "interop._secure_interop_test" + "module": "grpc.framework.foundation._logging_pool_test", + "pythonVersions": [ + "2.7" + ] }, { - "file": "test/compiler/python_plugin_test.py" + "module": "interop._insecure_interop_test", + "pythonVersions": [ + "2.7" + ] + }, + { + "module": "interop._secure_interop_test", + "pythonVersions": [ + "2.7" + ] + }, + { + "file": "test/compiler/python_plugin_test.py", + "pythonVersions": [ + "2.7" + ] } ] diff --git a/tools/run_tests/run_csharp.bat b/tools/run_tests/run_csharp.bat index 17c622cc2d..c86136767c 100644 --- a/tools/run_tests/run_csharp.bat +++ b/tools/run_tests/run_csharp.bat @@ -5,7 +5,10 @@ setlocal @rem enter this directory cd /d %~dp0\..\..\src\csharp -packages\NUnit.Runners.2.6.4\tools\nunit-console-x86.exe -labels "%1/bin/Debug/%1.dll" || goto :error +@rem set UUID variable to a random GUID, we will use it to put TestResults.xml to a dedicated directory, so that parallel test runs don't collide +for /F %%i in ('powershell -Command "[guid]::NewGuid().ToString()"') do (set UUID=%%i) + +packages\NUnit.Runners.2.6.4\tools\nunit-console-x86.exe -labels "%1/bin/Debug/%1.dll" -work test-results/%UUID% || goto :error endlocal diff --git a/tools/run_tests/run_python.sh b/tools/run_tests/run_python.sh index cab08f9358..4959c0241c 100755 --- a/tools/run_tests/run_python.sh +++ b/tools/run_tests/run_python.sh @@ -36,5 +36,5 @@ cd $(dirname $0)/../.. root=`pwd` export LD_LIBRARY_PATH=$root/libs/$CONFIG export DYLD_LIBRARY_PATH=$root/libs/$CONFIG -source python2.7_virtual_environment/bin/activate -python2.7 -B $* +source "python"$PYVER"_virtual_environment"/bin/activate +"python"$PYVER -B $* diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index fd90613ad6..1f44fc34fa 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -189,27 +189,55 @@ class PythonLanguage(object): def __init__(self): with open('tools/run_tests/python_tests.json') as f: self._tests = json.load(f) + self._build_python_versions = set([ + python_version + for test in self._tests + for python_version in test['pythonVersions']]) + self._has_python_versions = [] def test_specs(self, config, travis): - modules = [config.job_spec(['tools/run_tests/run_python.sh', '-m', - test['module']], - None, - environ=_FORCE_ENVIRON_FOR_WRAPPERS, - shortname=test['module']) - for test in self._tests if 'module' in test] - files = [config.job_spec(['tools/run_tests/run_python.sh', - test['file']], - None, - environ=_FORCE_ENVIRON_FOR_WRAPPERS, - shortname=test['file']) - for test in self._tests if 'file' in test] - return files + modules + job_specifications = [] + for test in self._tests: + command = None + short_name = None + if 'module' in test: + command = ['tools/run_tests/run_python.sh', '-m', test['module']] + short_name = test['module'] + elif 'file' in test: + command = ['tools/run_tests/run_python.sh', test['file']] + short_name = test['file'] + else: + raise ValueError('expected input to be a module or file to run ' + 'unittests from') + for python_version in test['pythonVersions']: + if python_version in self._has_python_versions: + environment = dict(_FORCE_ENVIRON_FOR_WRAPPERS) + environment['PYVER'] = python_version + job_specifications.append(config.job_spec( + command, None, environ=environment, shortname=short_name)) + else: + jobset.message( + 'WARNING', + 'Could not find Python {}; skipping test'.format(python_version), + '{}\n'.format(command), do_newline=True) + return job_specifications def make_targets(self): return ['static_c', 'grpc_python_plugin', 'shared_c'] def build_steps(self): - return [['tools/run_tests/build_python.sh']] + commands = [] + for python_version in self._build_python_versions: + try: + with open(os.devnull, 'w') as output: + subprocess.check_call(['which', 'python' + python_version], + stdout=output, stderr=output) + commands.append(['tools/run_tests/build_python.sh', python_version]) + self._has_python_versions.append(python_version) + except: + jobset.message('WARNING', 'Missing Python ' + python_version, + do_newline=True) + return commands def supports_multi_config(self): return False diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index 3f13b7b9ad..1fc5c20fe4 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -1486,6 +1486,19 @@ { "deps": [ "gpr", + "grpc", + "grpc++" + ], + "headers": [], + "language": "c++", + "name": "secure_auth_context_test", + "src": [ + "test/cpp/common/secure_auth_context_test.cc" + ] + }, + { + "deps": [ + "gpr", "gpr_test_util", "grpc", "grpc++", @@ -9475,6 +9488,7 @@ "headers": [ "include/grpc++/async_generic_service.h", "include/grpc++/async_unary_call.h", + "include/grpc++/auth_context.h", "include/grpc++/byte_buffer.h", "include/grpc++/channel_arguments.h", "include/grpc++/channel_interface.h", @@ -9512,6 +9526,8 @@ "include/grpc++/time.h", "src/cpp/client/channel.h", "src/cpp/client/secure_credentials.h", + "src/cpp/common/create_auth_context.h", + "src/cpp/common/secure_auth_context.h", "src/cpp/server/secure_server_credentials.h", "src/cpp/server/thread_pool.h" ], @@ -9520,6 +9536,7 @@ "src": [ "include/grpc++/async_generic_service.h", "include/grpc++/async_unary_call.h", + "include/grpc++/auth_context.h", "include/grpc++/byte_buffer.h", "include/grpc++/channel_arguments.h", "include/grpc++/channel_interface.h", @@ -9569,7 +9586,11 @@ "src/cpp/client/secure_credentials.h", "src/cpp/common/call.cc", "src/cpp/common/completion_queue.cc", + "src/cpp/common/create_auth_context.h", "src/cpp/common/rpc_method.cc", + "src/cpp/common/secure_auth_context.cc", + "src/cpp/common/secure_auth_context.h", + "src/cpp/common/secure_create_auth_context.cc", "src/cpp/proto/proto_utils.cc", "src/cpp/server/async_generic_service.cc", "src/cpp/server/create_default_thread_pool.cc", @@ -9638,6 +9659,7 @@ "headers": [ "include/grpc++/async_generic_service.h", "include/grpc++/async_unary_call.h", + "include/grpc++/auth_context.h", "include/grpc++/byte_buffer.h", "include/grpc++/channel_arguments.h", "include/grpc++/channel_interface.h", @@ -9674,6 +9696,7 @@ "include/grpc++/thread_pool_interface.h", "include/grpc++/time.h", "src/cpp/client/channel.h", + "src/cpp/common/create_auth_context.h", "src/cpp/server/thread_pool.h" ], "language": "c++", @@ -9681,6 +9704,7 @@ "src": [ "include/grpc++/async_generic_service.h", "include/grpc++/async_unary_call.h", + "include/grpc++/auth_context.h", "include/grpc++/byte_buffer.h", "include/grpc++/channel_arguments.h", "include/grpc++/channel_interface.h", @@ -9727,6 +9751,8 @@ "src/cpp/client/internal_stub.cc", "src/cpp/common/call.cc", "src/cpp/common/completion_queue.cc", + "src/cpp/common/create_auth_context.h", + "src/cpp/common/insecure_create_auth_context.cc", "src/cpp/common/rpc_method.cc", "src/cpp/proto/proto_utils.cc", "src/cpp/server/async_generic_service.cc", diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index 12b1f22b12..cc05870640 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -723,6 +723,15 @@ { "flaky": false, "language": "c++", + "name": "secure_auth_context_test", + "platforms": [ + "windows", + "posix" + ] + }, + { + "flaky": false, + "language": "c++", "name": "server_crash_test", "platforms": [ "windows", |