diff options
Diffstat (limited to 'tools')
24 files changed, 678 insertions, 46 deletions
diff --git a/tools/buildgen/plugins/transitive_dependencies.py b/tools/buildgen/plugins/transitive_dependencies.py index c2d3da3a3b..01e7f61ea9 100644 --- a/tools/buildgen/plugins/transitive_dependencies.py +++ b/tools/buildgen/plugins/transitive_dependencies.py @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -36,10 +36,13 @@ of the list of dependencies. """ def get_lib(libs, name): - return next(lib for lib in libs if lib['name']==name) + try: + return next(lib for lib in libs if lib['name']==name) + except StopIteration: + return None def transitive_deps(lib, libs): - if 'deps' in lib: + if lib is not None and 'deps' in lib: # Recursively call transitive_deps on each dependency, and take the union return set.union(set(lib['deps']), *[set(transitive_deps(get_lib(libs, dep), libs)) @@ -58,6 +61,10 @@ def mako_plugin(dictionary): node_modules = dictionary.get('node_modules') targets = dictionary.get('targets') - for target_list in (libs, node_modules, targets): + for target_list in (libs, targets, node_modules): for target in target_list: target['transitive_deps'] = transitive_deps(target, libs) + + python_dependencies = dictionary.get('python_dependencies') + python_dependencies['transitive_deps'] = ( + transitive_deps(python_dependencies, libs)) diff --git a/tools/distrib/check_copyright.py b/tools/distrib/check_copyright.py index f54e5fad80..0c0669083a 100755 --- a/tools/distrib/check_copyright.py +++ b/tools/distrib/check_copyright.py @@ -54,6 +54,9 @@ argp.add_argument('-a', '--ancient', default=0, action='store_const', const=1) +argp.add_argument('-f', '--fix', + default=False, + action='store_true'); args = argp.parse_args() # open the license text @@ -90,7 +93,7 @@ KNOWN_BAD = set([ ]) -RE_YEAR = r'Copyright (?:[0-9]+\-)?([0-9]+), Google Inc\.' +RE_YEAR = r'Copyright (?P<first_year>[0-9]+\-)?(?P<last_year>[0-9]+), Google Inc\.' RE_LICENSE = dict( (k, r'\n'.join( LICENSE_PREFIX[k] + @@ -101,8 +104,11 @@ RE_LICENSE = dict( def load(name): with open(name) as f: - return '\n'.join(line.rstrip() for line in f.read().splitlines()) + return f.read() +def save(name, text): + with open(name, 'w') as f: + f.write(text) assert(re.search(RE_LICENSE['LICENSE'], load('LICENSE'))) assert(re.search(RE_LICENSE['Makefile'], load('Makefile'))) @@ -117,6 +123,7 @@ def log(cond, why, filename): # scan files, validate the text +ok = True for filename in subprocess.check_output('git ls-tree -r --name-only -r HEAD', shell=True).splitlines(): if filename in KNOWN_BAD: continue @@ -130,17 +137,27 @@ for filename in subprocess.check_output('git ls-tree -r --name-only -r HEAD', log(args.skips, 'skip', filename) continue text = load(filename) - ok = True m = re.search(re_license, text) if m: + gdict = m.groupdict() last_modified = int(subprocess.check_output('git log -1 --format="%ad" --date=short -- ' + filename, shell=True)[0:4]) - latest_claimed = int(m.group(1)) + latest_claimed = int(gdict['last_year']) if last_modified > latest_claimed: print '%s modified %d but copyright only extends to %d' % (filename, last_modified, latest_claimed) ok = False + if args.fix: + span_start, span_end = m.span(2) + if not gdict['first_year']: + # prepend the old year to the current one. + text = '{}-{}{}'.format(text[:span_end], last_modified, text[span_end:]) + else: # already a year range + # simply update the last year + text = '{}{}{}'.format(text[:span_start], last_modified, text[span_end:]) + save(filename, text) + print 'Fixed!' + ok = True elif 'DO NOT EDIT' not in text and 'AssemblyInfo.cs' not in filename and filename != 'src/boringssl/err_data.c': log(1, 'copyright missing', filename) ok = False sys.exit(0 if ok else 1) - diff --git a/tools/distrib/clang_format_code.sh b/tools/distrib/clang_format_code.sh index 612074acdf..6bfa278cae 100755 --- a/tools/distrib/clang_format_code.sh +++ b/tools/distrib/clang_format_code.sh @@ -38,4 +38,3 @@ docker build -t grpc_clang_format tools/dockerfile/grpc_clang_format # run clang-format against the checked out codebase docker run -e TEST=$TEST --rm=true -v ${HOST_GIT_ROOT:-`pwd`}:/local-code -t grpc_clang_format /clang_format_all_the_things.sh - diff --git a/tools/distrib/python/submit.py b/tools/distrib/python/submit.py index dffbefd5fe..9b012be672 100755 --- a/tools/distrib/python/submit.py +++ b/tools/distrib/python/submit.py @@ -1,5 +1,5 @@ #!/usr/bin/env python2.7 -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -55,11 +55,19 @@ parser.add_argument( help='Password to authenticate with the repository. Not needed if you have ' 'configured your .pypirc to include your password.' ) +parser.add_argument( + '--bdist', '-b', action='store_true', + help='Generate a binary distribution (wheel) for the current OS.' +) +parser.add_argument( + '--dist-args', type=str, + help='Additional arguments to pass to the *dist setup.py command.' +) args = parser.parse_args() # Move to the root directory of Python GRPC. pkgdir = os.path.join(os.path.dirname(os.path.abspath(__file__)), - '../../../src/python/grpcio') + '../../../') # Remove previous distributions; they somehow confuse twine. try: shutil.rmtree(os.path.join(pkgdir, 'dist/')) @@ -73,7 +81,12 @@ cmd = ['python', 'setup.py', 'build_ext', '--inplace'] subprocess.call(cmd, cwd=pkgdir, env=build_env) # Make the push. -cmd = ['python', 'setup.py', 'sdist'] +if args.bdist: + cmd = ['python', 'setup.py', 'bdist_wheel'] +else: + cmd = ['python', 'setup.py', 'sdist'] +if args.dist_args: + cmd += args.dist_args.split() subprocess.call(cmd, cwd=pkgdir) cmd = ['twine', 'upload', '-r', args.repository] diff --git a/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh b/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh index ededc6e809..87445c71ce 100755 --- a/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh +++ b/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh @@ -65,4 +65,3 @@ else false fi fi - diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index aef5bec86b..2280fde425 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1014,6 +1014,7 @@ src/core/surface/metadata_array.c \ src/core/surface/server.c \ src/core/surface/server_chttp2.c \ src/core/surface/server_create.c \ +src/core/surface/validate_metadata.c \ src/core/surface/version.c \ src/core/transport/byte_stream.c \ src/core/transport/chttp2/alpn.c \ diff --git a/tools/jenkins/build_interop_stress_image.sh b/tools/jenkins/build_interop_stress_image.sh new file mode 100755 index 0000000000..6b22dce6c0 --- /dev/null +++ b/tools/jenkins/build_interop_stress_image.sh @@ -0,0 +1,86 @@ +#!/bin/bash +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# This script is invoked by run_interop_tests.py to build the docker image +# for interop testing. You should never need to call this script on your own. + +set -x + +# Params: +# INTEROP_IMAGE - name of tag of the final interop image +# BASE_NAME - base name used to locate the base Dockerfile and build script +# TTY_FLAG - optional -t flag to make docker allocate tty +# BUILD_INTEROP_DOCKER_EXTRA_ARGS - optional args to be passed to the +# docker run command + +cd `dirname $0`/../.. +GRPC_ROOT=`pwd` +MOUNT_ARGS="-v $GRPC_ROOT:/var/local/jenkins/grpc:ro" + +mkdir -p /tmp/ccache + +# Mount service account dir if available. +# If service_directory does not contain the service account JSON file, +# some of the tests will fail. +if [ -e $HOME/service_account ] +then + MOUNT_ARGS+=" -v $HOME/service_account:/var/local/jenkins/service_account:ro" +fi + +# Use image name based on Dockerfile checksum +BASE_IMAGE=${BASE_NAME}_base:`sha1sum tools/jenkins/$BASE_NAME/Dockerfile | cut -f1 -d\ ` + +# Make sure base docker image has been built. Should be instantaneous if so. +docker build -t $BASE_IMAGE --force-rm=true tools/jenkins/$BASE_NAME || exit $? + +# Create a local branch so the child Docker script won't complain +git branch -f jenkins-docker + +CONTAINER_NAME="build_${BASE_NAME}_$(uuidgen)" + +# Prepare image for interop tests, commit it on success. +(docker run \ + -e CCACHE_DIR=/tmp/ccache \ + -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \ + -i $TTY_FLAG \ + $MOUNT_ARGS \ + $BUILD_INTEROP_DOCKER_EXTRA_ARGS \ + -v /tmp/ccache:/tmp/ccache \ + --name=$CONTAINER_NAME \ + $BASE_IMAGE \ + bash -l /var/local/jenkins/grpc/tools/jenkins/$BASE_NAME/build_interop_stress.sh \ + && docker commit $CONTAINER_NAME $INTEROP_IMAGE \ + && echo "Successfully built image $INTEROP_IMAGE") +EXITCODE=$? + +# remove intermediate container, possibly killing it first +docker rm -f $CONTAINER_NAME + +exit $EXITCODE diff --git a/tools/jenkins/grpc_interop_python/Dockerfile b/tools/jenkins/grpc_interop_python/Dockerfile index 6034cbf955..047604b1b7 100644 --- a/tools/jenkins/grpc_interop_python/Dockerfile +++ b/tools/jenkins/grpc_interop_python/Dockerfile @@ -48,6 +48,7 @@ RUN apt-get update && apt-get install -y \ libc6-dbg \ libc6-dev \ libgtest-dev \ + libssl-dev \ libtool \ make \ strace \ diff --git a/tools/jenkins/grpc_interop_python/build_interop.sh b/tools/jenkins/grpc_interop_python/build_interop.sh index 8f5bfd11e2..39c93677d8 100755 --- a/tools/jenkins/grpc_interop_python/build_interop.sh +++ b/tools/jenkins/grpc_interop_python/build_interop.sh @@ -43,5 +43,5 @@ make install-certs make # build Python interop client and server -CONFIG=opt ./tools/run_tests/build_python.sh 2.7 +CONFIG=opt ./tools/run_tests/build_python.sh diff --git a/tools/jenkins/grpc_interop_stress_cxx/Dockerfile b/tools/jenkins/grpc_interop_stress_cxx/Dockerfile new file mode 100644 index 0000000000..1fa1907533 --- /dev/null +++ b/tools/jenkins/grpc_interop_stress_cxx/Dockerfile @@ -0,0 +1,75 @@ +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# A work-in-progress Dockerfile that allows running gRPC test suites +# inside a docker container. + +FROM debian:jessie + +# Install Git. +RUN apt-get update && apt-get install -y \ + autoconf \ + autotools-dev \ + build-essential \ + bzip2 \ + ccache \ + curl \ + gcc \ + gcc-multilib \ + git \ + gyp \ + libc6 \ + libc6-dbg \ + libc6-dev \ + libgtest-dev \ + libtool \ + make \ + strace \ + python-dev \ + python-setuptools \ + python-yaml \ + telnet \ + unzip \ + wget \ + zip && apt-get clean + +# Prepare ccache +RUN ln -s /usr/bin/ccache /usr/local/bin/gcc +RUN ln -s /usr/bin/ccache /usr/local/bin/g++ +RUN ln -s /usr/bin/ccache /usr/local/bin/cc +RUN ln -s /usr/bin/ccache /usr/local/bin/c++ +RUN ln -s /usr/bin/ccache /usr/local/bin/clang +RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ + +################## +# C++ dependencies +RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang + +# Define the default command. +CMD ["bash"] diff --git a/tools/jenkins/grpc_interop_stress_cxx/build_interop_stress.sh b/tools/jenkins/grpc_interop_stress_cxx/build_interop_stress.sh new file mode 100755 index 0000000000..01f9a9c02e --- /dev/null +++ b/tools/jenkins/grpc_interop_stress_cxx/build_interop_stress.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Builds C++ interop server and client in a base image. +set -e + +mkdir -p /var/local/git +git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc + +# copy service account keys if available +cp -r /var/local/jenkins/service_account $HOME || true + +cd /var/local/git/grpc + +make install-certs + +# build C++ interop stress client, interop client and server +make stress_test interop_client interop_server diff --git a/tools/jenkins/grpc_jenkins_slave/Dockerfile b/tools/jenkins/grpc_jenkins_slave/Dockerfile index f3bf6bc4f0..b1ac024dfb 100644 --- a/tools/jenkins/grpc_jenkins_slave/Dockerfile +++ b/tools/jenkins/grpc_jenkins_slave/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/jenkins/grpc_jenkins_slave_32bits/Dockerfile b/tools/jenkins/grpc_jenkins_slave_32bits/Dockerfile index 1a86c5a5d7..348a333d3e 100644 --- a/tools/jenkins/grpc_jenkins_slave_32bits/Dockerfile +++ b/tools/jenkins/grpc_jenkins_slave_32bits/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/jenkins/run_interop_stress.sh b/tools/jenkins/run_interop_stress.sh new file mode 100755 index 0000000000..22d81db8bc --- /dev/null +++ b/tools/jenkins/run_interop_stress.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# This script is invoked by Jenkins and runs interop test suite. +set -ex + +# Enter the gRPC repo root +cd $(dirname $0)/../.. + +tools/run_tests/run_stress_tests.py -l all -s all -j 12 $@ || true diff --git a/tools/jenkins/run_jenkins.sh b/tools/jenkins/run_jenkins.sh index 9b6ba71948..84b4ea51ed 100755 --- a/tools/jenkins/run_jenkins.sh +++ b/tools/jenkins/run_jenkins.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -92,4 +92,3 @@ if [ "$TESTS_FAILED" != "" ] then exit 1 fi - diff --git a/tools/run_tests/build_python.sh b/tools/run_tests/build_python.sh index 57080ce934..e0fcbb602d 100755 --- a/tools/run_tests/build_python.sh +++ b/tools/run_tests/build_python.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -34,16 +34,14 @@ set -ex cd $(dirname $0)/../.. ROOT=`pwd` -GRPCIO=$ROOT/src/python/grpcio export LD_LIBRARY_PATH=$ROOT/libs/$CONFIG export DYLD_LIBRARY_PATH=$ROOT/libs/$CONFIG export PATH=$ROOT/bins/$CONFIG:$ROOT/bins/$CONFIG/protobuf:$PATH -export CFLAGS="-I$ROOT/include -std=c89" +export CFLAGS="-I$ROOT/include -std=gnu99" export LDFLAGS="-L$ROOT/libs/$CONFIG" export GRPC_PYTHON_BUILD_WITH_CYTHON=1 export GRPC_PYTHON_ENABLE_CYTHON_TRACING=1 -cd $GRPCIO tox --notest -$GRPCIO/.tox/py27/bin/python $GRPCIO/setup.py build +$ROOT/.tox/py27/bin/python $ROOT/setup.py build diff --git a/tools/run_tests/build_ruby.sh b/tools/run_tests/build_ruby.sh index 6d23c316c5..8acb40dc62 100755 --- a/tools/run_tests/build_ruby.sh +++ b/tools/run_tests/build_ruby.sh @@ -34,7 +34,7 @@ set -ex export GRPC_CONFIG=${CONFIG:-opt} # change to grpc's ruby directory -cd $(dirname $0)/../../src/ruby +cd $(dirname $0)/../.. rm -rf ./tmp rake compile:grpc diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py index 0b01bc4bec..e33433daf2 100755 --- a/tools/run_tests/jobset.py +++ b/tools/run_tests/jobset.py @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -450,4 +450,3 @@ def run(cmdlines, js.set_remaining(remaining) js.finish() return js.get_num_failures(), js.resultset - diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index e69e9877c5..10566d6bc8 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -298,11 +298,8 @@ class PythonLanguage: def client_cmd(self, args): return [ - 'src/python/grpcio/.tox/py27/bin/python', - 'src/python/grpcio/setup.py', - 'run_interop', - '--client', - '--args=\'{}\''.format(' '.join(args)) + 'tox -einterop_client --', + ' '.join(args) ] def cloud_to_prod_env(self): @@ -310,11 +307,8 @@ class PythonLanguage: def server_cmd(self, args): return [ - 'src/python/grpcio/.tox/py27/bin/python', - 'src/python/grpcio/setup.py', - 'run_interop', - '--server', - '--args=\'{}\''.format(' '.join(args) + ' --use_tls=true') + 'tox -einterop_server --', + ' '.join(args) + ' --use_tls=true' ] def global_env(self): @@ -555,7 +549,7 @@ def aggregate_http2_results(stdout): match = re.search(r'\{"cases[^\]]*\]\}', stdout) if not match: return None - + results = json.loads(match.group(0)) skipped = 0 passed = 0 @@ -748,7 +742,7 @@ try: for test_case in _HTTP2_TEST_CASES: if server_name == "go": # TODO(carl-mastrangelo): Reenable after https://github.com/grpc/grpc-go/issues/434 - continue + continue test_job = cloud_to_cloud_jobspec(http2Interop, test_case, server_name, @@ -777,7 +771,7 @@ try: job[0].http2results = aggregate_http2_results(job[0].message) report_utils.render_interop_html_report( - set([str(l) for l in languages]), servers, _TEST_CASES, _AUTH_TEST_CASES, + set([str(l) for l in languages]), servers, _TEST_CASES, _AUTH_TEST_CASES, _HTTP2_TEST_CASES, resultset, num_failures, args.cloud_to_prod_auth or args.cloud_to_prod, args.http2_interop) diff --git a/tools/run_tests/run_python.sh b/tools/run_tests/run_python.sh index 042b40485d..ffe9c12af1 100755 --- a/tools/run_tests/run_python.sh +++ b/tools/run_tests/run_python.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -34,7 +34,6 @@ set -ex cd $(dirname $0)/../.. ROOT=`pwd` -GRPCIO=$ROOT/src/python/grpcio export LD_LIBRARY_PATH=$ROOT/libs/$CONFIG export DYLD_LIBRARY_PATH=$ROOT/libs/$CONFIG export PATH=$ROOT/bins/$CONFIG:$ROOT/bins/$CONFIG/protobuf:$PATH @@ -43,9 +42,8 @@ export LDFLAGS="-L$ROOT/libs/$CONFIG" export GRPC_PYTHON_BUILD_WITH_CYTHON=1 export GRPC_PYTHON_ENABLE_CYTHON_TRACING=1 -cd $GRPCIO tox mkdir -p $ROOT/reports rm -rf $ROOT/reports/python-coverage -(mv -T $GRPCIO/htmlcov $ROOT/reports/python-coverage) || true +(mv -T $ROOT/htmlcov $ROOT/reports/python-coverage) || true diff --git a/tools/run_tests/run_ruby.sh b/tools/run_tests/run_ruby.sh index b82ce52af3..73a84ac361 100755 --- a/tools/run_tests/run_ruby.sh +++ b/tools/run_tests/run_ruby.sh @@ -31,6 +31,6 @@ set -ex # change to grpc repo root -cd $(dirname $0)/../../src/ruby +cd $(dirname $0)/../.. rake diff --git a/tools/run_tests/run_stress_tests.py b/tools/run_tests/run_stress_tests.py new file mode 100755 index 0000000000..b01a07af90 --- /dev/null +++ b/tools/run_tests/run_stress_tests.py @@ -0,0 +1,328 @@ +#!/usr/bin/env python +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +"""Run stress test in C++""" + +import argparse +import atexit +import dockerjob +import itertools +import jobset +import json +import multiprocessing +import os +import re +import subprocess +import sys +import tempfile +import time +import uuid + +# Docker doesn't clean up after itself, so we do it on exit. +atexit.register(lambda: subprocess.call(['stty', 'echo'])) + +ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) +os.chdir(ROOT) + +_DEFAULT_SERVER_PORT = 8080 +_DEFAULT_METRICS_PORT = 8081 +_DEFAULT_TEST_CASES = 'empty_unary:20,large_unary:20,client_streaming:20,server_streaming:20,empty_stream:20' +_DEFAULT_NUM_CHANNELS_PER_SERVER = 5 +_DEFAULT_NUM_STUBS_PER_CHANNEL = 10 + +# 15 mins default +_DEFAULT_TEST_DURATION_SECS = 900 + +class CXXLanguage: + + def __init__(self): + self.client_cwd = None + self.server_cwd = None + self.safename = 'cxx' + + def client_cmd(self, args): + return ['bins/opt/stress_test'] + args + + def server_cmd(self, args): + return ['bins/opt/interop_server'] + args + + def global_env(self): + return {} + + def __str__(self): + return 'c++' + + +_LANGUAGES = {'c++': CXXLanguage(),} + +# languages supported as cloud_to_cloud servers +_SERVERS = ['c++'] + +DOCKER_WORKDIR_ROOT = '/var/local/git/grpc' + + +def docker_run_cmdline(cmdline, image, docker_args=[], cwd=None, environ=None): + """Wraps given cmdline array to create 'docker run' cmdline from it.""" + docker_cmdline = ['docker', 'run', '-i', '--rm=true'] + + # turn environ into -e docker args + if environ: + for k, v in environ.iteritems(): + docker_cmdline += ['-e', '%s=%s' % (k, v)] + + # set working directory + workdir = DOCKER_WORKDIR_ROOT + if cwd: + workdir = os.path.join(workdir, cwd) + docker_cmdline += ['-w', workdir] + + docker_cmdline += docker_args + [image] + cmdline + return docker_cmdline + + +def bash_login_cmdline(cmdline): + """Creates bash -l -c cmdline from args list.""" + # Use login shell: + # * rvm and nvm require it + # * makes error messages clearer if executables are missing + return ['bash', '-l', '-c', ' '.join(cmdline)] + + +def _job_kill_handler(job): + if job._spec.container_name: + dockerjob.docker_kill(job._spec.container_name) + # When the job times out and we decide to kill it, + # we need to wait a before restarting the job + # to prevent "container name already in use" error. + # TODO(jtattermusch): figure out a cleaner way to to this. + time.sleep(2) + + +def cloud_to_cloud_jobspec(language, + test_cases, + server_addresses, + test_duration_secs, + num_channels_per_server, + num_stubs_per_channel, + metrics_port, + docker_image=None): + """Creates jobspec for cloud-to-cloud interop test""" + cmdline = bash_login_cmdline(language.client_cmd([ + '--test_cases=%s' % test_cases, '--server_addresses=%s' % + server_addresses, '--test_duration_secs=%s' % test_duration_secs, + '--num_stubs_per_channel=%s' % num_stubs_per_channel, + '--num_channels_per_server=%s' % num_channels_per_server, + '--metrics_port=%s' % metrics_port + ])) + print cmdline + cwd = language.client_cwd + environ = language.global_env() + if docker_image: + container_name = dockerjob.random_name('interop_client_%s' % + language.safename) + cmdline = docker_run_cmdline( + cmdline, + image=docker_image, + environ=environ, + cwd=cwd, + docker_args=['--net=host', '--name', container_name]) + cwd = None + + test_job = jobset.JobSpec(cmdline=cmdline, + cwd=cwd, + environ=environ, + shortname='cloud_to_cloud:%s:%s_server:stress_test' % ( + language, server_name), + timeout_seconds=test_duration_secs * 2, + flake_retries=0, + timeout_retries=0, + kill_handler=_job_kill_handler) + test_job.container_name = container_name + return test_job + + +def server_jobspec(language, docker_image, test_duration_secs): + """Create jobspec for running a server""" + container_name = dockerjob.random_name('interop_server_%s' % + language.safename) + cmdline = bash_login_cmdline(language.server_cmd(['--port=%s' % + _DEFAULT_SERVER_PORT])) + environ = language.global_env() + docker_cmdline = docker_run_cmdline( + cmdline, + image=docker_image, + cwd=language.server_cwd, + environ=environ, + docker_args=['-p', str(_DEFAULT_SERVER_PORT), '--name', container_name]) + + server_job = jobset.JobSpec(cmdline=docker_cmdline, + environ=environ, + shortname='interop_server_%s' % language, + timeout_seconds=test_duration_secs * 3) + server_job.container_name = container_name + return server_job + + +def build_interop_stress_image_jobspec(language, tag=None): + """Creates jobspec for building stress test docker image for a language""" + if not tag: + tag = 'grpc_interop_stress_%s:%s' % (language.safename, uuid.uuid4()) + env = {'INTEROP_IMAGE': tag, + 'BASE_NAME': 'grpc_interop_stress_%s' % language.safename} + build_job = jobset.JobSpec(cmdline=['tools/jenkins/build_interop_stress_image.sh'], + environ=env, + shortname='build_docker_%s' % (language), + timeout_seconds=30 * 60) + build_job.tag = tag + return build_job + +argp = argparse.ArgumentParser(description='Run stress tests.') +argp.add_argument('-l', + '--language', + choices=['all'] + sorted(_LANGUAGES), + nargs='+', + default=['all'], + help='Clients to run.') +argp.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int) +argp.add_argument( + '-s', + '--server', + choices=['all'] + sorted(_SERVERS), + action='append', + help='Run cloud_to_cloud servers in a separate docker ' + 'image.', + default=[]) +argp.add_argument( + '--override_server', + action='append', + type=lambda kv: kv.split('='), + help= + 'Use servername=HOST:PORT to explicitly specify a server. E.g. ' + 'csharp=localhost:50000', + default=[]) +argp.add_argument('--test_duration_secs', + help='The duration of the test in seconds', + default=_DEFAULT_TEST_DURATION_SECS) + +args = argp.parse_args() + +servers = set( + s + for s in itertools.chain.from_iterable(_SERVERS if x == 'all' else [x] + for x in args.server)) + +languages = set(_LANGUAGES[l] + for l in itertools.chain.from_iterable(_LANGUAGES.iterkeys( + ) if x == 'all' else [x] for x in args.language)) + +docker_images = {} +# languages for which to build docker images +languages_to_build = set( + _LANGUAGES[k] + for k in set([str(l) for l in languages] + [s for s in servers])) +build_jobs = [] +for l in languages_to_build: + job = build_interop_stress_image_jobspec(l) + docker_images[str(l)] = job.tag + build_jobs.append(job) + +if build_jobs: + jobset.message('START', 'Building interop docker images.', do_newline=True) + num_failures, _ = jobset.run(build_jobs, + newline_on_success=True, + maxjobs=args.jobs) + if num_failures == 0: + jobset.message('SUCCESS', + 'All docker images built successfully.', + do_newline=True) + else: + jobset.message('FAILED', + 'Failed to build interop docker images.', + do_newline=True) + for image in docker_images.itervalues(): + dockerjob.remove_image(image, skip_nonexistent=True) + sys.exit(1) + +# Start interop servers. +server_jobs = {} +server_addresses = {} +try: + for s in servers: + lang = str(s) + spec = server_jobspec(_LANGUAGES[lang], docker_images.get(lang), args.test_duration_secs) + job = dockerjob.DockerJob(spec) + server_jobs[lang] = job + server_addresses[lang] = ('localhost', + job.mapped_port(_DEFAULT_SERVER_PORT)) + + jobs = [] + + for server in args.override_server: + server_name = server[0] + (server_host, server_port) = server[1].split(':') + server_addresses[server_name] = (server_host, server_port) + + for server_name, server_address in server_addresses.iteritems(): + (server_host, server_port) = server_address + for language in languages: + test_job = cloud_to_cloud_jobspec( + language, + _DEFAULT_TEST_CASES, + ('%s:%s' % (server_host, server_port)), + args.test_duration_secs, + _DEFAULT_NUM_CHANNELS_PER_SERVER, + _DEFAULT_NUM_STUBS_PER_CHANNEL, + _DEFAULT_METRICS_PORT, + docker_image=docker_images.get(str(language))) + jobs.append(test_job) + + if not jobs: + print 'No jobs to run.' + for image in docker_images.itervalues(): + dockerjob.remove_image(image, skip_nonexistent=True) + sys.exit(1) + + num_failures, resultset = jobset.run(jobs, + newline_on_success=True, + maxjobs=args.jobs) + if num_failures: + jobset.message('FAILED', 'Some tests failed', do_newline=True) + else: + jobset.message('SUCCESS', 'All tests passed', do_newline=True) + +finally: + # Check if servers are still running. + for server, job in server_jobs.iteritems(): + if not job.is_running(): + print 'Server "%s" has exited prematurely.' % server + + dockerjob.finish_jobs([j for j in server_jobs.itervalues()]) + + for image in docker_images.itervalues(): + print 'Removing docker image %s' % image + dockerjob.remove_image(image) diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index 8d86fa3bf3..3e42c59ed3 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -1504,6 +1504,23 @@ "grpc", "grpc++", "grpc++_test_util", + "grpc_test_util", + "qps" + ], + "headers": [], + "language": "c++", + "name": "generic_async_streaming_ping_pong_test", + "src": [ + "test/cpp/qps/generic_async_streaming_ping_pong_test.cc" + ] + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc++", + "grpc++_test_util", "grpc_test_util" ], "headers": [], @@ -3212,6 +3229,7 @@ "src/core/surface/server_chttp2.c", "src/core/surface/server_create.c", "src/core/surface/surface_trace.h", + "src/core/surface/validate_metadata.c", "src/core/surface/version.c", "src/core/transport/byte_stream.c", "src/core/transport/byte_stream.h", @@ -3683,6 +3701,7 @@ "src/core/surface/server_chttp2.c", "src/core/surface/server_create.c", "src/core/surface/surface_trace.h", + "src/core/surface/validate_metadata.c", "src/core/surface/version.c", "src/core/transport/byte_stream.c", "src/core/transport/byte_stream.h", diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index d4839e235f..2757ce445d 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -1828,6 +1828,23 @@ "ci_platforms": [ "linux", "mac", + "posix" + ], + "exclude_configs": [], + "flaky": false, + "language": "c++", + "name": "generic_async_streaming_ping_pong_test", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, + { + "args": [], + "ci_platforms": [ + "linux", + "mac", "posix", "windows" ], |