diff options
Diffstat (limited to 'tools')
-rwxr-xr-x | tools/buildgen/generate_projects.py | 77 | ||||
-rwxr-xr-x | tools/buildgen/generate_projects.sh | 28 | ||||
-rwxr-xr-x | tools/distrib/python/submit.py | 6 | ||||
-rw-r--r-- | tools/dockerfile/grpc_java/Dockerfile | 6 | ||||
-rw-r--r-- | tools/dockerfile/grpc_java_android/Dockerfile | 27 | ||||
-rw-r--r-- | tools/dockerfile/grpc_java_android/README.md | 10 | ||||
-rw-r--r-- | tools/dockerfile/grpc_java_base/Dockerfile | 17 | ||||
-rw-r--r-- | tools/doxygen/Doxyfile.c++ | 1 | ||||
-rw-r--r-- | tools/doxygen/Doxyfile.c++.internal | 5 | ||||
-rw-r--r-- | tools/doxygen/Doxyfile.core.internal | 1 | ||||
-rw-r--r-- | tools/jenkins/grpc_linuxbrew/Dockerfile | 62 | ||||
-rwxr-xr-x | tools/jenkins/run_jenkins.sh | 7 | ||||
-rwxr-xr-x | tools/jenkins/run_linuxbrew.sh | 55 | ||||
-rwxr-xr-x | tools/run_tests/build_python.sh | 35 | ||||
-rwxr-xr-x | tools/run_tests/jobset.py | 3 | ||||
-rwxr-xr-x | tools/run_tests/python_tests.json | 90 | ||||
-rw-r--r-- | tools/run_tests/run_csharp.bat | 5 | ||||
-rwxr-xr-x | tools/run_tests/run_python.sh | 4 | ||||
-rwxr-xr-x | tools/run_tests/run_tests.py | 56 | ||||
-rw-r--r-- | tools/run_tests/sources_and_headers.json | 36 | ||||
-rw-r--r-- | tools/run_tests/tests.json | 11 |
21 files changed, 426 insertions, 116 deletions
diff --git a/tools/buildgen/generate_projects.py b/tools/buildgen/generate_projects.py new file mode 100755 index 0000000000..95d338652b --- /dev/null +++ b/tools/buildgen/generate_projects.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python2.7 + +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import glob +import os +import sys +import tempfile +sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..', 'run_tests')) + +assert sys.argv[1:], 'run generate_projects.sh instead of this directly' + +import jobset + +os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '..', '..')) +json = sys.argv[1:] + +test = {} if 'TEST' in os.environ else None + +plugins = sorted(glob.glob('tools/buildgen/plugins/*.py')) + +jobs = [] +for root, dirs, files in os.walk('templates'): + for f in files: + if os.path.splitext(f)[1] == '.template': + out = '.' + root[len('templates'):] + '/' + os.path.splitext(f)[0] + cmd = ['tools/buildgen/mako_renderer.py'] + for plugin in plugins: + cmd.append('-p') + cmd.append(plugin) + for js in json: + cmd.append('-d') + cmd.append(js) + cmd.append('-o') + if test is None: + cmd.append(out) + else: + tf = tempfile.mkstemp() + test[out] = tf[1] + os.close(tf[0]) + cmd.append(test[out]) + cmd.append(root + '/' + f) + jobs.append(jobset.JobSpec(cmd, shortname=out)) + +jobset.run(jobs) + +if test is not None: + for s, g in test.iteritems(): + assert(0 == os.system('diff %s %s' % (s, g))) + os.unlink(g) diff --git a/tools/buildgen/generate_projects.sh b/tools/buildgen/generate_projects.sh index 5399867746..32fc90fef5 100755 --- a/tools/buildgen/generate_projects.sh +++ b/tools/buildgen/generate_projects.sh @@ -45,32 +45,6 @@ fi . tools/buildgen/generate_build_additions.sh -global_plugins=`find ./tools/buildgen/plugins -name '*.py' | - sort | grep -v __init__ | awk ' { printf "-p %s ", $0 } '` - -for dir in . ; do - local_plugins=`find $dir/templates -name '*.py' | - sort | grep -v __init__ | awk ' { printf "-p %s ", $0 } '` - - plugins="$global_plugins $local_plugins" - - find -L $dir/templates -type f -and -name *.template | while read file ; do - out=${dir}/${file#$dir/templates/} # strip templates dir prefix - out=${out%.*} # strip template extension - echo "generating file: $out" - json_files="build.json $gen_build_files" - data=`for i in $json_files ; do echo $i ; done | awk ' { printf "-d %s ", $0 } '` - if [ "x$TEST" = "xtrue" ] ; then - actual_out=$out - out=`mktemp /tmp/gentXXXXXX` - fi - mkdir -p `dirname $out` # make sure dest directory exist - $mako_renderer $plugins $data -o $out $file - if [ "x$TEST" = "xtrue" ] ; then - diff -q $out $actual_out - rm $out - fi - done -done +tools/buildgen/generate_projects.py build.json $gen_build_files rm $gen_build_files diff --git a/tools/distrib/python/submit.py b/tools/distrib/python/submit.py index dd48f440ba..a3615b3640 100755 --- a/tools/distrib/python/submit.py +++ b/tools/distrib/python/submit.py @@ -66,6 +66,12 @@ try: except: pass +# Build the Cython C files +build_env = os.environ.copy() +build_env['GRPC_PYTHON_BUILD_WITH_CYTHON'] = "1" +cmd = ['python', 'setup.py', 'build_ext', '--inplace'] +subprocess.call(cmd, cwd=pkgdir, env=build_env) + # Make the push. cmd = ['python', 'setup.py', 'sdist'] subprocess.call(cmd, cwd=pkgdir) diff --git a/tools/dockerfile/grpc_java/Dockerfile b/tools/dockerfile/grpc_java/Dockerfile index 15fce276ba..7b1fe56358 100644 --- a/tools/dockerfile/grpc_java/Dockerfile +++ b/tools/dockerfile/grpc_java/Dockerfile @@ -30,10 +30,8 @@ # Dockerfile for the gRPC Java dev image FROM grpc/java_base -RUN git clone --recursive --depth 1 https://github.com/grpc/grpc-java.git /var/local/git/grpc-java -RUN cd /var/local/git/grpc-java/lib/netty && \ - mvn -pl codec-http2 -am -DskipTests install clean -RUN cd /var/local/git/grpc-java && \ +RUN git clone --recursive --depth 1 https://github.com/grpc/grpc-java.git /var/local/git/grpc-java && \ + cd /var/local/git/grpc-java && \ ./gradlew :grpc-interop-testing:installDist -PskipCodegen=true # Add a service_account directory containing the auth creds file diff --git a/tools/dockerfile/grpc_java_android/Dockerfile b/tools/dockerfile/grpc_java_android/Dockerfile index 2dc2202e9d..8df0707758 100644 --- a/tools/dockerfile/grpc_java_android/Dockerfile +++ b/tools/dockerfile/grpc_java_android/Dockerfile @@ -43,27 +43,20 @@ ENV PATH $PATH:$ANDROID_HOME/platform-tools # Some old Docker versions consider '/' as HOME ENV HOME /root -# Update sdk for android 5.1 (API level 22) -RUN echo y | android update sdk --all --filter platform-tools,build-tools-22.0.1,sys-img-armeabi-v7a-addon-google_apis-google-22,sys-img-armeabi-v7a-addon-google_apis-google-21,sys-img-armeabi-v7a-android-19,addon-google_apis-google-22,addon-google_apis-google-21,addon-google_apis-google-19,extra-android-m2repository,extra-google-m2repository --no-ui --force +# Update sdk for android API level 19 (4.4), 21 (5.0), 22 (5.1). +RUN echo y | android update sdk --all --filter platform-tools,build-tools-22.0.1,sys-img-armeabi-v7a-addon-google_apis-google-22,sys-img-armeabi-v7a-addon-google_apis-google-21,sys-img-armeabi-v7a-android-19,android-22,android-21,android-19,addon-google_apis-google-22,addon-google_apis-google-21,addon-google_apis-google-19,extra-android-m2repository,extra-google-m2repository --no-ui --force -# Create an AVD with API level 22 -RUN echo no | android create avd --force -n avd-google-api-22 -t "Google Inc.:Google APIs:22" --abi google_apis/armeabi-v7a -RUN echo no | android create avd --force -n avd-google-api-21 -t "Google Inc.:Google APIs:21" --abi google_apis/armeabi-v7a -RUN echo no | android create avd --force -n avd-google-api-19 -t "Google Inc.:Google APIs:19" --abi default/armeabi-v7a +# Create AVDs with API level 19,21,22 +RUN echo no | android create avd --force -n avd-google-api-22 -t "Google Inc.:Google APIs:22" --abi google_apis/armeabi-v7a && \ + echo no | android create avd --force -n avd-google-api-21 -t "Google Inc.:Google APIs:21" --abi google_apis/armeabi-v7a && \ + echo no | android create avd --force -n avd-google-api-19 -t "Google Inc.:Google APIs:19" --abi default/armeabi-v7a # Pull gRPC Java and trigger download of needed Maven and Gradle artifacts. RUN git clone --depth 1 https://github.com/grpc/grpc-java.git /var/local/git/grpc-java && \ cd /var/local/git/grpc-java && \ - ./gradlew grpc-core:install grpc-stub:install grpc-okhttp:install grpc-protobuf-nano:install && \ - rm -r "$(pwd)" + ./gradlew grpc-core:install grpc-stub:install grpc-okhttp:install grpc-protobuf-nano:install grpc-compiler:install -# Pull gRPC Android integration test App -RUN git clone --depth 1 https://github.com/madongfly/grpc-android-test.git /var/local/git/grpc-android-test - -# Config android sdk for gradle -RUN cd /var/local/git/grpc-android-test && echo "sdk.dir=/usr/local/android-sdk-linux" > local.properties - -# Build apks to trigger download of needed Maven and Gradle artifacts. -RUN cd /var/local/git/grpc-android-test && ./gradlew assembleDebug -RUN cd /var/local/git/grpc-android-test && ./gradlew assembleDebugAndroidTest +# Config android sdk for gradle and build apk to trigger download of needed Maven and Gradle artifacts. +RUN cd /var/local/git/grpc-java/android-interop-testing && echo "sdk.dir=/usr/local/android-sdk-linux" > local.properties && \ + ../gradlew assembleDebug diff --git a/tools/dockerfile/grpc_java_android/README.md b/tools/dockerfile/grpc_java_android/README.md index b4c9645e6a..5c897cd779 100644 --- a/tools/dockerfile/grpc_java_android/README.md +++ b/tools/dockerfile/grpc_java_android/README.md @@ -15,23 +15,23 @@ Usage Start the emulator in a detached container, the argument is the name of the AVD you want to start: ``` -$ sudo docker run --name=grpc_android_test -d grpc/android /var/local/git/grpc-android-test/start-emulator.sh avd-api-22 +$ sudo docker run --name=grpc_android_test -d grpc/android /var/local/git/grpc-java/android-interop-testing/start-emulator.sh avd-google-api-22 ``` You can use the following cammand to wait until the emulator is ready: ``` -$ sudo docker exec grpc_android_test /var/local/git/grpc-android-test/wait-for-emulator.sh +$ sudo docker exec grpc_android_test /var/local/git/grpc-java/android-interop-testing/wait-for-emulator.sh ``` When you want to update the apk, run: ``` -$ sudo docker exec grpc_android_test /var/local/git/grpc-android-test/update-apk.sh +$ sudo docker exec grpc_android_test bash -c "cd /var/local/git/grpc-java && git pull origin master && ./gradlew grpc-core:install grpc-stub:install grpc-okhttp:install grpc-protobuf-nano:install grpc-compiler:install && cd android-interop-testing && ../gradlew installDebug" ``` -It will pull the fresh code of gRpc Java and our integration test app from github, build and install it to the runing emulator (so you need to make sure there is a runing emulator). +It pulls the fresh code of gRpc Java and our interop test app from github, build and install it to the runing emulator (so you need to make sure there is a runing emulator). Trigger the integration test: ``` -$ sudo docker exec grpc_android_test /var/local/git/grpc-android-test/run-test.sh -e server_host <hostname or ip address> -e server_port 8030 -e server_host_override foo.test.google.fr -e use_tls true -e use_test_ca true +$ sudo docker exec grpc_android_test adb -e shell am instrument -w -e server_host <hostname or ip address> -e server_port 8030 -e server_host_override foo.test.google.fr -e use_tls true -e use_test_ca true -e test_case all io.grpc.android.integrationtest/.TesterInstrumentation ``` You can also use the android/adb cammands to get more info, such as: diff --git a/tools/dockerfile/grpc_java_base/Dockerfile b/tools/dockerfile/grpc_java_base/Dockerfile index 55fbe94eaf..7bf79c8e41 100644 --- a/tools/dockerfile/grpc_java_base/Dockerfile +++ b/tools/dockerfile/grpc_java_base/Dockerfile @@ -44,20 +44,11 @@ RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true && \ apt-get clean && rm -r /var/cache/oracle-jdk8-installer/ -# Install maven -RUN wget -O - http://mirror.olnevhost.net/pub/apache/maven/binaries/apache-maven-3.2.1-bin.tar.gz | \ - tar xz -C /var/local - ENV JAVA_HOME /usr/lib/jvm/java-8-oracle -ENV M2_HOME /var/local/apache-maven-3.2.1 -ENV PATH $PATH:$JAVA_HOME/bin:$M2_HOME/bin -ENV LD_LIBRARY_PATH /usr/local/lib +ENV PATH $PATH:$JAVA_HOME/bin -# Trigger download of as many Maven and Gradle artifacts as possible. We don't build grpc-java -# because we don't want to install netty +# Trigger download of as many Gradle artifacts as possible. RUN git clone --recursive --depth 1 https://github.com/grpc/grpc-java.git && \ - cd grpc-java/lib/netty && \ - mvn -pl codec-http2 -am -DskipTests verify && \ - cd ../.. && \ - ./gradlew && \ + cd grpc-java && \ + ./gradlew build -PskipCodegen=true && \ rm -r "$(pwd)" diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index d782dc18f7..feb7ad8bb9 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -762,6 +762,7 @@ WARN_LOGFILE = INPUT = include/grpc++/async_generic_service.h \ include/grpc++/async_unary_call.h \ +include/grpc++/auth_context.h \ include/grpc++/byte_buffer.h \ include/grpc++/channel_arguments.h \ include/grpc++/channel_interface.h \ diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 575cd95753..67718d8976 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -762,6 +762,7 @@ WARN_LOGFILE = INPUT = include/grpc++/async_generic_service.h \ include/grpc++/async_unary_call.h \ +include/grpc++/auth_context.h \ include/grpc++/byte_buffer.h \ include/grpc++/channel_arguments.h \ include/grpc++/channel_interface.h \ @@ -798,11 +799,15 @@ include/grpc++/stream.h \ include/grpc++/thread_pool_interface.h \ include/grpc++/time.h \ src/cpp/client/secure_credentials.h \ +src/cpp/common/secure_auth_context.h \ src/cpp/server/secure_server_credentials.h \ src/cpp/client/channel.h \ +src/cpp/common/create_auth_context.h \ src/cpp/server/thread_pool.h \ src/cpp/client/secure_channel_arguments.cc \ src/cpp/client/secure_credentials.cc \ +src/cpp/common/secure_auth_context.cc \ +src/cpp/common/secure_create_auth_context.cc \ src/cpp/server/secure_server_credentials.cc \ src/cpp/client/channel.cc \ src/cpp/client/channel_arguments.cc \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index ac411e3c02..f11df21388 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -985,6 +985,7 @@ src/core/surface/server.c \ src/core/surface/server_chttp2.c \ src/core/surface/server_create.c \ src/core/surface/surface_trace.c \ +src/core/surface/version.c \ src/core/transport/chttp2/alpn.c \ src/core/transport/chttp2/bin_encoder.c \ src/core/transport/chttp2/frame_data.c \ diff --git a/tools/jenkins/grpc_linuxbrew/Dockerfile b/tools/jenkins/grpc_linuxbrew/Dockerfile new file mode 100644 index 0000000000..9b76e3cea0 --- /dev/null +++ b/tools/jenkins/grpc_linuxbrew/Dockerfile @@ -0,0 +1,62 @@ +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# A work-in-progress Dockerfile that allows running gRPC homebrew +# installations inside docker containers +FROM debian:jessie + +# Core dependencies +RUN apt-get update && apt-get install -y \ + bzip2 curl git ruby wget + +# Install linuxbrew +ENV PATH /home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin:$PATH +RUN git clone https://github.com/Homebrew/linuxbrew.git /home/linuxbrew/.linuxbrew +RUN brew doctor || true + +# Python dependency +RUN apt-get update && apt-get install -y python-dev +RUN curl https://bootstrap.pypa.io/get-pip.py | python + +# NodeJS dependency +RUN touch .profile +RUN curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.25.4/install.sh | bash +RUN /bin/bash -l -c "nvm install 0.12" + +# Ruby dependency +RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 +RUN /bin/bash -l -c "\curl -sSL https://get.rvm.io | bash -s stable" +RUN /bin/bash -l -c "rvm install ruby-2.1" + +# PHP dependency +RUN apt-get update && apt-get install -y php5 php5-dev phpunit unzip + +RUN /bin/bash -l -c "echo 'export PATH=/home/linuxbrew/.linuxbrew/bin:\$PATH' >> ~/.bashrc" + +CMD ["bash"] diff --git a/tools/jenkins/run_jenkins.sh b/tools/jenkins/run_jenkins.sh index 88476b55b1..af49ea0bd0 100755 --- a/tools/jenkins/run_jenkins.sh +++ b/tools/jenkins/run_jenkins.sh @@ -70,7 +70,8 @@ then DOCKER_CID=`cat docker.cid` docker kill $DOCKER_CID docker cp $DOCKER_CID:/var/local/git/grpc/report.xml $git_root - docker rm $DOCKER_CID + sleep 4 + docker rm $DOCKER_CID || true elif [ "$platform" == "windows" ] then @@ -83,12 +84,12 @@ then /cygdrive/c/nuget/nuget.exe restore vsprojects/grpc.sln /cygdrive/c/nuget/nuget.exe restore src/csharp/Grpc.sln - python tools/run_tests/run_tests.py -t -l $language -x report.xml + python tools/run_tests/run_tests.py -t -l $language -x report.xml || true elif [ "$platform" == "macos" ] then echo "building $language on MacOS" - ./tools/run_tests/run_tests.py -t -l $language -c $config -x report.xml + ./tools/run_tests/run_tests.py -t -l $language -c $config -x report.xml || true else echo "Unknown platform $platform" exit 1 diff --git a/tools/jenkins/run_linuxbrew.sh b/tools/jenkins/run_linuxbrew.sh new file mode 100755 index 0000000000..10c41b4099 --- /dev/null +++ b/tools/jenkins/run_linuxbrew.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# This script is invoked by Jenkins and triggers a test run of +# linuxbrew installation of a selected language +set -ex + +sha1=$(sha1sum tools/jenkins/grpc_linuxbrew/Dockerfile | cut -f1 -d\ ) +DOCKER_IMAGE_NAME=grpc_linuxbrew_$sha1 + +docker build -t $DOCKER_IMAGE_NAME tools/jenkins/grpc_linuxbrew + +supported="python nodejs ruby php" + +if [ "$language" == "core" ]; then + command="curl -fsSL https://goo.gl/getgrpc | bash -" +elif [[ "$supported" =~ "$language" ]]; then + command="curl -fsSL https://goo.gl/getgrpc | bash -s $language" +else + echo "unsupported language $language" + exit 1 +fi + +docker run $DOCKER_IMAGE_NAME bash -l \ + -c "nvm use 0.12; \ + npm set unsafe-perm true; \ + rvm use ruby-2.1; \ + $command" diff --git a/tools/run_tests/build_python.sh b/tools/run_tests/build_python.sh index d9b7644f44..ae0fb42241 100755 --- a/tools/run_tests/build_python.sh +++ b/tools/run_tests/build_python.sh @@ -34,9 +34,32 @@ set -ex cd $(dirname $0)/../.. root=`pwd` -rm -rf python2.7_virtual_environment -virtualenv -p /usr/bin/python2.7 python2.7_virtual_environment -source python2.7_virtual_environment/bin/activate -pip install -r src/python/requirements.txt -CFLAGS="-I$root/include -std=c89 -Werror" LDFLAGS=-L$root/libs/$CONFIG pip install src/python/src -pip install src/python/interop + +make_virtualenv() { + virtualenv_name="python"$1"_virtual_environment" + if [ ! -d $virtualenv_name ] + then + # Build the entire virtual environment + virtualenv -p `which "python"$1` $virtualenv_name + source $virtualenv_name/bin/activate + pip install -r src/python/requirements.txt + CFLAGS="-I$root/include -std=c89" LDFLAGS=-L$root/libs/$CONFIG GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install src/python/src + pip install src/python/interop + else + source $virtualenv_name/bin/activate + # Uninstall and re-install the packages we care about. Don't use + # --force-reinstall or --ignore-installed to avoid propagating this + # unnecessarily to dependencies. Don't use --no-deps to avoid missing + # dependency upgrades. + (yes | pip uninstall grpcio) || true + (yes | pip uninstall interop) || true + (CFLAGS="-I$root/include -std=c89" LDFLAGS=-L$root/libs/$CONFIG GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install src/python/src) || ( + # Fall back to rebuilding the entire environment + rm -rf $virtualenv_name + make_virtualenv $1 + ) + pip install src/python/interop + fi +} + +make_virtualenv $1 diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py index baa126ba5f..b9af169739 100755 --- a/tools/run_tests/jobset.py +++ b/tools/run_tests/jobset.py @@ -81,6 +81,7 @@ _CLEAR_LINE = '\x1b[2K' _TAG_COLOR = { 'FAILED': 'red', + 'WARNING': 'yellow', 'TIMEOUT': 'red', 'PASSED': 'green', 'START': 'gray', @@ -205,7 +206,7 @@ class Job(object): do_newline=self._newline_on_success or self._travis) if self._bin_hash: update_cache.finished(self._spec.identity(), self._bin_hash) - elif self._state == _RUNNING and time.time() - self._start > 300: + elif self._state == _RUNNING and time.time() - self._start > 600: self._tempfile.seek(0) stdout = self._tempfile.read() message('TIMEOUT', self._spec.shortname, stdout, do_newline=True) diff --git a/tools/run_tests/python_tests.json b/tools/run_tests/python_tests.json index 6c969d765f..77473db2ed 100755 --- a/tools/run_tests/python_tests.json +++ b/tools/run_tests/python_tests.json @@ -1,56 +1,110 @@ [ { - "module": "grpc._adapter._c_test" + "module": "grpc._adapter._c_test", + "pythonVersions": [ + "2.7" + ] }, { - "module": "grpc._adapter._low_test" + "module": "grpc._adapter._low_test", + "pythonVersions": [ + "2.7" + ] }, { - "module": "grpc._adapter._intermediary_low_test" + "module": "grpc._adapter._intermediary_low_test", + "pythonVersions": [ + "2.7" + ] }, { - "module": "grpc._adapter._links_test" + "module": "grpc._adapter._links_test", + "pythonVersions": [ + "2.7" + ] }, { - "module": "grpc._adapter._lonely_rear_link_test" + "module": "grpc._adapter._lonely_rear_link_test", + "pythonVersions": [ + "2.7" + ] }, { - "module": "grpc._adapter._blocking_invocation_inline_service_test" + "module": "grpc._adapter._blocking_invocation_inline_service_test", + "pythonVersions": [ + "2.7" + ] }, { - "module": "grpc._adapter._event_invocation_synchronous_event_service_test" + "module": "grpc._adapter._event_invocation_synchronous_event_service_test", + "pythonVersions": [ + "2.7" + ] }, { - "module": "grpc._adapter._future_invocation_asynchronous_event_service_test" + "module": "grpc._adapter._future_invocation_asynchronous_event_service_test", + "pythonVersions": [ + "2.7" + ] }, { - "module": "grpc.early_adopter.implementations_test" + "module": "grpc.early_adopter.implementations_test", + "pythonVersions": [ + "2.7" + ] }, { - "module": "grpc.framework.base.implementations_test" + "module": "grpc.framework.base.implementations_test", + "pythonVersions": [ + "2.7" + ] }, { - "module": "grpc.framework.face.blocking_invocation_inline_service_test" + "module": "grpc.framework.face.blocking_invocation_inline_service_test", + "pythonVersions": [ + "2.7" + ] }, { - "module": "grpc.framework.face.event_invocation_synchronous_event_service_test" + "module": "grpc.framework.face.event_invocation_synchronous_event_service_test", + "pythonVersions": [ + "2.7" + ] }, { - "module": "grpc.framework.face.future_invocation_asynchronous_event_service_test" + "module": "grpc.framework.face.future_invocation_asynchronous_event_service_test", + "pythonVersions": [ + "2.7" + ] }, { - "module": "grpc.framework.foundation._later_test" + "module": "grpc.framework.foundation._later_test", + "pythonVersions": [ + "2.7" + ] }, { - "module": "grpc.framework.foundation._logging_pool_test" + "module": "grpc.framework.foundation._logging_pool_test", + "pythonVersions": [ + "2.7" + ] }, { - "module": "interop._insecure_interop_test" + "module": "interop._insecure_interop_test", + "pythonVersions": [ + "2.7" + ] }, { - "module": "interop._secure_interop_test" + "module": "interop._secure_interop_test", + "pythonVersions": [ + "2.7" + ] }, { - "file": "test/compiler/python_plugin_test.py" + "file": "test/compiler/python_plugin_test.py", + "pythonVersions": [ + "2.7" + ] } ] diff --git a/tools/run_tests/run_csharp.bat b/tools/run_tests/run_csharp.bat index 17c622cc2d..c86136767c 100644 --- a/tools/run_tests/run_csharp.bat +++ b/tools/run_tests/run_csharp.bat @@ -5,7 +5,10 @@ setlocal @rem enter this directory cd /d %~dp0\..\..\src\csharp -packages\NUnit.Runners.2.6.4\tools\nunit-console-x86.exe -labels "%1/bin/Debug/%1.dll" || goto :error +@rem set UUID variable to a random GUID, we will use it to put TestResults.xml to a dedicated directory, so that parallel test runs don't collide +for /F %%i in ('powershell -Command "[guid]::NewGuid().ToString()"') do (set UUID=%%i) + +packages\NUnit.Runners.2.6.4\tools\nunit-console-x86.exe -labels "%1/bin/Debug/%1.dll" -work test-results/%UUID% || goto :error endlocal diff --git a/tools/run_tests/run_python.sh b/tools/run_tests/run_python.sh index cab08f9358..4959c0241c 100755 --- a/tools/run_tests/run_python.sh +++ b/tools/run_tests/run_python.sh @@ -36,5 +36,5 @@ cd $(dirname $0)/../.. root=`pwd` export LD_LIBRARY_PATH=$root/libs/$CONFIG export DYLD_LIBRARY_PATH=$root/libs/$CONFIG -source python2.7_virtual_environment/bin/activate -python2.7 -B $* +source "python"$PYVER"_virtual_environment"/bin/activate +"python"$PYVER -B $* diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index fd90613ad6..1f44fc34fa 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -189,27 +189,55 @@ class PythonLanguage(object): def __init__(self): with open('tools/run_tests/python_tests.json') as f: self._tests = json.load(f) + self._build_python_versions = set([ + python_version + for test in self._tests + for python_version in test['pythonVersions']]) + self._has_python_versions = [] def test_specs(self, config, travis): - modules = [config.job_spec(['tools/run_tests/run_python.sh', '-m', - test['module']], - None, - environ=_FORCE_ENVIRON_FOR_WRAPPERS, - shortname=test['module']) - for test in self._tests if 'module' in test] - files = [config.job_spec(['tools/run_tests/run_python.sh', - test['file']], - None, - environ=_FORCE_ENVIRON_FOR_WRAPPERS, - shortname=test['file']) - for test in self._tests if 'file' in test] - return files + modules + job_specifications = [] + for test in self._tests: + command = None + short_name = None + if 'module' in test: + command = ['tools/run_tests/run_python.sh', '-m', test['module']] + short_name = test['module'] + elif 'file' in test: + command = ['tools/run_tests/run_python.sh', test['file']] + short_name = test['file'] + else: + raise ValueError('expected input to be a module or file to run ' + 'unittests from') + for python_version in test['pythonVersions']: + if python_version in self._has_python_versions: + environment = dict(_FORCE_ENVIRON_FOR_WRAPPERS) + environment['PYVER'] = python_version + job_specifications.append(config.job_spec( + command, None, environ=environment, shortname=short_name)) + else: + jobset.message( + 'WARNING', + 'Could not find Python {}; skipping test'.format(python_version), + '{}\n'.format(command), do_newline=True) + return job_specifications def make_targets(self): return ['static_c', 'grpc_python_plugin', 'shared_c'] def build_steps(self): - return [['tools/run_tests/build_python.sh']] + commands = [] + for python_version in self._build_python_versions: + try: + with open(os.devnull, 'w') as output: + subprocess.check_call(['which', 'python' + python_version], + stdout=output, stderr=output) + commands.append(['tools/run_tests/build_python.sh', python_version]) + self._has_python_versions.append(python_version) + except: + jobset.message('WARNING', 'Missing Python ' + python_version, + do_newline=True) + return commands def supports_multi_config(self): return False diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index 3f13b7b9ad..0493e06f58 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -1437,9 +1437,9 @@ ], "headers": [], "language": "c++", - "name": "qps_test", + "name": "qps_openloop_test", "src": [ - "test/cpp/qps/qps_test.cc" + "test/cpp/qps/qps_openloop_test.cc" ] }, { @@ -1455,9 +1455,9 @@ ], "headers": [], "language": "c++", - "name": "qps_test_openloop", + "name": "qps_test", "src": [ - "test/cpp/qps/qps_test_openloop.cc" + "test/cpp/qps/qps_test.cc" ] }, { @@ -1486,6 +1486,19 @@ { "deps": [ "gpr", + "grpc", + "grpc++" + ], + "headers": [], + "language": "c++", + "name": "secure_auth_context_test", + "src": [ + "test/cpp/common/secure_auth_context_test.cc" + ] + }, + { + "deps": [ + "gpr", "gpr_test_util", "grpc", "grpc++", @@ -9012,6 +9025,7 @@ "src/core/surface/server_create.c", "src/core/surface/surface_trace.c", "src/core/surface/surface_trace.h", + "src/core/surface/version.c", "src/core/transport/chttp2/alpn.c", "src/core/transport/chttp2/alpn.h", "src/core/transport/chttp2/bin_encoder.c", @@ -9413,6 +9427,7 @@ "src/core/surface/server_create.c", "src/core/surface/surface_trace.c", "src/core/surface/surface_trace.h", + "src/core/surface/version.c", "src/core/transport/chttp2/alpn.c", "src/core/transport/chttp2/alpn.h", "src/core/transport/chttp2/bin_encoder.c", @@ -9475,6 +9490,7 @@ "headers": [ "include/grpc++/async_generic_service.h", "include/grpc++/async_unary_call.h", + "include/grpc++/auth_context.h", "include/grpc++/byte_buffer.h", "include/grpc++/channel_arguments.h", "include/grpc++/channel_interface.h", @@ -9512,6 +9528,8 @@ "include/grpc++/time.h", "src/cpp/client/channel.h", "src/cpp/client/secure_credentials.h", + "src/cpp/common/create_auth_context.h", + "src/cpp/common/secure_auth_context.h", "src/cpp/server/secure_server_credentials.h", "src/cpp/server/thread_pool.h" ], @@ -9520,6 +9538,7 @@ "src": [ "include/grpc++/async_generic_service.h", "include/grpc++/async_unary_call.h", + "include/grpc++/auth_context.h", "include/grpc++/byte_buffer.h", "include/grpc++/channel_arguments.h", "include/grpc++/channel_interface.h", @@ -9569,7 +9588,11 @@ "src/cpp/client/secure_credentials.h", "src/cpp/common/call.cc", "src/cpp/common/completion_queue.cc", + "src/cpp/common/create_auth_context.h", "src/cpp/common/rpc_method.cc", + "src/cpp/common/secure_auth_context.cc", + "src/cpp/common/secure_auth_context.h", + "src/cpp/common/secure_create_auth_context.cc", "src/cpp/proto/proto_utils.cc", "src/cpp/server/async_generic_service.cc", "src/cpp/server/create_default_thread_pool.cc", @@ -9638,6 +9661,7 @@ "headers": [ "include/grpc++/async_generic_service.h", "include/grpc++/async_unary_call.h", + "include/grpc++/auth_context.h", "include/grpc++/byte_buffer.h", "include/grpc++/channel_arguments.h", "include/grpc++/channel_interface.h", @@ -9674,6 +9698,7 @@ "include/grpc++/thread_pool_interface.h", "include/grpc++/time.h", "src/cpp/client/channel.h", + "src/cpp/common/create_auth_context.h", "src/cpp/server/thread_pool.h" ], "language": "c++", @@ -9681,6 +9706,7 @@ "src": [ "include/grpc++/async_generic_service.h", "include/grpc++/async_unary_call.h", + "include/grpc++/auth_context.h", "include/grpc++/byte_buffer.h", "include/grpc++/channel_arguments.h", "include/grpc++/channel_interface.h", @@ -9727,6 +9753,8 @@ "src/cpp/client/internal_stub.cc", "src/cpp/common/call.cc", "src/cpp/common/completion_queue.cc", + "src/cpp/common/create_auth_context.h", + "src/cpp/common/insecure_create_auth_context.cc", "src/cpp/common/rpc_method.cc", "src/cpp/proto/proto_utils.cc", "src/cpp/server/async_generic_service.cc", diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index 12b1f22b12..642a9717ce 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -705,6 +705,15 @@ { "flaky": false, "language": "c++", + "name": "qps_openloop_test", + "platforms": [ + "windows", + "posix" + ] + }, + { + "flaky": false, + "language": "c++", "name": "qps_test", "platforms": [ "windows", @@ -714,7 +723,7 @@ { "flaky": false, "language": "c++", - "name": "qps_test_openloop", + "name": "secure_auth_context_test", "platforms": [ "windows", "posix" |