diff options
Diffstat (limited to 'tools')
26 files changed, 1129 insertions, 204 deletions
diff --git a/tools/codegen/core/gen_load_balancing_proto.sh b/tools/codegen/core/gen_load_balancing_proto.sh new file mode 100755 index 0000000000..fb6a468ee0 --- /dev/null +++ b/tools/codegen/core/gen_load_balancing_proto.sh @@ -0,0 +1,138 @@ +#!/bin/bash + +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# +# Example usage: +# tools/codegen/core/gen_load_balancing_proto.sh \ +# src/proto/grpc/lb/v0/load_balancer.proto + +read -r -d '' COPYRIGHT <<'EOF' +/* + * + * Copyright <YEAR>, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +EOF + +CURRENT_YEAR=$(date +%Y) +COPYRIGHT_FILE=$(mktemp) +echo "${COPYRIGHT/<YEAR>/$CURRENT_YEAR}" > $COPYRIGHT_FILE + +set -ex +if [ $# -eq 0 ]; then + echo "Usage: $0 <load_balancer.proto> [output dir]" + exit 1 +fi + +readonly GRPC_ROOT=$PWD + +OUTPUT_DIR="$GRPC_ROOT/src/core/proto/grpc/lb/v0" +if [ $# -eq 2 ]; then + mkdir -p "$2" + if [ $? != 0 ]; then + echo "Error creating output directory $2" + exit 2 + fi + OUTPUT_DIR="$2" +fi + +readonly EXPECTED_OPTIONS_FILE_PATH="${1%.*}.options" + +if [[ ! -f "$1" ]]; then + echo "Input proto file '$1' doesn't exist." + exit 3 +fi +if [[ ! -f "${EXPECTED_OPTIONS_FILE_PATH}" ]]; then + echo "Expected nanopb options file '${EXPECTED_OPTIONS_FILE_PATH}' missing" + exit 4 +fi + +readonly VENV_DIR=$(mktemp -d) +readonly VENV_NAME="nanopb-$(date '+%Y%m%d_%H%M%S_%N')" +pushd $VENV_DIR +virtualenv $VENV_NAME +. $VENV_NAME/bin/activate +popd + +# this should be the same version as the submodule we compile against +# ideally we'd update this as a template to ensure that +pip install protobuf==3.0.0b2 + +pushd "$(dirname $1)" > /dev/null + +protoc \ +--plugin=protoc-gen-nanopb="$GRPC_ROOT/third_party/nanopb/generator/protoc-gen-nanopb" \ +--nanopb_out='-T -L#include\ \"third_party/nanopb/pb.h\"'":$OUTPUT_DIR" \ +"$(basename $1)" + +readonly PROTO_BASENAME=$(basename $1 .proto) +sed -i "s:$PROTO_BASENAME.pb.h:src/core/proto/grpc/lb/v0/$PROTO_BASENAME.pb.h:g" \ + "$OUTPUT_DIR/$PROTO_BASENAME.pb.c" + +# prepend copyright +TMPFILE=$(mktemp) +cat $COPYRIGHT_FILE "$OUTPUT_DIR/$PROTO_BASENAME.pb.c" > $TMPFILE +mv -v $TMPFILE "$OUTPUT_DIR/$PROTO_BASENAME.pb.c" +cat $COPYRIGHT_FILE "$OUTPUT_DIR/$PROTO_BASENAME.pb.h" > $TMPFILE +mv -v $TMPFILE "$OUTPUT_DIR/$PROTO_BASENAME.pb.h" + +deactivate +rm -rf $VENV_DIR + +popd > /dev/null diff --git a/tools/distrib/check_nanopb_output.sh b/tools/distrib/check_nanopb_output.sh new file mode 100755 index 0000000000..5f49ebb93e --- /dev/null +++ b/tools/distrib/check_nanopb_output.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# Copyright 2015-2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set -ex + +apt-get install -y autoconf automake libtool curl python-virtualenv + +readonly NANOPB_TMP_OUTPUT="$(mktemp -d)" + +# install protoc version 3 +pushd third_party/protobuf +./autogen.sh +./configure +make +make install +ldconfig +popd + +if [ ! -x "/usr/local/bin/protoc" ]; then + echo "Error: protoc not found in path" + exit 1 +fi +readonly PROTOC_PATH='/usr/local/bin' +# stack up and change to nanopb's proto generator directory +pushd third_party/nanopb/generator/proto +PATH="$PROTOC_PATH:$PATH" make + +# back to the root directory +popd + + +# nanopb-compile the proto to a temp location +PATH="$PROTOC_PATH:$PATH" ./tools/codegen/core/gen_load_balancing_proto.sh \ + src/proto/grpc/lb/v0/load_balancer.proto \ + $NANOPB_TMP_OUTPUT + +# compare outputs to checked compiled code +if ! diff -r $NANOPB_TMP_OUTPUT src/core/proto/grpc/lb/v0; then + echo "Outputs differ: $NANOPB_TMP_OUTPUT vs src/core/proto/grpc/lb/v0" + exit 2 +fi diff --git a/tools/dockerfile/distribtest/php_jessie_x64/Dockerfile b/tools/dockerfile/distribtest/php_jessie_x64/Dockerfile new file mode 100644 index 0000000000..d5d3fd626f --- /dev/null +++ b/tools/dockerfile/distribtest/php_jessie_x64/Dockerfile @@ -0,0 +1,32 @@ +# Copyright 2015-2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +FROM debian:jessie + +RUN apt-get update && apt-get install -y php5 php5-dev php-pear phpunit diff --git a/tools/dockerfile/grpc_artifact_linux_x64/Dockerfile b/tools/dockerfile/grpc_artifact_linux_x64/Dockerfile index 41dc427454..d048b725c8 100644 --- a/tools/dockerfile/grpc_artifact_linux_x64/Dockerfile +++ b/tools/dockerfile/grpc_artifact_linux_x64/Dockerfile @@ -91,6 +91,13 @@ RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc" RUN /bin/bash -l -c "echo 'rvm --default use ruby-2.1' >> ~/.bashrc" RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc" + +################## +# PHP dependencies + +RUN apt-get update && apt-get install -y \ + php5 php5-dev php-pear phpunit + RUN mkdir /var/local/jenkins # Define the default command. diff --git a/tools/dockerfile/grpc_artifact_protoc/Dockerfile b/tools/dockerfile/grpc_artifact_protoc/Dockerfile new file mode 100644 index 0000000000..1bbc6e021b --- /dev/null +++ b/tools/dockerfile/grpc_artifact_protoc/Dockerfile @@ -0,0 +1,63 @@ +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Docker file for building protoc and gRPC protoc plugin artifacts. +# forked from https://github.com/google/protobuf/blob/master/protoc-artifacts/Dockerfile + +FROM centos:6.6 + +RUN yum install -y git \ + tar \ + wget \ + make \ + autoconf \ + curl-devel \ + unzip \ + automake \ + libtool \ + glibc-static.i686 \ + glibc-devel \ + glibc-devel.i686 + +# Install GCC 4.7 to support -static-libstdc++ +RUN wget http://people.centos.org/tru/devtools-1.1/devtools-1.1.repo -P /etc/yum.repos.d +RUN bash -c 'echo "enabled=1" >> /etc/yum.repos.d/devtools-1.1.repo' +RUN bash -c "sed -e 's/\$basearch/i386/g' /etc/yum.repos.d/devtools-1.1.repo > /etc/yum.repos.d/devtools-i386-1.1.repo" +RUN sed -e 's/testing-/testing-i386-/g' -i /etc/yum.repos.d/devtools-i386-1.1.repo + +# We'll get and "Rpmdb checksum is invalid: dCDPT(pkg checksums)" error caused by +# docker issue when using overlay storage driver, but all the stuff we need +# will be installed, so for now we just ignore the error. +# https://github.com/docker/docker/issues/10180 +RUN yum install -y devtoolset-1.1 \ + devtoolset-1.1-libstdc++-devel \ + devtoolset-1.1-libstdc++-devel.i686 || true + +# Start in devtoolset environment that uses GCC 4.7 +CMD ["scl", "enable", "devtoolset-1.1", "bash"] diff --git a/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh b/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh index dd8ea1ac30..d56bc01831 100755 --- a/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh +++ b/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh @@ -44,7 +44,7 @@ for dir in $DIRS do for glob in $GLOB do - files="$files `find /local-code/$dir -name $glob -and -not -name *.generated.*`" + files="$files `find /local-code/$dir -name $glob -and -not -name *.generated.* -and -not -name *.pb.h -and -not -name *.pb.c`" done done diff --git a/tools/dockerfile/test/cxx_squeeze_x64/Dockerfile b/tools/dockerfile/test/cxx_squeeze_x64/Dockerfile new file mode 100644 index 0000000000..0848887434 --- /dev/null +++ b/tools/dockerfile/test/cxx_squeeze_x64/Dockerfile @@ -0,0 +1,84 @@ +# Copyright 2015-2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +FROM debian:squeeze + +# Install Git and basic packages. +RUN apt-get update && apt-get install -y \ + autoconf \ + autotools-dev \ + build-essential \ + bzip2 \ + ccache \ + curl \ + gcc \ + gcc-multilib \ + git \ + gyp \ + lcov \ + libc6 \ + libc6-dbg \ + libc6-dev \ + libgtest-dev \ + libtool \ + make \ + perl \ + strace \ + python-dev \ + python-setuptools \ + python-yaml \ + telnet \ + unzip \ + wget \ + zip && apt-get clean + +#================ +# Build profiling +RUN apt-get update && apt-get install -y time && apt-get clean + + +# libgflags-dev is not available on squeezy +RUN apt-get update && apt-get -y install libgtest-dev libc++-dev clang && apt-get clean + +RUN apt-get update && apt-get -y install python-pip && apt-get clean +RUN pip install argparse + +# Prepare ccache +RUN ln -s /usr/bin/ccache /usr/local/bin/gcc +RUN ln -s /usr/bin/ccache /usr/local/bin/g++ +RUN ln -s /usr/bin/ccache /usr/local/bin/cc +RUN ln -s /usr/bin/ccache /usr/local/bin/c++ +RUN ln -s /usr/bin/ccache /usr/local/bin/clang +RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ + + +RUN mkdir /var/local/jenkins + +# Define the default command. +CMD ["bash"] diff --git a/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile b/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile new file mode 100644 index 0000000000..c6fe79b42c --- /dev/null +++ b/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile @@ -0,0 +1,86 @@ +# Copyright 2015-2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +FROM ubuntu:16.04 + +# Install Git and basic packages. +RUN apt-get update && apt-get install -y \ + autoconf \ + autotools-dev \ + build-essential \ + bzip2 \ + ccache \ + curl \ + gcc \ + gcc-multilib \ + git \ + golang \ + gyp \ + lcov \ + libc6 \ + libc6-dbg \ + libc6-dev \ + libgtest-dev \ + libtool \ + make \ + perl \ + strace \ + python-dev \ + python-setuptools \ + python-yaml \ + telnet \ + unzip \ + wget \ + zip && apt-get clean + +#================ +# Build profiling +RUN apt-get update && apt-get install -y time && apt-get clean + +#================= +# C++ dependencies +RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang && apt-get clean + +# Prepare ccache +RUN ln -s /usr/bin/ccache /usr/local/bin/gcc +RUN ln -s /usr/bin/ccache /usr/local/bin/g++ +RUN ln -s /usr/bin/ccache /usr/local/bin/cc +RUN ln -s /usr/bin/ccache /usr/local/bin/c++ +RUN ln -s /usr/bin/ccache /usr/local/bin/clang +RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ + +#====================== +# Zookeeper dependencies +# TODO(jtattermusch): is zookeeper still needed? +RUN apt-get install -y libzookeeper-mt-dev + +RUN mkdir /var/local/jenkins + +# Define the default command. +CMD ["bash"] diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 1f9b9ecc18..b0a1561df9 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -787,6 +787,7 @@ src/core/channel/subchannel_call_holder.h \ src/core/client_config/client_config.h \ src/core/client_config/connector.h \ src/core/client_config/initial_connect_string.h \ +src/core/client_config/lb_policies/load_balancer_api.h \ src/core/client_config/lb_policies/pick_first.h \ src/core/client_config/lb_policies/round_robin.h \ src/core/client_config/lb_policy.h \ @@ -847,6 +848,7 @@ src/core/json/json.h \ src/core/json/json_common.h \ src/core/json/json_reader.h \ src/core/json/json_writer.h \ +src/core/proto/grpc/lb/v0/load_balancer.pb.h \ src/core/statistics/census_interface.h \ src/core/statistics/census_rpc_stats.h \ src/core/surface/api_trace.h \ @@ -903,6 +905,10 @@ src/core/tsi/transport_security_interface.h \ src/core/census/aggregation.h \ src/core/census/mlog.h \ src/core/census/rpc_metric_id.h \ +third_party/nanopb/pb.h \ +third_party/nanopb/pb_common.h \ +third_party/nanopb/pb_decode.h \ +third_party/nanopb/pb_encode.h \ src/core/census/grpc_context.c \ src/core/census/grpc_filter.c \ src/core/channel/channel_args.c \ @@ -918,6 +924,7 @@ src/core/client_config/client_config.c \ src/core/client_config/connector.c \ src/core/client_config/default_initial_connect_string.c \ src/core/client_config/initial_connect_string.c \ +src/core/client_config/lb_policies/load_balancer_api.c \ src/core/client_config/lb_policies/pick_first.c \ src/core/client_config/lb_policies/round_robin.c \ src/core/client_config/lb_policy.c \ @@ -982,6 +989,7 @@ src/core/json/json.c \ src/core/json/json_reader.c \ src/core/json/json_string.c \ src/core/json/json_writer.c \ +src/core/proto/grpc/lb/v0/load_balancer.pb.c \ src/core/surface/alarm.c \ src/core/surface/api_trace.c \ src/core/surface/byte_buffer.c \ @@ -1058,6 +1066,9 @@ src/core/census/mlog.c \ src/core/census/operation.c \ src/core/census/placeholders.c \ src/core/census/tracing.c \ +third_party/nanopb/pb_common.c \ +third_party/nanopb/pb_decode.c \ +third_party/nanopb/pb_encode.c \ include/grpc/support/alloc.h \ include/grpc/support/atm.h \ include/grpc/support/atm_gcc_atomic.h \ diff --git a/tools/jenkins/build_docker_and_run_tests.sh b/tools/jenkins/build_docker_and_run_tests.sh index e2ac7518f0..458e8ca0c7 100755 --- a/tools/jenkins/build_docker_and_run_tests.sh +++ b/tools/jenkins/build_docker_and_run_tests.sh @@ -60,6 +60,9 @@ docker build -t $DOCKER_IMAGE_NAME $DOCKERFILE_DIR # Choose random name for docker container CONTAINER_NAME="run_tests_$(uuidgen)" +# Git root as seen by the docker instance +docker_instance_git_root=/var/local/jenkins/grpc + # Run tests inside docker docker run \ -e "RUN_TESTS_COMMAND=$RUN_TESTS_COMMAND" \ @@ -69,9 +72,10 @@ docker run \ -e XDG_CACHE_HOME=/tmp/xdg-cache-home \ -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \ -e HOST_GIT_ROOT=$git_root \ + -e LOCAL_GIT_ROOT=$docker_instance_git_root \ -e "BUILD_ID=$BUILD_ID" \ -i $TTY_FLAG \ - -v "$git_root:/var/local/jenkins/grpc" \ + -v "$git_root:$docker_instance_git_root" \ -v /tmp/ccache:/tmp/ccache \ -v /tmp/npm-cache:/tmp/npm-cache \ -v /tmp/xdg-cache-home:/tmp/xdg-cache-home \ @@ -82,11 +86,6 @@ docker run \ $DOCKER_IMAGE_NAME \ bash -l "/var/local/jenkins/grpc/$DOCKER_RUN_SCRIPT" || DOCKER_FAILED="true" -if [ "$XML_REPORT" != "" ] -then - docker cp "$CONTAINER_NAME:/var/local/git/grpc/$XML_REPORT" $git_root || true -fi - docker cp "$CONTAINER_NAME:/var/local/git/grpc/reports.zip" $git_root || true unzip -o $git_root/reports.zip -d $git_root || true rm -f reports.zip diff --git a/tools/jenkins/docker_run_tests.sh b/tools/jenkins/docker_run_tests.sh index 282b857351..2e40f8fd65 100755 --- a/tools/jenkins/docker_run_tests.sh +++ b/tools/jenkins/docker_run_tests.sh @@ -60,5 +60,6 @@ echo '</body></html>' >> index.html cd .. zip -r reports.zip reports +find . -name reports.xml | xargs zip reports.zip exit $exit_code diff --git a/tools/jenkins/run_performance.sh b/tools/jenkins/run_performance.sh index c80685b23a..fbc078330f 100755 --- a/tools/jenkins/run_performance.sh +++ b/tools/jenkins/run_performance.sh @@ -47,6 +47,11 @@ PID1=$! bins/$config/qps_worker -driver_port 10010 & PID2=$! +# +# Put a timeout on these tests +# +((sleep 900; kill $$ && killall qps_worker && rm -f /tmp/qps-test.$$ )&) + export QPS_WORKERS="localhost:10000,localhost:10010" # big is the size in bytes of large messages (0 is the size otherwise) diff --git a/tools/run_tests/artifact_targets.py b/tools/run_tests/artifact_targets.py index b565fbb3f0..803d3d106b 100644 --- a/tools/run_tests/artifact_targets.py +++ b/tools/run_tests/artifact_targets.py @@ -79,6 +79,12 @@ def macos_arch_env(arch): raise Exception('Unsupported arch') return {'CFLAGS': arch_arg, 'LDFLAGS': arch_arg} +_MACOS_COMPAT_FLAG = '-mmacosx-version-min=10.7' + +_ARCH_FLAG_MAP = { + 'x86': '-m32', + 'x64': '-m64' +} python_version_arch_map = { 'x86': 'Python27_32bits', @@ -199,6 +205,7 @@ class CSharpExtArtifact: def __str__(self): return self.name + node_gyp_arch_map = { 'x86': 'ia32', 'x64': 'x64' @@ -234,11 +241,72 @@ class NodeExtArtifact: ['tools/run_tests/build_artifact_node.sh', self.gyp_arch]) +class PHPArtifact: + """Builds PHP PECL package""" + + def __init__(self, platform, arch): + self.name = 'php_pecl_package_{0}_{1}'.format(platform, arch) + self.platform = platform + self.arch = arch + self.labels = ['artifact', 'php', platform, arch] + + def pre_build_jobspecs(self): + return [] + + def build_jobspec(self): + return create_docker_jobspec( + self.name, + 'tools/dockerfile/grpc_artifact_linux_{}'.format(self.arch), + 'tools/run_tests/build_artifact_php.sh') + +class ProtocArtifact: + """Builds protoc and protoc-plugin artifacts""" + + def __init__(self, platform, arch): + self.name = 'protoc_%s_%s' % (platform, arch) + self.platform = platform + self.arch = arch + self.labels = ['artifact', 'protoc', platform, arch] + + def pre_build_jobspecs(self): + return [] + + def build_jobspec(self): + if self.platform != 'windows': + cxxflags = '-DNDEBUG %s' % _ARCH_FLAG_MAP[self.arch] + ldflags = '%s' % _ARCH_FLAG_MAP[self.arch] + if self.platform != 'macos': + ldflags += ' -static-libgcc -static-libstdc++ -s' + environ={'CONFIG': 'opt', + 'CXXFLAGS': cxxflags, + 'LDFLAGS': ldflags, + 'PROTOBUF_LDFLAGS_EXTRA': ldflags} + if self.platform == 'linux': + return create_docker_jobspec(self.name, + 'tools/dockerfile/grpc_artifact_protoc', + 'tools/run_tests/build_artifact_protoc.sh', + environ=environ) + else: + environ['CXXFLAGS'] += ' -std=c++11 -stdlib=libc++ %s' % _MACOS_COMPAT_FLAG + return create_jobspec(self.name, + ['tools/run_tests/build_artifact_protoc.sh'], + environ=environ) + else: + generator = 'Visual Studio 12 Win64' if self.arch == 'x64' else 'Visual Studio 12' + vcplatform = 'x64' if self.arch == 'x64' else 'Win32' + return create_jobspec(self.name, + ['tools\\run_tests\\build_artifact_protoc.bat'], + environ={'generator': generator, + 'Platform': vcplatform}) + + def __str__(self): + return self.name + def targets(): """Gets list of supported targets""" return ([Cls(platform, arch) - for Cls in (CSharpExtArtifact, NodeExtArtifact) + for Cls in (CSharpExtArtifact, NodeExtArtifact, ProtocArtifact) for platform in ('linux', 'macos', 'windows') for arch in ('x86', 'x64')] + [PythonArtifact('linux', 'x86'), @@ -248,4 +316,6 @@ def targets(): PythonArtifact('windows', 'x64'), RubyArtifact('linux', 'x86'), RubyArtifact('linux', 'x64'), - RubyArtifact('macos', 'x64')]) + RubyArtifact('macos', 'x64'), + PHPArtifact('linux', 'x64'), + PHPArtifact('macos', 'x64')]) diff --git a/tools/run_tests/build_artifact_php.sh b/tools/run_tests/build_artifact_php.sh new file mode 100755 index 0000000000..50bf0ea821 --- /dev/null +++ b/tools/run_tests/build_artifact_php.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +PHP_TARGET_ARCH=$1 +set -ex + +cd $(dirname $0)/../.. + +mkdir -p artifacts + +pear package + +cp -r grpc-*.tgz artifacts/grpc-php.tgz diff --git a/tools/run_tests/build_artifact_protoc.bat b/tools/run_tests/build_artifact_protoc.bat new file mode 100644 index 0000000000..e1dc032188 --- /dev/null +++ b/tools/run_tests/build_artifact_protoc.bat @@ -0,0 +1,51 @@ +@rem Copyright 2016, Google Inc. +@rem All rights reserved. +@rem +@rem Redistribution and use in source and binary forms, with or without +@rem modification, are permitted provided that the following conditions are +@rem met: +@rem +@rem * Redistributions of source code must retain the above copyright +@rem notice, this list of conditions and the following disclaimer. +@rem * Redistributions in binary form must reproduce the above +@rem copyright notice, this list of conditions and the following disclaimer +@rem in the documentation and/or other materials provided with the +@rem distribution. +@rem * Neither the name of Google Inc. nor the names of its +@rem contributors may be used to endorse or promote products derived from +@rem this software without specific prior written permission. +@rem +@rem THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +@rem "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +@rem LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +@rem A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +@rem OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +@rem SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +@rem LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +@rem DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +@rem THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +@rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +@rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +mkdir artifacts + +setlocal +cd third_party/protobuf + +powershell -Command "Invoke-WebRequest https://googlemock.googlecode.com/files/gmock-1.7.0.zip -OutFile gmock.zip" +powershell -Command "Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('gmock.zip', '.');" +rename gmock-1.7.0 gmock + +cd cmake +cmake -G "%generator%" || goto :error +endlocal + +call vsprojects/build_plugins.bat || goto :error + +xcopy /Y third_party\protobuf\cmake\Release\protoc.exe artifacts\ || goto :error +xcopy /Y vsprojects\Release\*_plugin.exe artifacts\ || xcopy /Y vsprojects\x64\Release\*_plugin.exe artifacts\ || goto :error + +goto :EOF + +:error +exit /b 1
\ No newline at end of file diff --git a/tools/run_tests/build_artifact_protoc.sh b/tools/run_tests/build_artifact_protoc.sh new file mode 100755 index 0000000000..161d3a84d6 --- /dev/null +++ b/tools/run_tests/build_artifact_protoc.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Use devtoolset environment that has GCC 4.7 before set -ex +source scl_source enable devtoolset-1.1 + +set -ex + +cd $(dirname $0)/../.. + +make plugins + +mkdir -p artifacts +cp bins/opt/protobuf/protoc bins/opt/*_plugin artifacts/ diff --git a/tools/run_tests/build_package_php.sh b/tools/run_tests/build_package_php.sh new file mode 100755 index 0000000000..56e3319ed9 --- /dev/null +++ b/tools/run_tests/build_package_php.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set -ex + +cd $(dirname $0)/../.. + +mkdir -p artifacts/ +cp -r $EXTERNAL_GIT_ROOT/architecture={x86,x64},language=php,platform={windows,linux,macos}/artifacts/* artifacts/ || true diff --git a/tools/run_tests/configs.json b/tools/run_tests/configs.json index 9d7b8a3c72..cbb8ec57b6 100644 --- a/tools/run_tests/configs.json +++ b/tools/run_tests/configs.json @@ -18,7 +18,7 @@ "environ": { "ASAN_OPTIONS": "detect_leaks=0:color=always" }, - "timeout_multiplier": 1.5 + "timeout_multiplier": 3 }, { "config": "ubsan", @@ -48,18 +48,18 @@ "ASAN_OPTIONS": "detect_leaks=1:color=always", "LSAN_OPTIONS": "suppressions=tools/lsan_suppressions.txt:report_objects=1" }, - "timeout_multiplier": 1.5 + "timeout_multiplier": 3 }, { "config": "tsan", "environ": { "TSAN_OPTIONS": "suppressions=tools/tsan_suppressions.txt:halt_on_error=1:second_deadlock_stack=1" }, - "timeout_multiplier": 2 + "timeout_multiplier": 5 }, { "config": "msan", - "timeout_multiplier": 2 + "timeout_multiplier": 4 }, { "config": "mutrace" diff --git a/tools/run_tests/distribtest_targets.py b/tools/run_tests/distribtest_targets.py index 261f44bc6d..0c02344d90 100644 --- a/tools/run_tests/distribtest_targets.py +++ b/tools/run_tests/distribtest_targets.py @@ -198,6 +198,33 @@ class RubyDistribTest(object): return self.name +class PHPDistribTest(object): + """Tests PHP package""" + + def __init__(self, platform, arch, docker_suffix): + self.name = 'php_%s_%s_%s' % (platform, arch, docker_suffix) + self.platform = platform + self.arch = arch + self.docker_suffix = docker_suffix + self.labels = ['distribtest', 'php', platform, arch, docker_suffix] + + def pre_build_jobspecs(self): + return [] + + def build_jobspec(self): + if not self.platform == 'linux': + raise Exception("Not supported yet.") + + return create_docker_jobspec(self.name, + 'tools/dockerfile/distribtest/php_%s_%s' % ( + self.docker_suffix, + self.arch), + 'test/distrib/php/run_distrib_test.sh') + + def __str__(self): + return self.name + + def targets(): """Gets list of supported targets""" return [CSharpDistribTest('linux', 'x64', 'wheezy'), @@ -241,7 +268,9 @@ def targets(): RubyDistribTest('linux', 'x64', 'ubuntu1510'), RubyDistribTest('linux', 'x64', 'ubuntu1604'), NodeDistribTest('macos', 'x64', None, '4'), - NodeDistribTest('linux', 'x86', 'jessie', '4') + NodeDistribTest('macos', 'x64', None, '5'), + NodeDistribTest('linux', 'x86', 'jessie', '4'), + PHPDistribTest('linux', 'x64', 'jessie'), ] + [ NodeDistribTest('linux', 'x64', os, version) for os in ('wheezy', 'jessie', 'ubuntu1204', 'ubuntu1404', diff --git a/tools/run_tests/package_targets.py b/tools/run_tests/package_targets.py index 4ca8279f1b..87bc4865ce 100644 --- a/tools/run_tests/package_targets.py +++ b/tools/run_tests/package_targets.py @@ -139,9 +139,27 @@ class PythonPackage: 'tools/run_tests/build_package_python.sh') +class PHPPackage: + """Copy PHP PECL package artifact""" + + def __init__(self): + self.name = 'php_package' + self.labels = ['package', 'php', 'linux'] + + def pre_build_jobspecs(self): + return [] + + def build_jobspec(self): + return create_docker_jobspec( + self.name, + 'tools/dockerfile/grpc_artifact_linux_x64', + 'tools/run_tests/build_package_php.sh') + + def targets(): """Gets list of supported targets""" return [CSharpPackage(), NodePackage(), RubyPackage(), - PythonPackage()] + PythonPackage(), + PHPPackage()] diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 0b3efa29e3..7b2bc53716 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -55,8 +55,8 @@ import report_utils import watch_dirs -ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) -os.chdir(ROOT) +_ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) +os.chdir(_ROOT) _FORCE_ENVIRON_FOR_WRAPPERS = {} @@ -118,6 +118,16 @@ def get_c_tests(travis, test_lang) : not (travis and tgt['flaky'])] +def _check_compiler(compiler, supported_compilers): + if compiler not in supported_compilers: + raise Exception('Compiler %s not supported.' % compiler) + + +def _is_use_docker_child(): + """Returns True if running running as a --use_docker child.""" + return True if os.getenv('RUN_TESTS_COMMAND') else False + + class CLanguage(object): def __init__(self, make_target, test_lang): @@ -125,44 +135,55 @@ class CLanguage(object): self.platform = platform_string() self.test_lang = test_lang - def test_specs(self, config, args): + def configure(self, config, args): + self.config = config + self.args = args + if self.platform == 'windows': + self._make_options = [_windows_toolset_option(self.args.compiler), + _windows_arch_option(self.args.arch)] + else: + self._make_options = [] + self._docker_distro = self._get_docker_distro(self.args.use_docker, + self.args.compiler) + + def test_specs(self): out = [] - binaries = get_c_tests(args.travis, self.test_lang) + binaries = get_c_tests(self.args.travis, self.test_lang) for target in binaries: - if config.build_config in target['exclude_configs']: + if self.config.build_config in target['exclude_configs']: continue if self.platform == 'windows': binary = 'vsprojects/%s%s/%s.exe' % ( - 'x64/' if args.arch == 'x64' else '', - _WINDOWS_CONFIG[config.build_config], + 'x64/' if self.args.arch == 'x64' else '', + _MSBUILD_CONFIG[self.config.build_config], target['name']) else: - binary = 'bins/%s/%s' % (config.build_config, target['name']) + binary = 'bins/%s/%s' % (self.config.build_config, target['name']) if os.path.isfile(binary): cmdline = [binary] + target['args'] - out.append(config.job_spec(cmdline, [binary], - shortname=' '.join(cmdline), - cpu_cost=target['cpu_cost'], - environ={'GRPC_DEFAULT_SSL_ROOTS_FILE_PATH': - os.path.abspath(os.path.dirname( - sys.argv[0]) + '/../../src/core/tsi/test_creds/ca.pem')})) - elif args.regex == '.*' or platform_string() == 'windows': + out.append(self.config.job_spec(cmdline, [binary], + shortname=' '.join(cmdline), + cpu_cost=target['cpu_cost'], + environ={'GRPC_DEFAULT_SSL_ROOTS_FILE_PATH': + _ROOT + '/src/core/tsi/test_creds/ca.pem'})) + elif self.args.regex == '.*' or self.platform == 'windows': print '\nWARNING: binary not found, skipping', binary return sorted(out) - def make_targets(self, test_regex): - if platform_string() != 'windows' and test_regex != '.*': + def make_targets(self): + test_regex = self.args.regex + if self.platform != 'windows' and self.args.regex != '.*': # use the regex to minimize the number of things to build return [os.path.basename(target['name']) for target in get_c_tests(False, self.test_lang) if re.search(test_regex, '/' + target['name'])] - if platform_string() == 'windows': + if self.platform == 'windows': # don't build tools on windows just yet return ['buildtests_%s' % self.make_target] return ['buildtests_%s' % self.make_target, 'tools_%s' % self.make_target] def make_options(self): - return [] + return self._make_options; def pre_build_steps(self): if self.platform == 'windows': @@ -182,11 +203,24 @@ class CLanguage(object): def makefile_name(self): return 'Makefile' - def supports_multi_config(self): - return True + def _get_docker_distro(self, use_docker, compiler): + if _is_use_docker_child(): + return "already_under_docker" + if not use_docker: + _check_compiler(compiler, ['default']) + + if compiler == 'gcc4.9' or compiler == 'default': + return 'jessie' + elif compiler == 'gcc4.4': + return 'squeeze' + elif compiler == 'gcc5.3': + return 'ubuntu1604' + else: + raise Exception('Compiler %s not supported.' % compiler) - def dockerfile_dir(self, config, arch): - return 'tools/dockerfile/test/cxx_jessie_%s' % _docker_arch_suffix(arch) + def dockerfile_dir(self): + return 'tools/dockerfile/test/cxx_%s_%s' % (self._docker_distro, + _docker_arch_suffix(self.args.arch)) def __str__(self): return self.make_target @@ -198,13 +232,18 @@ class NodeLanguage(object): self.platform = platform_string() self.node_version = '0.12' - def test_specs(self, config, args): + def configure(self, config, args): + self.config = config + self.args = args + _check_compiler(self.args.compiler, ['default']) + + def test_specs(self): if self.platform == 'windows': - return [config.job_spec(['tools\\run_tests\\run_node.bat'], None)] + return [self.config.job_spec(['tools\\run_tests\\run_node.bat'], None)] else: - return [config.job_spec(['tools/run_tests/run_node.sh', self.node_version], - None, - environ=_FORCE_ENVIRON_FOR_WRAPPERS)] + return [self.config.job_spec(['tools/run_tests/run_node.sh', self.node_version], + None, + environ=_FORCE_ENVIRON_FOR_WRAPPERS)] def pre_build_steps(self): if self.platform == 'windows': @@ -212,7 +251,7 @@ class NodeLanguage(object): else: return [['tools/run_tests/pre_build_node.sh', self.node_version]] - def make_targets(self, test_regex): + def make_targets(self): return [] def make_options(self): @@ -230,11 +269,8 @@ class NodeLanguage(object): def makefile_name(self): return 'Makefile' - def supports_multi_config(self): - return False - - def dockerfile_dir(self, config, arch): - return 'tools/dockerfile/test/node_jessie_%s' % _docker_arch_suffix(arch) + def dockerfile_dir(self): + return 'tools/dockerfile/test/node_jessie_%s' % _docker_arch_suffix(self.args.arch) def __str__(self): return 'node' @@ -242,14 +278,19 @@ class NodeLanguage(object): class PhpLanguage(object): - def test_specs(self, config, args): - return [config.job_spec(['src/php/bin/run_tests.sh'], None, - environ=_FORCE_ENVIRON_FOR_WRAPPERS)] + def configure(self, config, args): + self.config = config + self.args = args + _check_compiler(self.args.compiler, ['default']) + + def test_specs(self): + return [self.config.job_spec(['src/php/bin/run_tests.sh'], None, + environ=_FORCE_ENVIRON_FOR_WRAPPERS)] def pre_build_steps(self): return [] - def make_targets(self, test_regex): + def make_targets(self): return ['static_c', 'shared_c'] def make_options(self): @@ -264,11 +305,8 @@ class PhpLanguage(object): def makefile_name(self): return 'Makefile' - def supports_multi_config(self): - return False - - def dockerfile_dir(self, config, arch): - return 'tools/dockerfile/test/php_jessie_%s' % _docker_arch_suffix(arch) + def dockerfile_dir(self): + return 'tools/dockerfile/test/php_jessie_%s' % _docker_arch_suffix(self.args.arch) def __str__(self): return 'php' @@ -280,10 +318,15 @@ class PythonLanguage(object): self._build_python_versions = ['2.7'] self._has_python_versions = [] - def test_specs(self, config, args): + def configure(self, config, args): + self.config = config + self.args = args + _check_compiler(self.args.compiler, ['default']) + + def test_specs(self): environment = dict(_FORCE_ENVIRON_FOR_WRAPPERS) environment['PYVER'] = '2.7' - return [config.job_spec( + return [self.config.job_spec( ['tools/run_tests/run_python.sh'], None, environ=environment, @@ -294,7 +337,7 @@ class PythonLanguage(object): def pre_build_steps(self): return [] - def make_targets(self, test_regex): + def make_targets(self): return ['static_c', 'grpc_python_plugin', 'shared_c'] def make_options(self): @@ -320,11 +363,8 @@ class PythonLanguage(object): def makefile_name(self): return 'Makefile' - def supports_multi_config(self): - return False - - def dockerfile_dir(self, config, arch): - return 'tools/dockerfile/test/python_jessie_%s' % _docker_arch_suffix(arch) + def dockerfile_dir(self): + return 'tools/dockerfile/test/python_jessie_%s' % _docker_arch_suffix(self.args.arch) def __str__(self): return 'python' @@ -332,15 +372,20 @@ class PythonLanguage(object): class RubyLanguage(object): - def test_specs(self, config, args): - return [config.job_spec(['tools/run_tests/run_ruby.sh'], None, - timeout_seconds=10*60, - environ=_FORCE_ENVIRON_FOR_WRAPPERS)] + def configure(self, config, args): + self.config = config + self.args = args + _check_compiler(self.args.compiler, ['default']) + + def test_specs(self): + return [self.config.job_spec(['tools/run_tests/run_ruby.sh'], None, + timeout_seconds=10*60, + environ=_FORCE_ENVIRON_FOR_WRAPPERS)] def pre_build_steps(self): return [['tools/run_tests/pre_build_ruby.sh']] - def make_targets(self, test_regex): + def make_targets(self): return [] def make_options(self): @@ -355,27 +400,30 @@ class RubyLanguage(object): def makefile_name(self): return 'Makefile' - def supports_multi_config(self): - return False - - def dockerfile_dir(self, config, arch): - return 'tools/dockerfile/test/ruby_jessie_%s' % _docker_arch_suffix(arch) + def dockerfile_dir(self): + return 'tools/dockerfile/test/ruby_jessie_%s' % _docker_arch_suffix(self.args.arch) def __str__(self): return 'ruby' class CSharpLanguage(object): + def __init__(self): self.platform = platform_string() - def test_specs(self, config, args): + def configure(self, config, args): + self.config = config + self.args = args + _check_compiler(self.args.compiler, ['default']) + + def test_specs(self): with open('src/csharp/tests.json') as f: tests_json = json.load(f) assemblies = tests_json['assemblies'] tests = tests_json['tests'] - msbuild_config = _WINDOWS_CONFIG[config.build_config] + msbuild_config = _MSBUILD_CONFIG[self.config.build_config] assembly_files = ['%s/bin/%s/%s.dll' % (a, msbuild_config, a) for a in assemblies] @@ -387,13 +435,13 @@ class CSharpLanguage(object): else: script_name = 'tools/run_tests/run_csharp.sh' - if config.build_config == 'gcov': + if self.config.build_config == 'gcov': # On Windows, we only collect C# code coverage. # On Linux, we only collect coverage for native extension. # For code coverage all tests need to run as one suite. - return [config.job_spec([script_name] + extra_args, None, - shortname='csharp.coverage', - environ=_FORCE_ENVIRON_FOR_WRAPPERS)] + return [self.config.job_spec([script_name] + extra_args, None, + shortname='csharp.coverage', + environ=_FORCE_ENVIRON_FOR_WRAPPERS)] else: specs = [] for test in tests: @@ -402,9 +450,9 @@ class CSharpLanguage(object): # use different output directory for each test to prevent # TestResult.xml clash between parallel test runs. cmdline += ['-work=test-result/%s' % uuid.uuid4()] - specs.append(config.job_spec(cmdline, None, - shortname='csharp.%s' % test, - environ=_FORCE_ENVIRON_FOR_WRAPPERS)) + specs.append(self.config.job_spec(cmdline, None, + shortname='csharp.%s' % test, + environ=_FORCE_ENVIRON_FOR_WRAPPERS)) return specs def pre_build_steps(self): @@ -413,7 +461,7 @@ class CSharpLanguage(object): else: return [['tools/run_tests/pre_build_csharp.sh']] - def make_targets(self, test_regex): + def make_targets(self): # For Windows, this target doesn't really build anything, # everything is build by buildall script later. if self.platform == 'windows': @@ -440,11 +488,8 @@ class CSharpLanguage(object): def makefile_name(self): return 'Makefile' - def supports_multi_config(self): - return False - - def dockerfile_dir(self, config, arch): - return 'tools/dockerfile/test/csharp_jessie_%s' % _docker_arch_suffix(arch) + def dockerfile_dir(self): + return 'tools/dockerfile/test/csharp_jessie_%s' % _docker_arch_suffix(self.args.arch) def __str__(self): return 'csharp' @@ -452,14 +497,19 @@ class CSharpLanguage(object): class ObjCLanguage(object): - def test_specs(self, config, args): - return [config.job_spec(['src/objective-c/tests/run_tests.sh'], None, - environ=_FORCE_ENVIRON_FOR_WRAPPERS)] + def configure(self, config, args): + self.config = config + self.args = args + _check_compiler(self.args.compiler, ['default']) + + def test_specs(self): + return [self.config.job_spec(['src/objective-c/tests/run_tests.sh'], None, + environ=_FORCE_ENVIRON_FOR_WRAPPERS)] def pre_build_steps(self): return [] - def make_targets(self, test_regex): + def make_targets(self): return ['grpc_objective_c_plugin', 'interop_server'] def make_options(self): @@ -474,10 +524,7 @@ class ObjCLanguage(object): def makefile_name(self): return 'Makefile' - def supports_multi_config(self): - return False - - def dockerfile_dir(self, config, arch): + def dockerfile_dir(self): return None def __str__(self): @@ -486,18 +533,23 @@ class ObjCLanguage(object): class Sanity(object): - def test_specs(self, config, args): + def configure(self, config, args): + self.config = config + self.args = args + _check_compiler(self.args.compiler, ['default']) + + def test_specs(self): import yaml with open('tools/run_tests/sanity/sanity_tests.yaml', 'r') as f: - return [config.job_spec(cmd['script'].split(), None, - timeout_seconds=None, environ={'TEST': 'true'}, - cpu_cost=cmd.get('cpu_cost', 1)) + return [self.config.job_spec(cmd['script'].split(), None, + timeout_seconds=None, environ={'TEST': 'true'}, + cpu_cost=cmd.get('cpu_cost', 1)) for cmd in yaml.load(f)] def pre_build_steps(self): return [] - def make_targets(self, test_regex): + def make_targets(self): return ['run_dep_checks'] def make_options(self): @@ -512,55 +564,18 @@ class Sanity(object): def makefile_name(self): return 'Makefile' - def supports_multi_config(self): - return False - - def dockerfile_dir(self, config, arch): + def dockerfile_dir(self): return 'tools/dockerfile/test/sanity' def __str__(self): return 'sanity' -class Build(object): - - def test_specs(self, config, args): - return [] - - def pre_build_steps(self): - return [] - - def make_targets(self, test_regex): - return ['static'] - - def make_options(self): - return [] - - def build_steps(self): - return [] - - def post_tests_steps(self): - return [] - - def makefile_name(self): - return 'Makefile' - - def supports_multi_config(self): - return True - - def dockerfile_dir(self, config, arch): - return None - - def __str__(self): - return self.make_target - - # different configurations we can run under with open('tools/run_tests/configs.json') as f: _CONFIGS = dict((cfg['config'], Config(**cfg)) for cfg in ast.literal_eval(f.read())) -_DEFAULT = ['opt'] _LANGUAGES = { 'c++': CLanguage('cxx', 'c++'), 'c': CLanguage('c', 'c'), @@ -570,11 +585,11 @@ _LANGUAGES = { 'ruby': RubyLanguage(), 'csharp': CSharpLanguage(), 'objc' : ObjCLanguage(), - 'sanity': Sanity(), - 'build': Build(), + 'sanity': Sanity() } -_WINDOWS_CONFIG = { + +_MSBUILD_CONFIG = { 'dbg': 'Debug', 'opt': 'Release', 'gcov': 'Debug', @@ -651,14 +666,6 @@ def _docker_arch_suffix(arch): sys.exit(1) -def _get_dockerfile_dir(language, cfg, arch): - """Returns dockerfile to use""" - custom = language.dockerfile_dir(cfg, arch) - if custom: - return custom - else: - return 'tools/dockerfile/grpc_tests_multilang_%s' % _docker_arch_suffix(arch) - def runs_per_test_type(arg_str): """Auxilary function to parse the "runs_per_test" flag. @@ -682,9 +689,8 @@ def runs_per_test_type(arg_str): # parse command line argp = argparse.ArgumentParser(description='Run grpc tests.') argp.add_argument('-c', '--config', - choices=['all'] + sorted(_CONFIGS.keys()), - nargs='+', - default=_DEFAULT) + choices=sorted(_CONFIGS.keys()), + default='opt') argp.add_argument('-n', '--runs_per_test', default=1, type=runs_per_test_type, help='A positive integer or "inf". If "inf", all tests will run in an ' 'infinite loop. Especially useful in combination with "-f"') @@ -728,9 +734,11 @@ argp.add_argument('--arch', default='default', help='Selects architecture to target. For some platforms "default" is the only supported choice.') argp.add_argument('--compiler', - choices=['default', 'vs2010', 'vs2013', 'vs2015'], + choices=['default', + 'gcc4.4', 'gcc4.9', 'gcc5.3', + 'vs2010', 'vs2013', 'vs2015'], default='default', - help='Selects compiler to use. For some platforms "default" is the only supported choice.') + help='Selects compiler to use. Allowed values depend on the platform and language.') argp.add_argument('--build_only', default=False, action='store_const', @@ -776,11 +784,8 @@ if need_to_regenerate_projects: # grab config -run_configs = set(_CONFIGS[cfg] - for cfg in itertools.chain.from_iterable( - _CONFIGS.iterkeys() if x == 'all' else [x] - for x in args.config)) -build_configs = set(cfg.build_config for cfg in run_configs) +run_config = _CONFIGS[args.config] +build_config = run_config.build_config if args.travis: _FORCE_ENVIRON_FOR_WRAPPERS = {'GRPC_TRACE': 'api'} @@ -791,17 +796,13 @@ else: lang_list = args.language # We don't support code coverage on some languages if 'gcov' in args.config: - for bad in ['objc', 'sanity', 'build']: + for bad in ['objc', 'sanity']: if bad in lang_list: lang_list.remove(bad) languages = set(_LANGUAGES[l] for l in lang_list) - -if len(build_configs) > 1: - for language in languages: - if not language.supports_multi_config(): - print language, 'does not support multiple build configurations' - sys.exit(1) +for l in languages: + l.configure(run_config, args) language_make_options=[] if any(language.make_options() for language in languages): @@ -811,10 +812,6 @@ if any(language.make_options() for language in languages): else: language_make_options = next(iter(languages)).make_options() -if len(languages) != 1 or len(build_configs) != 1: - print 'Multi-language and multi-config testing is not supported.' - sys.exit(1) - if args.use_docker: if not args.travis: print 'Seen --use_docker flag, will run tests under docker.' @@ -824,14 +821,18 @@ if args.use_docker: print 'copied to the docker environment.' time.sleep(5) + dockerfile_dirs = set([l.dockerfile_dir() for l in languages]) + if len(dockerfile_dirs) > 1: + print 'Languages to be tested require running under different docker images.' + sys.exit(1) + dockerfile_dir = next(iter(dockerfile_dirs)) + child_argv = [ arg for arg in sys.argv if not arg == '--use_docker' ] run_tests_cmd = 'python tools/run_tests/run_tests.py %s' % ' '.join(child_argv[1:]) env = os.environ.copy() env['RUN_TESTS_COMMAND'] = run_tests_cmd - env['DOCKERFILE_DIR'] = _get_dockerfile_dir(next(iter(languages)), - next(iter(build_configs)), - args.arch) + env['DOCKERFILE_DIR'] = dockerfile_dir env['DOCKER_RUN_SCRIPT'] = 'tools/jenkins/docker_run_tests.sh' if args.xml_report: env['XML_REPORT'] = args.xml_report @@ -843,10 +844,6 @@ if args.use_docker: env=env) sys.exit(0) -if platform_string() != 'windows' and args.compiler != 'default': - print 'Compiler %s not supported on current platform.' % args.compiler - sys.exit(1) - _check_arch_option(args.arch) def make_jobspec(cfg, targets, makefile='Makefile'): @@ -861,9 +858,7 @@ def make_jobspec(cfg, targets, makefile='Makefile'): return [ jobset.JobSpec([_windows_build_bat(args.compiler), 'vsprojects\\%s.sln' % target, - '/p:Configuration=%s' % _WINDOWS_CONFIG[cfg], - _windows_toolset_option(args.compiler), - _windows_arch_option(args.arch)] + + '/p:Configuration=%s' % _MSBUILD_CONFIG[cfg]] + extra_args + language_make_options, shell=True, timeout_seconds=None) @@ -886,32 +881,29 @@ make_targets = {} for l in languages: makefile = l.makefile_name() make_targets[makefile] = make_targets.get(makefile, set()).union( - set(l.make_targets(args.regex))) + set(l.make_targets())) def build_step_environ(cfg): environ = {'CONFIG': cfg} - msbuild_cfg = _WINDOWS_CONFIG.get(cfg) + msbuild_cfg = _MSBUILD_CONFIG.get(cfg) if msbuild_cfg: environ['MSBUILD_CONFIG'] = msbuild_cfg return environ build_steps = list(set( - jobset.JobSpec(cmdline, environ=build_step_environ(cfg), flake_retries=5) - for cfg in build_configs + jobset.JobSpec(cmdline, environ=build_step_environ(build_config), flake_retries=5) for l in languages for cmdline in l.pre_build_steps())) if make_targets: - make_commands = itertools.chain.from_iterable(make_jobspec(cfg, list(targets), makefile) for cfg in build_configs for (makefile, targets) in make_targets.iteritems()) + make_commands = itertools.chain.from_iterable(make_jobspec(build_config, list(targets), makefile) for (makefile, targets) in make_targets.iteritems()) build_steps.extend(set(make_commands)) build_steps.extend(set( - jobset.JobSpec(cmdline, environ=build_step_environ(cfg), timeout_seconds=None) - for cfg in build_configs + jobset.JobSpec(cmdline, environ=build_step_environ(build_config), timeout_seconds=None) for l in languages for cmdline in l.build_steps())) post_tests_steps = list(set( - jobset.JobSpec(cmdline, environ=build_step_environ(cfg)) - for cfg in build_configs + jobset.JobSpec(cmdline, environ=build_step_environ(build_config)) for l in languages for cmdline in l.post_tests_steps())) runs_per_test = args.runs_per_test @@ -1024,7 +1016,7 @@ def _start_port_server(port_server_port): print 'last ditch attempt to contact port server succeeded' break except: - traceback.print_exc(); + traceback.print_exc() port_log = open(logfile, 'r').read() print port_log sys.exit(1) @@ -1044,7 +1036,7 @@ def _start_port_server(port_server_port): time.sleep(1) waits += 1 except: - traceback.print_exc(); + traceback.print_exc() port_server.kill() raise @@ -1102,9 +1094,8 @@ def _build_and_run( infinite_runs = runs_per_test == 0 one_run = set( spec - for config in run_configs for language in languages - for spec in language.test_specs(config, args) + for spec in language.test_specs() if re.search(args.regex, spec.shortname)) # When running on travis, we want out test runs to be as similar as possible # for reproducibility purposes. diff --git a/tools/run_tests/sanity/check_submodules.sh b/tools/run_tests/sanity/check_submodules.sh index f49230e49a..3c6dbb9ea1 100755 --- a/tools/run_tests/sanity/check_submodules.sh +++ b/tools/run_tests/sanity/check_submodules.sh @@ -44,6 +44,7 @@ cat << EOF | awk '{ print $1 }' | sort > $want_submodules 9f897b25800d2f54f5c442ef01a60721aeca6d87 third_party/boringssl (version_for_cocoapods_1.0-67-g9f897b2) 05b155ff59114735ec8cd089f669c4c3d8f59029 third_party/gflags (v2.1.0-45-g05b155f) c99458533a9b4c743ed51537e25989ea55944908 third_party/googletest (release-1.7.0) + f8ac463766281625ad710900479130c7fcb4d63b third_party/nanopb (nanopb-0.3.4-29-gf8ac463) d5fb408ddc281ffcadeb08699e65bb694656d0bd third_party/protobuf (v3.0.0-beta-2) 50893291621658f355bc5b4d450a8d06a563053d third_party/zlib (v1.2.8) EOF diff --git a/tools/run_tests/sanity/check_version.py b/tools/run_tests/sanity/check_version.py new file mode 100755 index 0000000000..41dd5efe38 --- /dev/null +++ b/tools/run_tests/sanity/check_version.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python2.7 + +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import sys +import yaml +import os +import re +import subprocess + +errors = 0 + +os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '../../..')) + +# hack import paths to pick up extra code +sys.path.insert(0, os.path.abspath('tools/buildgen/plugins')) +from expand_version import Version + +try: + branch_name = subprocess.check_output( + 'git rev-parse --abbrev-ref HEAD', + shell=True) +except: + print 'WARNING: not a git repository' + branch_name = None + +if branch_name is not None: + m = re.match(r'^release-([0-9]+)_([0-9]+)$', branch_name) + if m: + print 'RELEASE branch' + # version number should align with the branched version + check_version = lambda version: ( + version.major == int(m.group(1)) and + version.minor == int(m.group(2))) + warning = 'Version key "%%s" value "%%s" should have a major version %s and minor version %s' % (m.group(1), m.group(2)) + elif re.match(r'^debian/.*$', branch_name): + # no additional version checks for debian branches + check_version = lambda version: True + else: + # all other branches should have a -dev tag + check_version = lambda version: version.tag == 'dev' + warning = 'Version key "%s" value "%s" should have a -dev tag' +else: + check_version = lambda version: True + +with open('build.yaml', 'r') as f: + build_yaml = yaml.load(f.read()) + +settings = build_yaml['settings'] + +top_version = Version(settings['version']) +if not check_version(top_version): + errors += 1 + print warning % ('version', top_version) + +for tag, value in settings.iteritems(): + if re.match(r'^[a-z]+_version$', tag): + value = Version(value) + if value.major != top_version.major: + errors += 1 + print 'major version mismatch on %s: %d vs %d' % (tag, value.major, top_version.major) + if value.minor != top_version.minor: + errors += 1 + print 'minor version mismatch on %s: %d vs %d' % (tag, value.minor, top_version.minor) + if not check_version(value): + errors += 1 + print warning % (tag, value) + +sys.exit(errors) + diff --git a/tools/run_tests/sanity/sanity_tests.yaml b/tools/run_tests/sanity/sanity_tests.yaml index 809e6ce645..cffc180fb0 100644 --- a/tools/run_tests/sanity/sanity_tests.yaml +++ b/tools/run_tests/sanity/sanity_tests.yaml @@ -1,9 +1,11 @@ # a set of tests that are run in parallel for sanity tests +- script: tools/run_tests/sanity/check_cache_mk.sh - script: tools/run_tests/sanity/check_sources_and_headers.py - script: tools/run_tests/sanity/check_submodules.sh -- script: tools/run_tests/sanity/check_cache_mk.sh +- script: tools/run_tests/sanity/check_version.py - script: tools/buildgen/generate_projects.sh -j 3 cpu_cost: 3 - script: tools/distrib/check_copyright.py - script: tools/distrib/clang_format_code.sh - script: tools/distrib/check_trailing_newlines.sh +- script: tools/distrib/check_nanopb_output.sh diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index e44a7bc79e..88e7343399 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -1660,6 +1660,23 @@ }, { "deps": [ + "grpc", + "grpc++", + "grpc++_test_util", + "grpc_test_util" + ], + "headers": [ + "src/proto/grpc/lb/v0/load_balancer.grpc.pb.h", + "src/proto/grpc/lb/v0/load_balancer.pb.h" + ], + "language": "c++", + "name": "grpclb_api_test", + "src": [ + "test/cpp/grpclb/grpclb_api_test.cc" + ] + }, + { + "deps": [ "gpr", "gpr_test_util", "grpc", @@ -2998,6 +3015,7 @@ "src/core/client_config/client_config.h", "src/core/client_config/connector.h", "src/core/client_config/initial_connect_string.h", + "src/core/client_config/lb_policies/load_balancer_api.h", "src/core/client_config/lb_policies/pick_first.h", "src/core/client_config/lb_policies/round_robin.h", "src/core/client_config/lb_policy.h", @@ -3058,6 +3076,7 @@ "src/core/json/json_common.h", "src/core/json/json_reader.h", "src/core/json/json_writer.h", + "src/core/proto/grpc/lb/v0/load_balancer.pb.h", "src/core/security/auth_filters.h", "src/core/security/base64.h", "src/core/security/credentials.h", @@ -3110,7 +3129,11 @@ "src/core/tsi/ssl_transport_security.h", "src/core/tsi/ssl_types.h", "src/core/tsi/transport_security.h", - "src/core/tsi/transport_security_interface.h" + "src/core/tsi/transport_security_interface.h", + "third_party/nanopb/pb.h", + "third_party/nanopb/pb_common.h", + "third_party/nanopb/pb_decode.h", + "third_party/nanopb/pb_encode.h" ], "language": "c", "name": "grpc", @@ -3166,6 +3189,8 @@ "src/core/client_config/default_initial_connect_string.c", "src/core/client_config/initial_connect_string.c", "src/core/client_config/initial_connect_string.h", + "src/core/client_config/lb_policies/load_balancer_api.c", + "src/core/client_config/lb_policies/load_balancer_api.h", "src/core/client_config/lb_policies/pick_first.c", "src/core/client_config/lb_policies/pick_first.h", "src/core/client_config/lb_policies/round_robin.c", @@ -3291,6 +3316,8 @@ "src/core/json/json_string.c", "src/core/json/json_writer.c", "src/core/json/json_writer.h", + "src/core/proto/grpc/lb/v0/load_balancer.pb.c", + "src/core/proto/grpc/lb/v0/load_balancer.pb.h", "src/core/security/auth_filters.h", "src/core/security/base64.c", "src/core/security/base64.h", @@ -3537,6 +3564,7 @@ "src/core/client_config/client_config.h", "src/core/client_config/connector.h", "src/core/client_config/initial_connect_string.h", + "src/core/client_config/lb_policies/load_balancer_api.h", "src/core/client_config/lb_policies/pick_first.h", "src/core/client_config/lb_policies/round_robin.h", "src/core/client_config/lb_policy.h", @@ -3597,6 +3625,7 @@ "src/core/json/json_common.h", "src/core/json/json_reader.h", "src/core/json/json_writer.h", + "src/core/proto/grpc/lb/v0/load_balancer.pb.h", "src/core/statistics/census_interface.h", "src/core/statistics/census_rpc_stats.h", "src/core/surface/api_trace.h", @@ -3635,7 +3664,11 @@ "src/core/transport/metadata_batch.h", "src/core/transport/static_metadata.h", "src/core/transport/transport.h", - "src/core/transport/transport_impl.h" + "src/core/transport/transport_impl.h", + "third_party/nanopb/pb.h", + "third_party/nanopb/pb_common.h", + "third_party/nanopb/pb_decode.h", + "third_party/nanopb/pb_encode.h" ], "language": "c", "name": "grpc_unsecure", @@ -3690,6 +3723,8 @@ "src/core/client_config/default_initial_connect_string.c", "src/core/client_config/initial_connect_string.c", "src/core/client_config/initial_connect_string.h", + "src/core/client_config/lb_policies/load_balancer_api.c", + "src/core/client_config/lb_policies/load_balancer_api.h", "src/core/client_config/lb_policies/pick_first.c", "src/core/client_config/lb_policies/pick_first.h", "src/core/client_config/lb_policies/round_robin.c", @@ -3814,6 +3849,8 @@ "src/core/json/json_string.c", "src/core/json/json_writer.c", "src/core/json/json_writer.h", + "src/core/proto/grpc/lb/v0/load_balancer.pb.c", + "src/core/proto/grpc/lb/v0/load_balancer.pb.h", "src/core/statistics/census_interface.h", "src/core/statistics/census_rpc_stats.h", "src/core/surface/alarm.c", diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index ea9c129101..8868e2a93e 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -2051,6 +2051,26 @@ "exclude_configs": [], "flaky": false, "language": "c++", + "name": "grpclb_api_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ] + }, + { + "args": [], + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c++", "name": "hybrid_end2end_test", "platforms": [ "linux", @@ -2104,7 +2124,7 @@ "mac", "posix" ], - "cpu_cost": 1.0, + "cpu_cost": 10, "exclude_configs": [], "flaky": false, "language": "c++", @@ -2123,9 +2143,7 @@ "posix" ], "cpu_cost": 10, - "exclude_configs": [ - "tsan" - ], + "exclude_configs": [], "flaky": false, "language": "c++", "name": "qps_test", |