diff options
28 files changed, 361 insertions, 76 deletions
@@ -506,7 +506,7 @@ OPENSSL_ALPN_CHECK_CMD = $(PKG_CONFIG) --atleast-version=1.0.2 openssl OPENSSL_NPN_CHECK_CMD = $(PKG_CONFIG) --atleast-version=1.0.1 openssl ZLIB_CHECK_CMD = $(PKG_CONFIG) --exists zlib PROTOBUF_CHECK_CMD = $(PKG_CONFIG) --atleast-version=3.0.0 protobuf -CARES_CHECK_CMD = $(PKG_CONFIG) --exists libcares +CARES_CHECK_CMD = $(PKG_CONFIG) --atleast-version=1.11.0 libcares else # HAS_PKG_CONFIG ifeq ($(SYSTEM),MINGW32) diff --git a/templates/Makefile.template b/templates/Makefile.template index a9dd171b84..54093fbdfe 100644 --- a/templates/Makefile.template +++ b/templates/Makefile.template @@ -427,7 +427,7 @@ OPENSSL_NPN_CHECK_CMD = $(PKG_CONFIG) --atleast-version=1.0.1 openssl ZLIB_CHECK_CMD = $(PKG_CONFIG) --exists zlib PROTOBUF_CHECK_CMD = $(PKG_CONFIG) --atleast-version=3.0.0 protobuf - CARES_CHECK_CMD = $(PKG_CONFIG) --exists libcares + CARES_CHECK_CMD = $(PKG_CONFIG) --atleast-version=1.11.0 libcares else # HAS_PKG_CONFIG ifeq ($(SYSTEM),MINGW32) diff --git a/templates/tools/dockerfile/csharp_deps.include b/templates/tools/dockerfile/csharp_deps.include index 612b119e1c..3a40711e37 100644 --- a/templates/tools/dockerfile/csharp_deps.include +++ b/templates/tools/dockerfile/csharp_deps.include @@ -2,8 +2,8 @@ # C# dependencies # Update to a newer version of mono -RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF -RUN echo "deb http://download.mono-project.com/repo/debian wheezy main" | tee /etc/apt/sources.list.d/mono-xamarin.list +RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF +RUN echo "deb http://download.mono-project.com/repo/debian jessie main" | tee /etc/apt/sources.list.d/mono-official.list RUN echo "deb http://download.mono-project.com/repo/debian wheezy-apache24-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list RUN echo "deb http://download.mono-project.com/repo/debian wheezy-libjpeg62-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list diff --git a/templates/tools/dockerfile/interoptest/grpc_interop_go/build_interop.sh.template b/templates/tools/dockerfile/interoptest/grpc_interop_go/build_interop.sh.template new file mode 100644 index 0000000000..a08798b1d6 --- /dev/null +++ b/templates/tools/dockerfile/interoptest/grpc_interop_go/build_interop.sh.template @@ -0,0 +1,3 @@ +%YAML 1.2 +--- | + <%include file="../../go_build_interop.sh.include"/> diff --git a/templates/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile.template b/templates/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile.template index 3d5e7ee120..eff55345ea 100644 --- a/templates/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile.template +++ b/templates/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile.template @@ -1,45 +1,3 @@ %YAML 1.2 --- | - # Copyright 2015, Google Inc. - # All rights reserved. - # - # Redistribution and use in source and binary forms, with or without - # modification, are permitted provided that the following conditions are - # met: - # - # * Redistributions of source code must retain the above copyright - # notice, this list of conditions and the following disclaimer. - # * Redistributions in binary form must reproduce the above - # copyright notice, this list of conditions and the following disclaimer - # in the documentation and/or other materials provided with the - # distribution. - # * Neither the name of Google Inc. nor the names of its - # contributors may be used to endorse or promote products derived from - # this software without specific prior written permission. - # - # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - FROM debian:jessie - - <%include file="../../java_deps.include"/> - <%include file="../../python_deps.include"/> - - # Trigger download of as many Gradle artifacts as possible. - RUN git clone --recursive --depth 1 https://github.com/grpc/grpc-java.git && ${'\\'} - cd grpc-java && ${'\\'} - ./gradlew :grpc-interop-testing:installDist -PskipCodegen=true && ${'\\'} - rm -r "$(pwd)" - - # Define the default command. - CMD ["bash"] - + <%include file="../grpc_interop_java_oracle8/Dockerfile.include"/> diff --git a/templates/tools/dockerfile/interoptest/grpc_interop_java/build_interop.sh.template b/templates/tools/dockerfile/interoptest/grpc_interop_java/build_interop.sh.template new file mode 100644 index 0000000000..42eb4d8b96 --- /dev/null +++ b/templates/tools/dockerfile/interoptest/grpc_interop_java/build_interop.sh.template @@ -0,0 +1,3 @@ +%YAML 1.2 +--- | + <%include file="../../java_build_interop.sh.include"/> diff --git a/templates/tools/dockerfile/interoptest/grpc_interop_java/java_deps.include b/templates/tools/dockerfile/interoptest/grpc_interop_java/java_deps.include new file mode 100644 index 0000000000..40d70e06d1 --- /dev/null +++ b/templates/tools/dockerfile/interoptest/grpc_interop_java/java_deps.include @@ -0,0 +1,17 @@ +# Install JDK 8 and Git +# +RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections && ${'\\'} + echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee /etc/apt/sources.list.d/webupd8team-java.list && ${'\\'} + echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee -a /etc/apt/sources.list.d/webupd8team-java.list && ${'\\'} + apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886 + +RUN apt-get update && apt-get -y install ${'\\'} + git ${'\\'} + libapr1 ${'\\'} + oracle-java8-installer ${'\\'} + && ${'\\'} + apt-get clean && rm -r /var/cache/oracle-jdk8-installer/ + +ENV JAVA_HOME /usr/lib/jvm/java-8-oracle +ENV PATH $PATH:$JAVA_HOME/bin + diff --git a/templates/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile.include b/templates/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile.include new file mode 100644 index 0000000000..f71fedaa51 --- /dev/null +++ b/templates/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile.include @@ -0,0 +1,42 @@ +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +FROM debian:jessie + +<%include file="java_deps.include"/> +<%include file="../../python_deps.include"/> + +# Trigger download of as many Gradle artifacts as possible. +RUN git clone --recursive --depth 1 https://github.com/grpc/grpc-java.git && ${'\\'} + cd grpc-java && ${'\\'} + ./gradlew :grpc-interop-testing:installDist -PskipCodegen=true && ${'\\'} + rm -r "$(pwd)" + +# Define the default command. +CMD ["bash"]
\ No newline at end of file diff --git a/templates/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile.template b/templates/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile.template new file mode 100644 index 0000000000..8eaa9a9dbb --- /dev/null +++ b/templates/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile.template @@ -0,0 +1,3 @@ +%YAML 1.2 +--- | + <%include file="Dockerfile.include"/> diff --git a/templates/tools/dockerfile/interoptest/grpc_interop_java_oracle8/build_interop.sh.template b/templates/tools/dockerfile/interoptest/grpc_interop_java_oracle8/build_interop.sh.template new file mode 100644 index 0000000000..db5d40d548 --- /dev/null +++ b/templates/tools/dockerfile/interoptest/grpc_interop_java_oracle8/build_interop.sh.template @@ -0,0 +1,3 @@ +%YAML 1.2 +--- | + <%include file="../../java_build_interop.sh.include"/> diff --git a/templates/tools/dockerfile/interoptest/grpc_interop_java_oracle8/java_deps.include b/templates/tools/dockerfile/interoptest/grpc_interop_java_oracle8/java_deps.include new file mode 100644 index 0000000000..40d70e06d1 --- /dev/null +++ b/templates/tools/dockerfile/interoptest/grpc_interop_java_oracle8/java_deps.include @@ -0,0 +1,17 @@ +# Install JDK 8 and Git +# +RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections && ${'\\'} + echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee /etc/apt/sources.list.d/webupd8team-java.list && ${'\\'} + echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee -a /etc/apt/sources.list.d/webupd8team-java.list && ${'\\'} + apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886 + +RUN apt-get update && apt-get -y install ${'\\'} + git ${'\\'} + libapr1 ${'\\'} + oracle-java8-installer ${'\\'} + && ${'\\'} + apt-get clean && rm -r /var/cache/oracle-jdk8-installer/ + +ENV JAVA_HOME /usr/lib/jvm/java-8-oracle +ENV PATH $PATH:$JAVA_HOME/bin + diff --git a/templates/tools/dockerfile/java_build_interop.sh.include b/templates/tools/dockerfile/java_build_interop.sh.include new file mode 100755 index 0000000000..9997c63308 --- /dev/null +++ b/templates/tools/dockerfile/java_build_interop.sh.include @@ -0,0 +1,42 @@ +#!/bin/bash +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Builds Java interop server and client in a base image. +set -e + +mkdir -p /var/local/git +git clone --recursive --depth 1 /var/local/jenkins/grpc-java /var/local/git/grpc-java + +# copy service account keys if available +cp -r /var/local/jenkins/service_account $HOME || true + +cd /var/local/git/grpc-java + +./gradlew :grpc-interop-testing:installDist -PskipCodegen=true diff --git a/test/build/c-ares.c b/test/build/c-ares.c index c954e9397f..d86bbf5af6 100644 --- a/test/build/c-ares.c +++ b/test/build/c-ares.c @@ -33,6 +33,10 @@ #include <ares.h> +#if (ARES_VERSION < 0x010b00) + ARES_VERSION should not be smaller than 1.11.0 +#endif + int main(void) { ares_channel channelptr; diff --git a/test/cpp/microbenchmarks/bm_fullstack_trickle.cc b/test/cpp/microbenchmarks/bm_fullstack_trickle.cc index 9f616fe152..c201e2e867 100644 --- a/test/cpp/microbenchmarks/bm_fullstack_trickle.cc +++ b/test/cpp/microbenchmarks/bm_fullstack_trickle.cc @@ -101,8 +101,6 @@ class TrickledCHTTP2 : public EndpointPairFixture { } void AddToLabel(std::ostream& out, benchmark::State& state) { - grpc_chttp2_transport* client = - reinterpret_cast<grpc_chttp2_transport*>(client_transport_); out << " writes/iter:" << ((double)stats_.num_writes / (double)state.iterations()) << " cli_transport_stalls/iter:" @@ -118,8 +116,7 @@ class TrickledCHTTP2 : public EndpointPairFixture { (double)state.iterations()) << " svr_stream_stalls/iter:" << ((double)server_stats_.streams_stalled_due_to_stream_flow_control / - (double)state.iterations()) - << " cli_bw_est:" << (double)client->bdp_estimator.bw_est; + (double)state.iterations()); } void Log(int64_t iteration) { @@ -180,7 +177,6 @@ class TrickledCHTTP2 : public EndpointPairFixture { size_t server_backlog = grpc_trickle_endpoint_trickle(&exec_ctx, endpoint_pair_.server); grpc_exec_ctx_finish(&exec_ctx); - if (update_stats) { UpdateStats((grpc_chttp2_transport*)client_transport_, &client_stats_, client_backlog); @@ -374,7 +370,7 @@ static void BM_PumpUnbalancedUnary_Trickle(benchmark::State& state) { stub->AsyncEcho(&cli_ctx, send_request, fixture->cq())); void* t; bool ok; - TrickleCQNext(fixture.get(), &t, &ok, state.iterations()); + TrickleCQNext(fixture.get(), &t, &ok, in_warmup ? -1 : state.iterations()); GPR_ASSERT(ok); GPR_ASSERT(t == tag(0) || t == tag(1)); intptr_t slot = reinterpret_cast<intptr_t>(t); @@ -382,7 +378,8 @@ static void BM_PumpUnbalancedUnary_Trickle(benchmark::State& state) { senv->response_writer.Finish(send_response, Status::OK, tag(3)); response_reader->Finish(&recv_response, &recv_status, tag(4)); for (int i = (1 << 3) | (1 << 4); i != 0;) { - TrickleCQNext(fixture.get(), &t, &ok, state.iterations()); + TrickleCQNext(fixture.get(), &t, &ok, + in_warmup ? -1 : state.iterations()); GPR_ASSERT(ok); int tagnum = (int)reinterpret_cast<intptr_t>(t); GPR_ASSERT(i & (1 << tagnum)); @@ -419,17 +416,15 @@ static void BM_PumpUnbalancedUnary_Trickle(benchmark::State& state) { } static void UnaryTrickleArgs(benchmark::internal::Benchmark* b) { - // A selection of interesting numbers - const int cli_1024k = 1024 * 1024; - const int cli_32M = 32 * 1024 * 1024; - const int svr_256k = 256 * 1024; - const int svr_4M = 4 * 1024 * 1024; - const int svr_64M = 64 * 1024 * 1024; for (int bw = 64; bw <= 128 * 1024 * 1024; bw *= 16) { - for (auto svr : {svr_256k, svr_4M, svr_64M}) { - for (auto cli : {cli_1024k, cli_32M}) { - b->Args({cli, svr, bw}); - } + b->Args({1, 1, bw}); + for (int i = 64; i <= 128 * 1024 * 1024; i *= 64) { + double expected_time = + static_cast<double>(14 + i) / (125.0 * static_cast<double>(bw)); + if (expected_time > 2.0) continue; + b->Args({i, 1, bw}); + b->Args({1, i, bw}); + b->Args({i, i, bw}); } } } diff --git a/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile index f9e709dccb..90624e6010 100644 --- a/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile @@ -82,8 +82,8 @@ RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 # C# dependencies # Update to a newer version of mono -RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF -RUN echo "deb http://download.mono-project.com/repo/debian wheezy main" | tee /etc/apt/sources.list.d/mono-xamarin.list +RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF +RUN echo "deb http://download.mono-project.com/repo/debian jessie main" | tee /etc/apt/sources.list.d/mono-official.list RUN echo "deb http://download.mono-project.com/repo/debian wheezy-apache24-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list RUN echo "deb http://download.mono-project.com/repo/debian wheezy-libjpeg62-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list diff --git a/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile index f9e709dccb..90624e6010 100644 --- a/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile @@ -82,8 +82,8 @@ RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 # C# dependencies # Update to a newer version of mono -RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF -RUN echo "deb http://download.mono-project.com/repo/debian wheezy main" | tee /etc/apt/sources.list.d/mono-xamarin.list +RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF +RUN echo "deb http://download.mono-project.com/repo/debian jessie main" | tee /etc/apt/sources.list.d/mono-official.list RUN echo "deb http://download.mono-project.com/repo/debian wheezy-apache24-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list RUN echo "deb http://download.mono-project.com/repo/debian wheezy-libjpeg62-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list diff --git a/tools/dockerfile/interoptest/grpc_interop_go/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_go/build_interop.sh index 46aabc6b38..5eca90cdea 100755 --- a/tools/dockerfile/interoptest/grpc_interop_go/build_interop.sh +++ b/tools/dockerfile/interoptest/grpc_interop_go/build_interop.sh @@ -45,3 +45,4 @@ cp -r /var/local/jenkins/service_account $HOME || true # Build the interop client and server (cd src/google.golang.org/grpc/interop/client && go install) (cd src/google.golang.org/grpc/interop/server && go install) + diff --git a/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile index 2023467d59..61405ac172 100644 --- a/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2017, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/dockerfile/interoptest/grpc_interop_java/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_java/build_interop.sh index 9997c63308..452b92eb05 100755..100644 --- a/tools/dockerfile/interoptest/grpc_interop_java/build_interop.sh +++ b/tools/dockerfile/interoptest/grpc_interop_java/build_interop.sh @@ -40,3 +40,4 @@ cp -r /var/local/jenkins/service_account $HOME || true cd /var/local/git/grpc-java ./gradlew :grpc-interop-testing:installDist -PskipCodegen=true + diff --git a/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile new file mode 100644 index 0000000000..61405ac172 --- /dev/null +++ b/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile @@ -0,0 +1,73 @@ +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +FROM debian:jessie + +# Install JDK 8 and Git +# +RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections && \ + echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee /etc/apt/sources.list.d/webupd8team-java.list && \ + echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee -a /etc/apt/sources.list.d/webupd8team-java.list && \ + apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886 + +RUN apt-get update && apt-get -y install \ + git \ + libapr1 \ + oracle-java8-installer \ + && \ + apt-get clean && rm -r /var/cache/oracle-jdk8-installer/ + +ENV JAVA_HOME /usr/lib/jvm/java-8-oracle +ENV PATH $PATH:$JAVA_HOME/bin + + +#==================== +# Python dependencies + +# Install dependencies + +RUN apt-get update && apt-get install -y \ + python-all-dev \ + python3-all-dev \ + python-pip + +# Install Python packages from PyPI +RUN pip install pip --upgrade +RUN pip install virtualenv +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 + + +# Trigger download of as many Gradle artifacts as possible. +RUN git clone --recursive --depth 1 https://github.com/grpc/grpc-java.git && \ + cd grpc-java && \ + ./gradlew :grpc-interop-testing:installDist -PskipCodegen=true && \ + rm -r "$(pwd)" + +# Define the default command. +CMD ["bash"] diff --git a/tools/dockerfile/interoptest/grpc_interop_java_oracle8/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_java_oracle8/build_interop.sh new file mode 100644 index 0000000000..32eec5a0b7 --- /dev/null +++ b/tools/dockerfile/interoptest/grpc_interop_java_oracle8/build_interop.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Builds Java interop server and client in a base image. +set -e + +mkdir -p /var/local/git +git clone --recursive --depth 1 /var/local/jenkins/grpc-java /var/local/git/grpc-java + +# copy service account keys if available +cp -r /var/local/jenkins/service_account $HOME || true + +cd /var/local/git/grpc-java + +./gradlew :grpc-interop-testing:installDist -PskipCodegen=true + diff --git a/tools/dockerfile/test/csharp_coreclr_x64/Dockerfile b/tools/dockerfile/test/csharp_coreclr_x64/Dockerfile index f9e709dccb..90624e6010 100644 --- a/tools/dockerfile/test/csharp_coreclr_x64/Dockerfile +++ b/tools/dockerfile/test/csharp_coreclr_x64/Dockerfile @@ -82,8 +82,8 @@ RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 # C# dependencies # Update to a newer version of mono -RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF -RUN echo "deb http://download.mono-project.com/repo/debian wheezy main" | tee /etc/apt/sources.list.d/mono-xamarin.list +RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF +RUN echo "deb http://download.mono-project.com/repo/debian jessie main" | tee /etc/apt/sources.list.d/mono-official.list RUN echo "deb http://download.mono-project.com/repo/debian wheezy-apache24-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list RUN echo "deb http://download.mono-project.com/repo/debian wheezy-libjpeg62-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list diff --git a/tools/dockerfile/test/csharp_jessie_x64/Dockerfile b/tools/dockerfile/test/csharp_jessie_x64/Dockerfile index 0b21a22226..7c2a2df30c 100644 --- a/tools/dockerfile/test/csharp_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/csharp_jessie_x64/Dockerfile @@ -86,8 +86,8 @@ RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 # C# dependencies # Update to a newer version of mono -RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF -RUN echo "deb http://download.mono-project.com/repo/debian wheezy main" | tee /etc/apt/sources.list.d/mono-xamarin.list +RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF +RUN echo "deb http://download.mono-project.com/repo/debian jessie main" | tee /etc/apt/sources.list.d/mono-official.list RUN echo "deb http://download.mono-project.com/repo/debian wheezy-apache24-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list RUN echo "deb http://download.mono-project.com/repo/debian wheezy-libjpeg62-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list diff --git a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile index 9b50d85e2f..d85411810d 100644 --- a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile @@ -71,8 +71,8 @@ RUN pip install --upgrade google-api-python-client # C# dependencies # Update to a newer version of mono -RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF -RUN echo "deb http://download.mono-project.com/repo/debian wheezy main" | tee /etc/apt/sources.list.d/mono-xamarin.list +RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF +RUN echo "deb http://download.mono-project.com/repo/debian jessie main" | tee /etc/apt/sources.list.d/mono-official.list RUN echo "deb http://download.mono-project.com/repo/debian wheezy-apache24-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list RUN echo "deb http://download.mono-project.com/repo/debian wheezy-libjpeg62-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list diff --git a/tools/interop_matrix/client_matrix.py b/tools/interop_matrix/client_matrix.py index b06b0b7205..8ee468ada7 100644 --- a/tools/interop_matrix/client_matrix.py +++ b/tools/interop_matrix/client_matrix.py @@ -39,10 +39,12 @@ def get_github_repo(lang): # Dictionary of runtimes per language LANG_RUNTIME_MATRIX = { 'go': ['go1.7', 'go1.8'], + 'java': ['java_oracle8'], } # Dictionary of releases per language. For each language, we need to provide # a tuple of release tag (used as the tag for the GCR image) and also github hash. LANG_RELEASE_MATRIX = { 'go': ['v1.0.1-GA', 'v1.3.0'], + 'java': ['v1.0.3', 'v1.1.2'], } diff --git a/tools/interop_matrix/testcases/java__master b/tools/interop_matrix/testcases/java__master new file mode 100755 index 0000000000..cf431646e9 --- /dev/null +++ b/tools/interop_matrix/testcases/java__master @@ -0,0 +1,11 @@ +#!/bin/bash +echo "Testing ${docker_image:=grpc_interop_java:ea528843-be34-4ff3-a136-e4609424e061}" +docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=large_unary" +docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=empty_unary" +docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=ping_pong" +docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=empty_stream" +docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=client_streaming" +docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=server_streaming" +docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=cancel_after_begin" +docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=cancel_after_first_response" +docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=timeout_on_sleeping_server" diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 6da7b85430..b0435e53e3 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -202,6 +202,28 @@ class JavaLanguage: return 'java' +class JavaOkHttpClient: + + def __init__(self): + self.client_cwd = '../grpc-java' + self.safename = 'java' + + def client_cmd(self, args): + return ['./run-test-client.sh', '--use_okhttp=true'] + args + + def cloud_to_prod_env(self): + return {} + + def global_env(self): + return {} + + def unimplemented_test_cases(self): + return _SKIP_COMPRESSION + _SKIP_DATA_FRAME_PADDING + + def __str__(self): + return 'javaokhttp' + + class GoLanguage: def __init__(self): @@ -489,6 +511,7 @@ _LANGUAGES = { 'csharpcoreclr' : CSharpCoreCLRLanguage(), 'go' : GoLanguage(), 'java' : JavaLanguage(), + 'javaokhttp' : JavaOkHttpClient(), 'node' : NodeLanguage(), 'php' : PHPLanguage(), 'php7' : PHP7Language(), diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 204ed5c397..1155893328 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -65,6 +65,10 @@ try: except (ImportError): pass # It's ok to not import because this is only necessary to upload results to BQ. +gcp_utils_dir = os.path.abspath(os.path.join( + os.path.dirname(__file__), '../gcp/utils')) +sys.path.append(gcp_utils_dir) + _ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) os.chdir(_ROOT) @@ -73,7 +77,6 @@ _FORCE_ENVIRON_FOR_WRAPPERS = { 'GRPC_VERBOSITY': 'DEBUG', } - _POLLING_STRATEGIES = { 'linux': ['epollsig', 'poll', 'poll-cv'], # TODO(ctiller, sreecha): enable epoll1, epollex, epoll-thread-pool @@ -81,6 +84,35 @@ _POLLING_STRATEGIES = { } +def get_flaky_tests(limit=None): + import big_query_utils + + bq = big_query_utils.create_big_query() + query = """ + SELECT + test_name, + SUM(result != 'PASSED' + AND result != 'SKIPPED') AS count_failed, + FROM + [grpc-testing:jenkins_test_results.aggregate_results] + WHERE + timestamp >= DATE_ADD(DATE(CURRENT_TIMESTAMP()), -1, "WEEK") + AND NOT REGEXP_MATCH(job_name, '.*portability.*') + AND REGEXP_MATCH(job_name, '.*master.*') + GROUP BY + test_name + HAVING + count_failed > 0""" + if limit: + query += " limit {}".format(limit) + query_job = big_query_utils.sync_query_job(bq, 'grpc-testing', query) + page = bq.jobs().getQueryResults( + pageToken=None, + **query_job['jobReference']).execute(num_retries=3) + flake_names = [row['f'][0]['v'] for row in page['rows']] + return flake_names + + def platform_string(): return jobset.platform_string() @@ -119,6 +151,9 @@ class Config(object): actual_environ = self.environ.copy() for k, v in environ.items(): actual_environ[k] = v + if not flaky and shortname and shortname in flaky_tests: + print('Setting %s to flaky' % shortname) + flaky = True return jobset.JobSpec(cmdline=self.tool_prefix + cmdline, shortname=shortname, environ=actual_environ, @@ -1212,8 +1247,17 @@ argp.add_argument('--bq_result_table', type=str, nargs='?', help='Upload test results to a specified BQ table.') +argp.add_argument('--auto_set_flakes', default=True, type=bool, + help='Set flakiness data from historic data') args = argp.parse_args() +flaky_tests = set() +if args.auto_set_flakes: + try: + flaky_tests = set(get_flaky_tests()) + except: + print("Unexpected error getting flaky tests:", sys.exc_info()[0]) + if args.force_default_poller: _POLLING_STRATEGIES = {} |