Merge branch 'master' into limit_metadata_size

6 files changed, 491 insertions, 4 deletions

diff --git a/tools/buildgen/plugins/list_api.py b/tools/buildgen/plugins/list_api.py
index ff937a0ab8..1fc4f4123c 100755
--- a/tools/buildgen/plugins/list_api.py
+++ b/tools/buildgen/plugins/list_api.py
@@ -64,12 +64,13 @@ def headers_under(directory):
 
 def mako_plugin(dictionary):
   apis = []
+  headers = []
 
-#  for lib in dictionary['libs']:
-#    if lib['name'] == 'grpc':
-#      apis.extend(list_c_apis(lib['public_headers']))
-  apis.extend(list_c_apis(sorted(headers_under('include/grpc'))))
+  for lib in dictionary['libs']:
+    if lib['name'] in ['grpc', 'gpr']:
+      headers.extend(lib['public_headers'])
 
+  apis.extend(list_c_apis(sorted(set(headers))))
   dictionary['c_apis'] = apis
 
 
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_csharp/Dockerfile b/tools/dockerfile/stress_test/grpc_interop_stress_csharp/Dockerfile
new file mode 100644
index 0000000000..823fe948fb
--- /dev/null
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_csharp/Dockerfile
@@ -0,0 +1,101 @@
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+FROM debian:jessie
+
+# Install Git and basic packages.
+RUN apt-get update && apt-get install -y \
+  autoconf \
+  autotools-dev \
+  build-essential \
+  bzip2 \
+  ccache \
+  curl \
+  gcc \
+  gcc-multilib \
+  git \
+  golang \
+  gyp \
+  lcov \
+  libc6 \
+  libc6-dbg \
+  libc6-dev \
+  libgtest-dev \
+  libtool \
+  make \
+  perl \
+  strace \
+  python-dev \
+  python-setuptools \
+  python-yaml \
+  telnet \
+  unzip \
+  wget \
+  zip && apt-get clean
+
+#================
+# Build profiling
+RUN apt-get update && apt-get install -y time && apt-get clean
+
+# Prepare ccache
+RUN ln -s /usr/bin/ccache /usr/local/bin/gcc
+RUN ln -s /usr/bin/ccache /usr/local/bin/g++
+RUN ln -s /usr/bin/ccache /usr/local/bin/cc
+RUN ln -s /usr/bin/ccache /usr/local/bin/c++
+RUN ln -s /usr/bin/ccache /usr/local/bin/clang
+RUN ln -s /usr/bin/ccache /usr/local/bin/clang++
+
+#=================
+# C++ dependencies
+RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang && apt-get clean
+
+# Google Cloud platform API libraries
+RUN apt-get update && apt-get install -y python-pip && apt-get clean
+RUN pip install --upgrade google-api-python-client
+
+
+#================
+# C# dependencies
+
+# Update to a newer version of mono
+RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
+RUN echo "deb http://download.mono-project.com/repo/debian wheezy main" | tee /etc/apt/sources.list.d/mono-xamarin.list
+RUN echo "deb http://download.mono-project.com/repo/debian wheezy-apache24-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list
+RUN echo "deb http://download.mono-project.com/repo/debian wheezy-libjpeg62-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list
+RUN echo "deb http://download.mono-project.com/repo/debian wheezy-libtiff-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list
+
+# Install dependencies
+RUN apt-get update && apt-get -y dist-upgrade && apt-get install -y \
+    mono-devel \
+    ca-certificates-mono \
+    nuget \
+    && apt-get clean
+
+# Define the default command.
+CMD ["bash"]
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_csharp/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_csharp/build_interop_stress.sh
new file mode 100755
index 0000000000..1f4bf893cc
--- /dev/null
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_csharp/build_interop_stress.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Builds C# interop server and client in a base image.
+set -e
+
+mkdir -p /var/local/git
+git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+
+# Copy service account keys if available
+cp -r /var/local/jenkins/service_account $HOME || true
+
+cd /var/local/git/grpc
+
+# Build C++ metrics client (to query the metrics from csharp stress client)
+make metrics_client -j
+
+# Build C# interop client & server
+tools/run_tests/run_tests.py -l csharp -c dbg --build_only
+
diff --git a/tools/run_tests/build_package_ruby.sh b/tools/run_tests/build_package_ruby.sh
index 1a5b94348d..e44428bf7e 100755
--- a/tools/run_tests/build_package_ruby.sh
+++ b/tools/run_tests/build_package_ruby.sh
@@ -32,6 +32,8 @@ set -ex
 
 cd $(dirname $0)/../..
 
+base=$(pwd)
+
 mkdir -p artifacts/
 
 # All the ruby packages have been built in the artifact phase already
@@ -41,3 +43,25 @@ cp -r $EXTERNAL_GIT_ROOT/architecture={x86,x64},language=ruby,platform={windows,
 # TODO: all the artifact builder configurations generate a grpc-VERSION.gem
 # source distribution package, and only one of them will end up
 # in the artifacts/ directory. They should be all equivalent though.
+
+for arch in {x86,x64}; do
+  case $arch in
+    x64)
+      ruby_arch=x86_64
+      ;;
+    *)
+      ruby_arch=$arch
+      ;;
+  esac
+  for plat in {windows,linux,macos}; do
+    input_dir="$EXTERNAL_GIT_ROOT/architecture=$arch,language=protoc,platform=$plat/artifacts"
+    output_dir="$base/src/ruby/tools/bin/${ruby_arch}-${plat}"
+    mkdir -p $output_dir
+    cp $input_dir/protoc* $output_dir/
+    cp $input_dir/grpc_ruby_plugin* $output_dir/
+  done
+done
+
+cd $base/src/ruby/tools
+gem build grpc-tools.gemspec
+cp ./grpc-tools*.gem $base/artifacts/
diff --git a/tools/run_tests/stress_test/configs/csharp.json b/tools/run_tests/stress_test/configs/csharp.json
new file mode 100644
index 0000000000..b7090696b4
--- /dev/null
+++ b/tools/run_tests/stress_test/configs/csharp.json
@@ -0,0 +1,90 @@
+{
+  "dockerImages": {
+    "grpc_stress_csharp" : {
+      "buildScript": "tools/jenkins/build_interop_stress_image.sh",
+      "dockerFileDir": "grpc_interop_stress_csharp"
+    }
+  },
+
+  "clientTemplates": {
+    "baseTemplates": {
+      "default": {
+        "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py",
+        "pollIntervalSecs": 60,
+        "clientArgs": {
+          "num_channels_per_server":5,
+          "num_stubs_per_channel":10,
+          "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1",
+          "metrics_port": 8081
+        },
+        "metricsPort": 8081,
+        "metricsArgs": {
+          "metrics_server_address": "localhost:8081",
+          "total_only": "true"
+        }
+      }
+    },
+    "templates": {
+      "csharp_client": {
+        "baseTemplate": "default",
+        "stressClientCmd": [
+          "mono",
+          "/var/local/git/grpc/src/csharp/Grpc.IntegrationTesting.StressClient/bin/Debug/Grpc.IntegrationTesting.StressClient.exe"
+		],
+        "metricsClientCmd": ["/var/local/git/grpc/bins/opt/metrics_client"]
+      }
+    }
+  },
+
+  "serverTemplates": {
+    "baseTemplates":{
+      "default": {
+        "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py",
+        "serverPort": 8080,
+        "serverArgs": {
+          "port": 8080
+        }
+      }
+    },
+    "templates": {
+      "csharp_server": {
+        "baseTemplate": "default",
+        "stressServerCmd": [
+          "mono",
+          "/var/local/git/grpc/src/csharp/Grpc.IntegrationTesting.Server/bin/Debug/Grpc.IntegrationTesting.Server.exe"
+		]
+      }
+    }
+  },
+
+  "testMatrix": {
+    "serverPodSpecs": {
+      "stress-server-csharp": {
+        "serverTemplate": "csharp_server",
+        "dockerImage": "grpc_stress_csharp",
+        "numInstances": 1
+      }
+    },
+
+    "clientPodSpecs": {
+      "stress-client-csharp": {
+        "clientTemplate": "csharp_client",
+        "dockerImage": "grpc_stress_csharp",
+        "numInstances": 10,
+        "serverPodSpec": "stress-server-csharp"
+      }
+    }
+  },
+
+  "globalSettings": {
+    "buildDockerImages": true,
+    "pollIntervalSecs": 60,
+    "testDurationSecs": 7200,
+    "kubernetesProxyPort": 8001,
+    "datasetIdNamePrefix": "stress_test_csharp",
+    "summaryTableId": "summary",
+    "qpsTableId": "qps",
+    "podWarmupSecs": 60
+  }
+}
+
diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json
index 4a6c137327..511b63e8e0 100644
--- a/tools/run_tests/tests.json
+++ b/tools/run_tests/tests.json
@@ -57780,6 +57780,22 @@
   }, 
   {
     "args": [
+      "test/core/end2end/fuzzers/server_fuzzer_corpus/03a72675e1969f836094f1ecfec2a7b34418e306"
+    ], 
+    "ci_platforms": [
+      "linux"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "server_fuzzer_one_entry", 
+    "platforms": [
+      "linux"
+    ]
+  }, 
+  {
+    "args": [
       "test/core/end2end/fuzzers/server_fuzzer_corpus/03b9be1fa172dff5d1543be079b9c64fa2c9a278"
     ], 
     "ci_platforms": [
@@ -57796,6 +57812,22 @@
   }, 
   {
     "args": [
+      "test/core/end2end/fuzzers/server_fuzzer_corpus/0416afd6875d9ba55f1e5f86a6456a5445d5e576"
+    ], 
+    "ci_platforms": [
+      "linux"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "server_fuzzer_one_entry", 
+    "platforms": [
+      "linux"
+    ]
+  }, 
+  {
+    "args": [
       "test/core/end2end/fuzzers/server_fuzzer_corpus/052c8f28e5884bb48f0d504461272cd3a5893215"
     ], 
     "ci_platforms": [
@@ -57940,6 +57972,22 @@
   }, 
   {
     "args": [
+      "test/core/end2end/fuzzers/server_fuzzer_corpus/08c42ef29eff83052c5887855f2fa3e07ebe470c"
+    ], 
+    "ci_platforms": [
+      "linux"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "server_fuzzer_one_entry", 
+    "platforms": [
+      "linux"
+    ]
+  }, 
+  {
+    "args": [
       "test/core/end2end/fuzzers/server_fuzzer_corpus/09938e3256d06a8e168eb038d8a58b8462f7f697"
     ], 
     "ci_platforms": [
@@ -58388,6 +58436,22 @@
   }, 
   {
     "args": [
+      "test/core/end2end/fuzzers/server_fuzzer_corpus/1ba889ea1543297824e99e641e6ca8b91f45732e"
+    ], 
+    "ci_platforms": [
+      "linux"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "server_fuzzer_one_entry", 
+    "platforms": [
+      "linux"
+    ]
+  }, 
+  {
+    "args": [
       "test/core/end2end/fuzzers/server_fuzzer_corpus/1cf17783de9e662f3720847f2d83d86dcdcab500"
     ], 
     "ci_platforms": [
@@ -59172,6 +59236,22 @@
   }, 
   {
     "args": [
+      "test/core/end2end/fuzzers/server_fuzzer_corpus/3b09bf453c6f93983c24c4d5481e55d66213f93a"
+    ], 
+    "ci_platforms": [
+      "linux"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "server_fuzzer_one_entry", 
+    "platforms": [
+      "linux"
+    ]
+  }, 
+  {
+    "args": [
       "test/core/end2end/fuzzers/server_fuzzer_corpus/3ca5da2f.bin"
     ], 
     "ci_platforms": [
@@ -59524,6 +59604,22 @@
   }, 
   {
     "args": [
+      "test/core/end2end/fuzzers/server_fuzzer_corpus/49cb33cbb60f041e8e99dd718993acd2c3354416"
+    ], 
+    "ci_platforms": [
+      "linux"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "server_fuzzer_one_entry", 
+    "platforms": [
+      "linux"
+    ]
+  }, 
+  {
+    "args": [
       "test/core/end2end/fuzzers/server_fuzzer_corpus/4aa883d0.bin"
     ], 
     "ci_platforms": [
@@ -59972,6 +60068,22 @@
   }, 
   {
     "args": [
+      "test/core/end2end/fuzzers/server_fuzzer_corpus/59743fe120be6ae1aed1c02230ee1bb460f621ee"
+    ], 
+    "ci_platforms": [
+      "linux"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "server_fuzzer_one_entry", 
+    "platforms": [
+      "linux"
+    ]
+  }, 
+  {
+    "args": [
       "test/core/end2end/fuzzers/server_fuzzer_corpus/597fdab5.bin"
     ], 
     "ci_platforms": [
@@ -61364,6 +61476,22 @@
   }, 
   {
     "args": [
+      "test/core/end2end/fuzzers/server_fuzzer_corpus/a5ccb8f124d8ddb5350b90bc0d6b96db280cb7c9"
+    ], 
+    "ci_platforms": [
+      "linux"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "server_fuzzer_one_entry", 
+    "platforms": [
+      "linux"
+    ]
+  }, 
+  {
+    "args": [
       "test/core/end2end/fuzzers/server_fuzzer_corpus/a7e64803.bin"
     ], 
     "ci_platforms": [
@@ -61380,6 +61508,22 @@
   }, 
   {
     "args": [
+      "test/core/end2end/fuzzers/server_fuzzer_corpus/a7fac1265a384fe9e45a9ee3d708b79c4e80505e"
+    ], 
+    "ci_platforms": [
+      "linux"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "server_fuzzer_one_entry", 
+    "platforms": [
+      "linux"
+    ]
+  }, 
+  {
+    "args": [
       "test/core/end2end/fuzzers/server_fuzzer_corpus/a8d229374635fa6f2a75ca1669892e1bc244e719"
     ], 
     "ci_platforms": [
@@ -61524,6 +61668,22 @@
   }, 
   {
     "args": [
+      "test/core/end2end/fuzzers/server_fuzzer_corpus/aaf049720c707d4e14e47e7eb31d6a2dda60e66a"
+    ], 
+    "ci_platforms": [
+      "linux"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "server_fuzzer_one_entry", 
+    "platforms": [
+      "linux"
+    ]
+  }, 
+  {
+    "args": [
       "test/core/end2end/fuzzers/server_fuzzer_corpus/ad810f7f.bin"
     ], 
     "ci_platforms": [
@@ -61988,6 +62148,22 @@
   }, 
   {
     "args": [
+      "test/core/end2end/fuzzers/server_fuzzer_corpus/c4e4c7572e005e18d56eac407033da058737a5ab"
+    ], 
+    "ci_platforms": [
+      "linux"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "server_fuzzer_one_entry", 
+    "platforms": [
+      "linux"
+    ]
+  }, 
+  {
+    "args": [
       "test/core/end2end/fuzzers/server_fuzzer_corpus/c559f565.bin"
     ], 
     "ci_platforms": [
@@ -62292,6 +62468,22 @@
   }, 
   {
     "args": [
+      "test/core/end2end/fuzzers/server_fuzzer_corpus/crash-dae0f07934a527989f23f06e630710ff6ca8c809"
+    ], 
+    "ci_platforms": [
+      "linux"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "server_fuzzer_one_entry", 
+    "platforms": [
+      "linux"
+    ]
+  }, 
+  {
+    "args": [
       "test/core/end2end/fuzzers/server_fuzzer_corpus/crash-e34b0a9a428001cb4094a9ebca76329f578811a4"
     ], 
     "ci_platforms": [
@@ -62612,6 +62804,22 @@
   }, 
   {
     "args": [
+      "test/core/end2end/fuzzers/server_fuzzer_corpus/e96ad9c17795e52edc810a08d4fc61fe8790002a"
+    ], 
+    "ci_platforms": [
+      "linux"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "server_fuzzer_one_entry", 
+    "platforms": [
+      "linux"
+    ]
+  }, 
+  {
+    "args": [
       "test/core/end2end/fuzzers/server_fuzzer_corpus/e9bbe2fe47b7b9c2683e7f17f4a33625c6ffbd8c"
     ], 
     "ci_platforms": [
@@ -62932,6 +63140,22 @@
   }, 
   {
     "args": [
+      "test/core/end2end/fuzzers/server_fuzzer_corpus/fa202a5f51cd49f8ea5af60c5f403f797c01c504"
+    ], 
+    "ci_platforms": [
+      "linux"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "server_fuzzer_one_entry", 
+    "platforms": [
+      "linux"
+    ]
+  }, 
+  {
+    "args": [
       "test/core/end2end/fuzzers/server_fuzzer_corpus/fa36b4280d9e28edd81c5e4d192d1a5c2765e5e4"
     ], 
     "ci_platforms": [