Merge branch 'master' into cq_mpsc_based

26 files changed, 331 insertions, 80 deletions

diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal
index 88a9736d5b..9664234f9f 100644
--- a/tools/doxygen/Doxyfile.c++.internal
+++ b/tools/doxygen/Doxyfile.c++.internal
@@ -1090,7 +1090,7 @@ src/core/lib/surface/completion_queue_factory.h \
 src/core/lib/surface/event_string.c \
 src/core/lib/surface/event_string.h \
 src/core/lib/surface/init.h \
-src/core/lib/surface/lame_client.c \
+src/core/lib/surface/lame_client.cc \
 src/core/lib/surface/lame_client.h \
 src/core/lib/surface/metadata_array.c \
 src/core/lib/surface/server.c \
diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal
index 924595eb95..51d77c2b52 100644
--- a/tools/doxygen/Doxyfile.core.internal
+++ b/tools/doxygen/Doxyfile.core.internal
@@ -910,10 +910,14 @@ src/core/ext/filters/client_channel/http_proxy.c \
 src/core/ext/filters/client_channel/http_proxy.h \
 src/core/ext/filters/client_channel/lb_policy.c \
 src/core/ext/filters/client_channel/lb_policy.h \
+src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.c \
+src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.h \
 src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c \
 src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h \
 src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.h \
 src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel_secure.c \
+src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.c \
+src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h \
 src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.c \
 src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h \
 src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c \
@@ -1249,6 +1253,9 @@ src/core/lib/support/alloc.c \
 src/core/lib/support/arena.c \
 src/core/lib/support/arena.h \
 src/core/lib/support/atm.c \
+src/core/lib/support/atomic.h \
+src/core/lib/support/atomic_with_atm.h \
+src/core/lib/support/atomic_with_std.h \
 src/core/lib/support/avl.c \
 src/core/lib/support/backoff.c \
 src/core/lib/support/backoff.h \
@@ -1269,6 +1276,7 @@ src/core/lib/support/log_android.c \
 src/core/lib/support/log_linux.c \
 src/core/lib/support/log_posix.c \
 src/core/lib/support/log_windows.c \
+src/core/lib/support/memory.h \
 src/core/lib/support/mpscq.c \
 src/core/lib/support/mpscq.h \
 src/core/lib/support/murmur_hash.c \
@@ -1329,7 +1337,7 @@ src/core/lib/surface/event_string.h \
 src/core/lib/surface/init.c \
 src/core/lib/surface/init.h \
 src/core/lib/surface/init_secure.c \
-src/core/lib/surface/lame_client.c \
+src/core/lib/surface/lame_client.cc \
 src/core/lib/surface/lame_client.h \
 src/core/lib/surface/metadata_array.c \
 src/core/lib/surface/server.c \
@@ -1374,6 +1382,8 @@ src/core/tsi/ssl_transport_security.h \
 src/core/tsi/ssl_types.h \
 src/core/tsi/transport_security.c \
 src/core/tsi/transport_security.h \
+src/core/tsi/transport_security_adapter.c \
+src/core/tsi/transport_security_adapter.h \
 src/core/tsi/transport_security_interface.h \
 third_party/nanopb/pb.h \
 third_party/nanopb/pb_common.c \
diff --git a/tools/gce/create_linux_performance_worker.sh b/tools/gce/create_linux_performance_worker.sh
index 2c8cf0b96b..68710e13b0 100755
--- a/tools/gce/create_linux_performance_worker.sh
+++ b/tools/gce/create_linux_performance_worker.sh
@@ -42,14 +42,14 @@ CLOUD_PROJECT=grpc-testing
 ZONE=us-central1-b  # this zone allows 32core machines
 
 INSTANCE_NAME="${1:-grpc-performance-server1}"
-MACHINE_TYPE=n1-standard-8
+MACHINE_TYPE=n1-standard-32
 
 gcloud compute instances create $INSTANCE_NAME \
     --project="$CLOUD_PROJECT" \
     --zone "$ZONE" \
     --machine-type $MACHINE_TYPE \
     --image-project ubuntu-os-cloud \
-    --image-family ubuntu-1604-lts \
+    --image-family ubuntu-1610 \
     --boot-disk-size 300 \
     --scopes https://www.googleapis.com/auth/bigquery
 
diff --git a/tools/gce/linux_performance_worker_init.sh b/tools/gce/linux_performance_worker_init.sh
index 17f36fb4ff..78cdd31f0b 100755
--- a/tools/gce/linux_performance_worker_init.sh
+++ b/tools/gce/linux_performance_worker_init.sh
@@ -55,7 +55,10 @@ sudo apt-get install -y \
   libc6 \
   libc6-dbg \
   libc6-dev \
+  libcurl4-openssl-dev \
   libgtest-dev \
+  libreadline-dev \
+  libssl-dev \
   libtool \
   make \
   strace \
@@ -71,7 +74,8 @@ sudo apt-get install -y \
   telnet \
   unzip \
   wget \
-  zip
+  zip \
+  zlib1g-dev
 
 # perftools
 sudo apt-get install -y google-perftools libgoogle-perftools-dev
@@ -87,14 +91,15 @@ sudo pip install tabulate
 sudo pip install google-api-python-client
 sudo pip install virtualenv
 
-# TODO(jtattermusch): For some reason, building gRPC Python depends on python3.4
-# being installed, but python3.4 is not available on Ubuntu 16.04.
-# Temporarily fixing this by adding a PPA with python3.4, but we should
-# really remove this hack once possible.
-sudo add-apt-repository -y ppa:fkrull/deadsnakes
-sudo apt-get update
-sudo apt-get install -y python3.4 python3.4-dev
-python3.4 -m pip install virtualenv
+# Building gRPC Python depends on python3.4 being installed, but python3.4
+# is not available on Ubuntu 16.10, so install from source
+curl -O https://www.python.org/ftp/python/3.4.6/Python-3.4.6.tgz
+tar xzvf Python-3.4.6.tgz
+cd Python-3.4.6
+./configure --enable-shared --prefix=/usr/local LDFLAGS="-Wl,--rpath=/usr/local/lib"
+sudo make altinstall
+cd ..
+rm Python-3.4.6.tgz
 
 curl -O https://bootstrap.pypa.io/get-pip.py
 sudo pypy get-pip.py
@@ -117,18 +122,25 @@ sudo apt-get update
 sudo apt-get install -y mono-devel nuget
 
 # C# .NET Core dependencies (https://www.microsoft.com/net/core#ubuntu)
-sudo sh -c 'echo "deb [arch=amd64] https://apt-mo.trafficmanager.net/repos/dotnet-release/ xenial main" > /etc/apt/sources.list.d/dotnetdev.list'
+sudo sh -c 'echo "deb [arch=amd64] https://apt-mo.trafficmanager.net/repos/dotnet-release/ yakkety main" > /etc/apt/sources.list.d/dotnetdev.list'
 sudo apt-key adv --keyserver apt-mo.trafficmanager.net --recv-keys 417A0893
 sudo apt-get update
-sudo apt-get install -y dotnet-dev-1.0.0-preview2-003131
+sudo apt-get install -y dotnet-dev-1.0.0-preview2.1-003155
 sudo apt-get install -y dotnet-dev-1.0.1
 
 # Ruby dependencies
-gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3
-curl -sSL https://get.rvm.io | bash -s stable --ruby
+git clone https://github.com/rbenv/rbenv.git ~/.rbenv
+export PATH="$HOME/.rbenv/bin:$PATH"
+eval "$(rbenv init -)"
+
+git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build
+export PATH="$HOME/.rbenv/plugins/ruby-build/bin:$PATH"
+
+rbenv install 2.4.0
+rbenv global 2.4.0
+ruby -v
 
 # Install bundler (prerequisite for gRPC Ruby)
-source ~/.rvm/scripts/rvm
 gem install bundler
 
 # Java dependencies - nothing as we already have Java JDK 8
@@ -163,15 +175,7 @@ echo 4096 | sudo tee /proc/sys/kernel/perf_event_mlock_kb
 git clone -v https://github.com/brendangregg/FlameGraph ~/FlameGraph
 
 # Install scipy and numpy for benchmarking scripts
-sudo apt-get install python-scipy python-numpy
-
-# Update Linux kernel to 4.9
-wget \
-  kernel.ubuntu.com/~kernel-ppa/mainline/v4.9.20/linux-headers-4.9.20-040920_4.9.20-040920.201703310531_all.deb \
-  kernel.ubuntu.com/~kernel-ppa/mainline/v4.9.20/linux-headers-4.9.20-040920-generic_4.9.20-040920.201703310531_amd64.deb \
-  kernel.ubuntu.com/~kernel-ppa/mainline/v4.9.20/linux-image-4.9.20-040920-generic_4.9.20-040920.201703310531_amd64.deb
-sudo dpkg -i linux-headers-4.9*.deb linux-image-4.9*.deb
-rm linux-*
+sudo apt-get install -y python-scipy python-numpy
 
 # Add pubkey of jenkins@grpc-jenkins-master to authorized keys of jenkins@
 # This needs to happen as the last step to prevent Jenkins master from connecting
diff --git a/tools/internal_ci/linux/grpc_master.sh b/tools/internal_ci/linux/grpc_master.sh
index 9ecf123959..ec4bb2bda1 100755
--- a/tools/internal_ci/linux/grpc_master.sh
+++ b/tools/internal_ci/linux/grpc_master.sh
@@ -47,12 +47,4 @@ git submodule update --init
 
 # download docker images from dockerhub
 export DOCKERHUB_ORGANIZATION=grpctesting
-tools/run_tests/run_tests.py -l c -t -x sponge_log.xml || FAILED="true"
-
-# kill port_server.py to prevent the build from hanging
-ps aux | grep port_server\\.py | awk '{print $2}' | xargs kill -9
-
-if [ "$FAILED" != "" ]
-then
-  exit 1
-fi
+tools/run_tests/run_tests_matrix.py -f basictests linux --internal_ci
diff --git a/tools/internal_ci/linux/grpc_portability.sh b/tools/internal_ci/linux/grpc_portability.sh
index 58d3c58e70..613d5615f2 100755
--- a/tools/internal_ci/linux/grpc_portability.sh
+++ b/tools/internal_ci/linux/grpc_portability.sh
@@ -37,4 +37,4 @@ git submodule update --init
 
 # download docker images from dockerhub
 export DOCKERHUB_ORGANIZATION=grpctesting
-tools/run_tests/run_tests_matrix.py -f portability linux
+tools/run_tests/run_tests_matrix.py -f portability linux --internal_ci
diff --git a/tools/internal_ci/linux/grpc_portability_build_only.sh b/tools/internal_ci/linux/grpc_portability_build_only.sh
index 80b5c4cb96..787f0302c0 100755
--- a/tools/internal_ci/linux/grpc_portability_build_only.sh
+++ b/tools/internal_ci/linux/grpc_portability_build_only.sh
@@ -37,4 +37,4 @@ git submodule update --init
 
 # download docker images from dockerhub
 export DOCKERHUB_ORGANIZATION=grpctesting
-tools/run_tests/run_tests_matrix.py -f portability linux --build_only
+tools/run_tests/run_tests_matrix.py -f portability linux --internal_ci --build_only
diff --git a/tools/internal_ci/macos/grpc_master.cfg b/tools/internal_ci/macos/grpc_master.cfg
new file mode 100644
index 0000000000..039c99ec42
--- /dev/null
+++ b/tools/internal_ci/macos/grpc_master.cfg
@@ -0,0 +1,39 @@
+# Copyright 2017, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/macos/grpc_master.sh"
+timeout_mins: 240
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.xml"
+  }
+}
diff --git a/tools/internal_ci/macos/grpc_master.sh b/tools/internal_ci/macos/grpc_master.sh
new file mode 100755
index 0000000000..4ce1af73a5
--- /dev/null
+++ b/tools/internal_ci/macos/grpc_master.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+# Copyright 2017, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+set -ex
+
+# change to grpc repo root
+cd $(dirname $0)/../../..
+
+git submodule update --init
+
+tools/run_tests/run_tests_matrix.py -f basictests macos --internal_ci || FAILED="true"
+
+# kill port_server.py to prevent the build from hanging
+ps aux | grep port_server\\.py | awk '{print $2}' | xargs kill -9
+
+if [ "$FAILED" != "" ]
+then
+  exit 1
+fi
diff --git a/tools/internal_ci/windows/grpc_master.bat b/tools/internal_ci/windows/grpc_master.bat
index 8943390a8d..b6f3c8790f 100644
--- a/tools/internal_ci/windows/grpc_master.bat
+++ b/tools/internal_ci/windows/grpc_master.bat
@@ -36,7 +36,7 @@ cd /d %~dp0\..\..\..
 
 git submodule update --init
 
-python tools/run_tests/run_tests_matrix.py -f basictests windows -j 1 --inner_jobs 8 || goto :error
+python tools/run_tests/run_tests_matrix.py -f basictests windows -j 1 --inner_jobs 8 --internal_ci || goto :error
 goto :EOF
 
 :error
diff --git a/tools/internal_ci/windows/grpc_portability_master.bat b/tools/internal_ci/windows/grpc_portability_master.bat
index b98c70146c..789808664b 100644
--- a/tools/internal_ci/windows/grpc_portability_master.bat
+++ b/tools/internal_ci/windows/grpc_portability_master.bat
@@ -36,7 +36,7 @@ cd /d %~dp0\..\..\..
 
 git submodule update --init
 
-python tools/run_tests/run_tests_matrix.py -f portability windows -j 1 --inner_jobs 8 || goto :error
+python tools/run_tests/run_tests_matrix.py -f portability windows -j 1 --inner_jobs 8 --internal_ci || goto :error
 goto :EOF
 
 :error
diff --git a/tools/jenkins/run_bazel_basic_in_docker.sh b/tools/jenkins/run_bazel_basic_in_docker.sh
index b1d498a07d..5013f80b1d 100755
--- a/tools/jenkins/run_bazel_basic_in_docker.sh
+++ b/tools/jenkins/run_bazel_basic_in_docker.sh
@@ -39,4 +39,4 @@ git clone /var/local/jenkins/grpc /var/local/git/grpc
 && git submodule update --init --reference /var/local/jenkins/grpc/${name} \
 ${name}')
 cd /var/local/git/grpc
-bazel build --spawn_strategy=standalone --genrule_strategy=standalone :all test/... examples/cpp/...
+bazel build --spawn_strategy=standalone --genrule_strategy=standalone :all test/... examples/...
diff --git a/tools/jenkins/run_c_cpp_test.sh b/tools/jenkins/run_c_cpp_test.sh
index a7e574518e..afa2e780f7 100755
--- a/tools/jenkins/run_c_cpp_test.sh
+++ b/tools/jenkins/run_c_cpp_test.sh
@@ -35,10 +35,12 @@ set -ex
 # Enter the gRPC repo root
 cd $(dirname $0)/../..
 
-AFFECTS_C_CPP=`python -c 'import sys; \
+AFFECTS_C_CPP=`python -c 'import os; \
+               import sys; \
                sys.path.insert(0, "tools/run_tests/python_utils"); \
                import filter_pull_request_tests as filter; \
-               print(filter.affects_c_cpp("origin/$ghprbTargetBranch"))'`
+               github_target_branch = os.environ.get("ghprbTargetBranch"); \
+               print(filter.affects_c_cpp("origin/%s" % github_target_branch))'`
 
 if [ $AFFECTS_C_CPP == "False" ] ; then
   echo "This pull request does not affect C/C++. Tests do not need to be run."
diff --git a/tools/run_tests/artifacts/distribtest_targets.py b/tools/run_tests/artifacts/distribtest_targets.py
index 90bbde83cf..097fd2d8b5 100644
--- a/tools/run_tests/artifacts/distribtest_targets.py
+++ b/tools/run_tests/artifacts/distribtest_targets.py
@@ -38,11 +38,14 @@ import python_utils.jobset as jobset
 
 
 def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={},
-                   flake_retries=0, timeout_retries=0):
+                   flake_retries=0, timeout_retries=0,
+                   copy_rel_path=None):
   """Creates jobspec for a task running under docker."""
   environ = environ.copy()
   environ['RUN_COMMAND'] = shell_command
-  environ['RELATIVE_COPY_PATH'] = 'test/distrib'
+  # the entire repo will be cloned if copy_rel_path is not set.
+  if copy_rel_path:
+    environ['RELATIVE_COPY_PATH'] = copy_rel_path
 
   docker_args=[]
   for k,v in environ.items():
@@ -102,7 +105,8 @@ class CSharpDistribTest(object):
           'tools/dockerfile/distribtest/csharp_%s_%s' % (
               self.docker_suffix,
               self.arch),
-          'test/distrib/csharp/run_distrib_test%s.sh' % self.script_suffix)
+          'test/distrib/csharp/run_distrib_test%s.sh' % self.script_suffix,
+          copy_rel_path='test/distrib')
     elif self.platform == 'macos':
       return create_jobspec(self.name,
           ['test/distrib/csharp/run_distrib_test%s.sh' % self.script_suffix],
@@ -151,7 +155,8 @@ class NodeDistribTest(object):
                                        self.arch),
                                    '%s test/distrib/node/run_distrib_test.sh %s' % (
                                        linux32,
-                                       self.node_version))
+                                       self.node_version),
+                                   copy_rel_path='test/distrib')
     elif self.platform == 'macos':
       return create_jobspec(self.name,
                             ['test/distrib/node/run_distrib_test.sh',
@@ -185,7 +190,8 @@ class PythonDistribTest(object):
           'tools/dockerfile/distribtest/python_%s_%s' % (
               self.docker_suffix,
               self.arch),
-          'test/distrib/python/run_distrib_test.sh')
+          'test/distrib/python/run_distrib_test.sh',
+          copy_rel_path='test/distrib')
 
   def __str__(self):
     return self.name
@@ -212,7 +218,8 @@ class RubyDistribTest(object):
           'tools/dockerfile/distribtest/ruby_%s_%s' % (
               self.docker_suffix,
               self.arch),
-          'test/distrib/ruby/run_distrib_test.sh')
+          'test/distrib/ruby/run_distrib_test.sh',
+          copy_rel_path='test/distrib')
 
   def __str__(self):
     return self.name
@@ -237,7 +244,8 @@ class PHPDistribTest(object):
                                    'tools/dockerfile/distribtest/php_%s_%s' % (
                                        self.docker_suffix,
                                        self.arch),
-                                   'test/distrib/php/run_distrib_test.sh')
+                                   'test/distrib/php/run_distrib_test.sh',
+                                   copy_rel_path='test/distrib')
     elif self.platform == 'macos':
       return create_jobspec(self.name,
           ['test/distrib/php/run_distrib_test.sh'],
diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json
index 1f3bf7837f..c962823437 100644
--- a/tools/run_tests/generated/sources_and_headers.json
+++ b/tools/run_tests/generated/sources_and_headers.json
@@ -3291,6 +3291,29 @@
     ], 
     "is_filegroup": false, 
     "language": "c++", 
+    "name": "grpclb_end2end_test", 
+    "src": [
+      "test/cpp/end2end/grpclb_end2end_test.cc"
+    ], 
+    "third_party": false, 
+    "type": "target"
+  }, 
+  {
+    "deps": [
+      "gpr", 
+      "gpr_test_util", 
+      "grpc", 
+      "grpc++", 
+      "grpc++_test_util", 
+      "grpc_test_util"
+    ], 
+    "headers": [
+      "src/proto/grpc/lb/v1/load_balancer.grpc.pb.h", 
+      "src/proto/grpc/lb/v1/load_balancer.pb.h", 
+      "src/proto/grpc/lb/v1/load_balancer_mock.grpc.pb.h"
+    ], 
+    "is_filegroup": false, 
+    "language": "c++", 
     "name": "grpclb_test", 
     "src": [
       "test/cpp/grpclb/grpclb_test.cc"
@@ -3435,6 +3458,25 @@
   {
     "deps": [
       "gpr", 
+      "gpr_test_util", 
+      "grpc", 
+      "grpc++", 
+      "grpc++_test", 
+      "grpc_test_util"
+    ], 
+    "headers": [], 
+    "is_filegroup": false, 
+    "language": "c++", 
+    "name": "memory_test", 
+    "src": [
+      "test/core/support/memory_test.cc"
+    ], 
+    "third_party": false, 
+    "type": "target"
+  }, 
+  {
+    "deps": [
+      "gpr", 
       "grpc", 
       "grpc++", 
       "grpc++_test_config"
@@ -6007,8 +6049,7 @@
       "grpc++_codegen_base_src", 
       "grpc++_codegen_proto", 
       "grpc++_config_proto", 
-      "grpc_test_util", 
-      "thrift_util"
+      "grpc_test_util"
     ], 
     "headers": [
       "src/proto/grpc/health/v1/health.grpc.pb.h", 
@@ -7524,9 +7565,13 @@
       "include/grpc/support/useful.h", 
       "src/core/lib/profiling/timers.h", 
       "src/core/lib/support/arena.h", 
+      "src/core/lib/support/atomic.h", 
+      "src/core/lib/support/atomic_with_atm.h", 
+      "src/core/lib/support/atomic_with_std.h", 
       "src/core/lib/support/backoff.h", 
       "src/core/lib/support/block_annotate.h", 
       "src/core/lib/support/env.h", 
+      "src/core/lib/support/memory.h", 
       "src/core/lib/support/mpscq.h", 
       "src/core/lib/support/murmur_hash.h", 
       "src/core/lib/support/spinlock.h", 
@@ -7574,6 +7619,9 @@
       "src/core/lib/support/arena.c", 
       "src/core/lib/support/arena.h", 
       "src/core/lib/support/atm.c", 
+      "src/core/lib/support/atomic.h", 
+      "src/core/lib/support/atomic_with_atm.h", 
+      "src/core/lib/support/atomic_with_std.h", 
       "src/core/lib/support/avl.c", 
       "src/core/lib/support/backoff.c", 
       "src/core/lib/support/backoff.h", 
@@ -7594,6 +7642,7 @@
       "src/core/lib/support/log_linux.c", 
       "src/core/lib/support/log_posix.c", 
       "src/core/lib/support/log_windows.c", 
+      "src/core/lib/support/memory.h", 
       "src/core/lib/support/mpscq.c", 
       "src/core/lib/support/mpscq.h", 
       "src/core/lib/support/murmur_hash.c", 
@@ -7993,7 +8042,7 @@
       "src/core/lib/surface/event_string.c", 
       "src/core/lib/surface/event_string.h", 
       "src/core/lib/surface/init.h", 
-      "src/core/lib/surface/lame_client.c", 
+      "src/core/lib/surface/lame_client.cc", 
       "src/core/lib/surface/lame_client.h", 
       "src/core/lib/surface/metadata_array.c", 
       "src/core/lib/surface/server.c", 
@@ -8185,8 +8234,10 @@
       "nanopb"
     ], 
     "headers": [
+      "src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.h", 
       "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h", 
       "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.h", 
+      "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h", 
       "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h", 
       "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h"
     ], 
@@ -8194,10 +8245,14 @@
     "language": "c", 
     "name": "grpc_lb_policy_grpclb", 
     "src": [
+      "src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.c", 
+      "src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.h", 
       "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c", 
       "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h", 
       "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.c", 
       "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.h", 
+      "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.c", 
+      "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h", 
       "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.c", 
       "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h", 
       "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c", 
@@ -8215,8 +8270,10 @@
       "nanopb"
     ], 
     "headers": [
+      "src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.h", 
       "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h", 
       "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.h", 
+      "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h", 
       "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h", 
       "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h"
     ], 
@@ -8224,10 +8281,14 @@
     "language": "c", 
     "name": "grpc_lb_policy_grpclb_secure", 
     "src": [
+      "src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.c", 
+      "src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.h", 
       "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c", 
       "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h", 
       "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.h", 
       "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel_secure.c", 
+      "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.c", 
+      "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h", 
       "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.c", 
       "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h", 
       "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c", 
@@ -8779,6 +8840,7 @@
       "src/core/tsi/ssl_transport_security.h", 
       "src/core/tsi/ssl_types.h", 
       "src/core/tsi/transport_security.h", 
+      "src/core/tsi/transport_security_adapter.h", 
       "src/core/tsi/transport_security_interface.h"
     ], 
     "is_filegroup": true, 
@@ -8792,6 +8854,8 @@
       "src/core/tsi/ssl_types.h", 
       "src/core/tsi/transport_security.c", 
       "src/core/tsi/transport_security.h", 
+      "src/core/tsi/transport_security_adapter.c", 
+      "src/core/tsi/transport_security_adapter.h", 
       "src/core/tsi/transport_security_interface.h"
     ], 
     "third_party": false, 
@@ -9099,23 +9163,5 @@
     ], 
     "third_party": false, 
     "type": "filegroup"
-  }, 
-  {
-    "deps": [
-      "grpc++_codegen_base"
-    ], 
-    "headers": [
-      "include/grpc++/impl/codegen/thrift_serializer.h", 
-      "include/grpc++/impl/codegen/thrift_utils.h"
-    ], 
-    "is_filegroup": true, 
-    "language": "c++", 
-    "name": "thrift_util", 
-    "src": [
-      "include/grpc++/impl/codegen/thrift_serializer.h", 
-      "include/grpc++/impl/codegen/thrift_utils.h"
-    ], 
-    "third_party": false, 
-    "type": "filegroup"
   }
 ]
diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json
index 1ec6304de3..4b4c53b0ee 100644
--- a/tools/run_tests/generated/tests.json
+++ b/tools/run_tests/generated/tests.json
@@ -3407,6 +3407,28 @@
     "exclude_configs": [], 
     "exclude_iomgrs": [], 
     "flaky": false, 
+    "gtest": true, 
+    "language": "c++", 
+    "name": "grpclb_end2end_test", 
+    "platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ]
+  }, 
+  {
+    "args": [], 
+    "ci_platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ], 
+    "cpu_cost": 1.0, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
     "gtest": false, 
     "language": "c++", 
     "name": "grpclb_test", 
@@ -3495,6 +3517,28 @@
     "flaky": false, 
     "gtest": true, 
     "language": "c++", 
+    "name": "memory_test", 
+    "platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ]
+  }, 
+  {
+    "args": [], 
+    "ci_platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ], 
+    "cpu_cost": 1.0, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "gtest": true, 
+    "language": "c++", 
     "name": "mock_test", 
     "platforms": [
       "linux", 
diff --git a/tools/run_tests/helper_scripts/pre_build_cmake.bat b/tools/run_tests/helper_scripts/pre_build_cmake.bat
index c937b9e09f..c721e1624d 100644
--- a/tools/run_tests/helper_scripts/pre_build_cmake.bat
+++ b/tools/run_tests/helper_scripts/pre_build_cmake.bat
@@ -37,7 +37,10 @@ mkdir build
 cd build
 
 @rem TODO(jtattermusch): Stop hardcoding path to yasm once Jenkins workers can locate yasm correctly
-cmake -G "Visual Studio 14 2015" -DgRPC_BUILD_TESTS=ON -DCMAKE_ASM_NASM_COMPILER="C:/Program Files (x86)/yasm/yasm.exe" ../.. || goto :error
+@rem If yasm is not on the path, use hardcoded path instead.
+yasm --version || set USE_HARDCODED_YASM_PATH_MAYBE=-DCMAKE_ASM_NASM_COMPILER="C:/Program Files (x86)/yasm/yasm.exe"
+
+cmake -G "Visual Studio 14 2015" -DgRPC_BUILD_TESTS=ON %USE_HARDCODED_YASM_PATH_MAYBE% ../.. || goto :error
 
 endlocal
 
diff --git a/tools/run_tests/helper_scripts/pre_build_csharp.bat b/tools/run_tests/helper_scripts/pre_build_csharp.bat
index e59dac4edc..47ebd8bee3 100644
--- a/tools/run_tests/helper_scripts/pre_build_csharp.bat
+++ b/tools/run_tests/helper_scripts/pre_build_csharp.bat
@@ -42,8 +42,12 @@ mkdir build
 cd build
 mkdir %ARCHITECTURE%
 cd %ARCHITECTURE%
+
 @rem TODO(jtattermusch): Stop hardcoding path to yasm once Jenkins workers can locate yasm correctly
-cmake -G "Visual Studio 14 2015" -A %ARCHITECTURE% -DgRPC_BUILD_TESTS=OFF -DgRPC_MSVC_STATIC_RUNTIME=ON -DCMAKE_ASM_NASM_COMPILER="C:/Program Files (x86)/yasm/yasm.exe" ../../.. || goto :error
+@rem If yasm is not on the path, use hardcoded path instead.
+yasm --version || set USE_HARDCODED_YASM_PATH_MAYBE=-DCMAKE_ASM_NASM_COMPILER="C:/Program Files (x86)/yasm/yasm.exe"
+
+cmake -G "Visual Studio 14 2015" -A %ARCHITECTURE% -DgRPC_BUILD_TESTS=OFF -DgRPC_MSVC_STATIC_RUNTIME=ON %USE_HARDCODED_YASM_PATH_MAYBE% ../../.. || goto :error
 
 cd ..\..\..\src\csharp
 
diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py
index ce0808829f..08411bf9a4 100644
--- a/tools/run_tests/performance/scenario_config.py
+++ b/tools/run_tests/performance/scenario_config.py
@@ -410,6 +410,21 @@ class CSharpLanguage:
         categories=[SMOKETEST, SCALABLE])
 
     yield _ping_pong_scenario(
+        'csharp_generic_async_streaming_ping_pong_insecure_1MB', rpc_type='STREAMING',
+        client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
+        req_size=1024*1024, resp_size=1024*1024,
+        use_generic_payload=True,
+        secure=False,
+        categories=[SMOKETEST, SCALABLE])
+
+    yield _ping_pong_scenario(
+        'csharp_generic_async_streaming_qps_unconstrained_insecure', rpc_type='STREAMING',
+        client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
+        unconstrained_client='async', use_generic_payload=True,
+        secure=False,
+        categories=[SMOKETEST, SCALABLE])
+
+    yield _ping_pong_scenario(
         'csharp_protobuf_async_streaming_ping_pong', rpc_type='STREAMING',
         client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER')
 
@@ -469,7 +484,6 @@ class CSharpLanguage:
         req_size=1024*1024, resp_size=1024*1024,
         categories=[SMOKETEST, SCALABLE])
 
-
   def __str__(self):
     return 'csharp'
 
diff --git a/tools/run_tests/python_utils/jobset.py b/tools/run_tests/python_utils/jobset.py
index 460f359cf3..d01e82a76b 100755
--- a/tools/run_tests/python_utils/jobset.py
+++ b/tools/run_tests/python_utils/jobset.py
@@ -208,6 +208,11 @@ class JobSpec(object):
   def __repr__(self):
     return 'JobSpec(shortname=%s, cmdline=%s)' % (self.shortname, self.cmdline)
 
+  def __str__(self):
+    return '%s: %s %s' % (self.shortname,
+                          ' '.join('%s=%s' % kv for kv in self.environ.items()),
+                          ' '.join(self.cmdline))
+
 
 class JobResult(object):
   def __init__(self):
diff --git a/tools/run_tests/python_utils/report_utils.py b/tools/run_tests/python_utils/report_utils.py
index 3b2b4f8712..c7c0ceea92 100644
--- a/tools/run_tests/python_utils/report_utils.py
+++ b/tools/run_tests/python_utils/report_utils.py
@@ -77,6 +77,10 @@ def render_junit_xml_report(resultset, xml_report, suite_package='grpc',
         ET.SubElement(xml_test, 'error', message='Timeout')
       elif result.state == 'SKIPPED':
         ET.SubElement(xml_test, 'skipped', message='Skipped')
+  # ensure the report directory exists
+  report_dir = os.path.dirname(os.path.abspath(xml_report))
+  if not os.path.exists(report_dir):
+    os.makedirs(report_dir)
   tree = ET.ElementTree(root)
   tree.write(xml_report, encoding='UTF-8')
 
diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py
index 44e93eb9cc..867d9e6f7b 100755
--- a/tools/run_tests/run_interop_tests.py
+++ b/tools/run_tests/run_interop_tests.py
@@ -888,6 +888,10 @@ argp.add_argument('-t', '--travis',
                   default=False,
                   action='store_const',
                   const=True)
+argp.add_argument('-v', '--verbose',
+                  default=False,
+                  action='store_const',
+                  const=True)
 argp.add_argument('--use_docker',
                   default=False,
                   action='store_const',
@@ -989,6 +993,9 @@ if args.use_docker:
 
   if build_jobs:
     jobset.message('START', 'Building interop docker images.', do_newline=True)
+    if args.verbose:
+      print('Jobs to run: \n%s\n' % '\n'.join(str(j) for j in build_jobs))
+
     num_failures, _ = jobset.run(
         build_jobs, newline_on_success=True, maxjobs=args.jobs)
     if num_failures == 0:
@@ -1164,6 +1171,9 @@ try:
   if args.manual_run:
     print('All tests will skipped --manual_run option is active.')
 
+  if args.verbose:
+    print('Jobs to run: \n%s\n' % '\n'.join(str(job) for job in jobs))
+
   num_failures, resultset = jobset.run(jobs, newline_on_success=True,
                                        maxjobs=args.jobs,
                                        skip_jobs=args.manual_run)
diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py
index a1ec1b2f45..4da2ba4c3b 100755
--- a/tools/run_tests/run_tests.py
+++ b/tools/run_tests/run_tests.py
@@ -763,7 +763,7 @@ class CSharpLanguage(object):
         self._make_options = ['EMBED_OPENSSL=true']
         if self.args.compiler != 'coreclr':
           # On Mac, official distribution of mono is 32bit.
-          self._make_options += ['CFLAGS=-m32', 'LDFLAGS=-m32']
+          self._make_options += ['ARCH_FLAGS=-m32', 'LDFLAGS=-m32']
       else:
         self._make_options = ['EMBED_OPENSSL=true', 'EMBED_ZLIB=true']
 
@@ -1353,7 +1353,8 @@ def make_jobspec(cfg, targets, makefile='Makefile'):
                               '-f', makefile,
                               '-j', '%d' % args.jobs,
                               'EXTRA_DEFINES=GRPC_TEST_SLOWDOWN_MACHINE_FACTOR=%f' % args.slowdown,
-                              'CONFIG=%s' % cfg] +
+                              'CONFIG=%s' % cfg,
+                              'Q='] +
                               language_make_options +
                              ([] if not args.travis else ['JENKINS_BUILD=1']) +
                              targets,
diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py
index 02f0ec5eff..70340cd747 100755
--- a/tools/run_tests/run_tests_matrix.py
+++ b/tools/run_tests/run_tests_matrix.py
@@ -55,6 +55,16 @@ _DEFAULT_INNER_JOBS = 2
 _REPORT_SUFFIX = 'sponge_log.xml'
 
 
+def _report_filename(name):
+  """Generates report file name"""
+  return 'report_%s_%s' % (name, _REPORT_SUFFIX)
+
+
+def _report_filename_internal_ci(name):
+  """Generates report file name that leads to better presentation by internal CI"""
+  return '%s/%s' % (name, _REPORT_SUFFIX)
+
+
 def _docker_jobspec(name, runtests_args=[], runtests_envs={},
                     inner_jobs=_DEFAULT_INNER_JOBS):
   """Run a single instance of run_tests.py in a docker container"""
@@ -63,7 +73,7 @@ def _docker_jobspec(name, runtests_args=[], runtests_envs={},
                    '--use_docker',
                    '-t',
                    '-j', str(inner_jobs),
-                   '-x', 'report_%s_%s' % (name, _REPORT_SUFFIX),
+                   '-x', _report_filename(name),
                    '--report_suite_name', '%s' % name] + runtests_args,
           environ=runtests_envs,
           shortname='run_tests_%s' % name,
@@ -83,7 +93,7 @@ def _workspace_jobspec(name, runtests_args=[], workspace_name=None,
                    'tools/run_tests/helper_scripts/run_tests_in_workspace.sh',
                    '-t',
                    '-j', str(inner_jobs),
-                   '-x', '../report_%s_%s' % (name, _REPORT_SUFFIX),
+                   '-x', '../%s' % _report_filename(name),
                    '--report_suite_name', '%s' % name] + runtests_args,
           environ=env,
           shortname='run_tests_%s' % name,
@@ -380,8 +390,17 @@ if __name__ == "__main__":
   argp.add_argument('--max_time', default=-1, type=int,
                     help='Maximum amount of time to run tests for' +
                          '(other tests will be skipped)')
+  argp.add_argument('--internal_ci',
+                    default=False,
+                    action='store_const',
+                    const=True,
+                    help='Put reports into subdirectories to improve presentation of '
+                    'results by Internal CI.')
   args = argp.parse_args()
 
+  if args.internal_ci:
+    _report_filename = _report_filename_internal_ci  # override the function
+
   extra_args = []
   if args.build_only:
     extra_args.append('--build_only')
@@ -450,7 +469,7 @@ if __name__ == "__main__":
     ignored_num_skipped_failures, skipped_results = jobset.run(
         skipped_jobs, skip_jobs=True)
     resultset.update(skipped_results)
-  report_utils.render_junit_xml_report(resultset, 'report_%s' % _REPORT_SUFFIX,
+  report_utils.render_junit_xml_report(resultset, _report_filename('aggregate_tests'),
                                        suite_name='aggregate_tests')
 
   if num_failures == 0:
diff --git a/tools/run_tests/sanity/check_submodules.sh b/tools/run_tests/sanity/check_submodules.sh
index 0a9c1cc046..6be7a39d07 100755
--- a/tools/run_tests/sanity/check_submodules.sh
+++ b/tools/run_tests/sanity/check_submodules.sh
@@ -47,7 +47,6 @@ cat << EOF | awk '{ print $1 }' | sort > $want_submodules
  30dbc81fb5ffdc98ea9b14b1918bfe4e8779b26e third_party/gflags (v2.2.0)
  ec44c6c1675c25b9827aacd08c02433cccde7780 third_party/googletest (release-1.8.0)
  593e917c176b5bc5aafa57bf9f6030d749d91cd5 third_party/protobuf (v3.1.0-alpha-1-326-g593e917)
- bcad91771b7f0bff28a1cac1981d7ef2b9bcef3c third_party/thrift (bcad917)
  cacf7f1d4e3d44d871b605da3b647f07d718623f third_party/zlib (v1.2.11)
  7691f773af79bf75a62d1863fd0f13ebf9dc51b1 third_party/cares/cares (1.12.0)
 EOF
diff --git a/tools/run_tests/sanity/core_banned_functions.py b/tools/run_tests/sanity/core_banned_functions.py
index c3c3cbec76..2387c5f1da 100755
--- a/tools/run_tests/sanity/core_banned_functions.py
+++ b/tools/run_tests/sanity/core_banned_functions.py
@@ -50,6 +50,7 @@ BANNED_EXCEPT = {
     'grpc_os_error(': ['src/core/lib/iomgr/error.c'],
     'grpc_wsa_error(': ['src/core/lib/iomgr/error.c'],
     'grpc_log_if_error(': ['src/core/lib/iomgr/error.c'],
+    'grpc_slice_malloc(': ['src/core/lib/slice/slice.c'],
 }
 
 errors = 0