Merge branch 'master' into gpr_review3

20 files changed, 153 insertions, 34 deletions

diff --git a/tools/codegen/core/gen_static_metadata.py b/tools/codegen/core/gen_static_metadata.py
index d4c0052b28..066f28416d 100755
--- a/tools/codegen/core/gen_static_metadata.py
+++ b/tools/codegen/core/gen_static_metadata.py
@@ -54,8 +54,8 @@ CONFIG = [
     # well known method names
     '/grpc.lb.v1.LoadBalancer/BalanceLoad',
     # compression algorithm names
-    'message/deflate',
-    'message/gzip',
+    'deflate',
+    'gzip',
     'stream/gzip',
     # metadata elements
     ('grpc-status', '0'),
diff --git a/tools/distrib/clang_format_code.sh b/tools/distrib/clang_format_code.sh
index 9233a230c9..cd7553e917 100755
--- a/tools/distrib/clang_format_code.sh
+++ b/tools/distrib/clang_format_code.sh
@@ -25,7 +25,9 @@ then
   docker build -t grpc_clang_format tools/dockerfile/grpc_clang_format
 
   # run clang-format against the checked out codebase
-  docker run -e TEST=$TEST -e CHANGED_FILES="$CHANGED_FILES" -e CLANG_FORMAT_ROOT="/local-code" --rm=true -v "${REPO_ROOT}":/local-code -t grpc_clang_format /clang_format_all_the_things.sh
+  # when modifying the checked-out files, the current user will be impersonated
+  # so that the updated files don't end up being owned by "root".
+  docker run -e TEST="$TEST" -e CHANGED_FILES="$CHANGED_FILES" -e CLANG_FORMAT_ROOT="/local-code" --rm=true -v "${REPO_ROOT}":/local-code --user "$(id -u):$(id -g)" -t grpc_clang_format /clang_format_all_the_things.sh
 else
   CLANG_FORMAT_ROOT="${REPO_ROOT}" tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh
 fi
diff --git a/tools/distrib/clang_tidy_code.sh b/tools/distrib/clang_tidy_code.sh
index 5da86aa277..9262b6bd3e 100755
--- a/tools/distrib/clang_tidy_code.sh
+++ b/tools/distrib/clang_tidy_code.sh
@@ -27,7 +27,9 @@ then
   docker build -t grpc_clang_tidy tools/dockerfile/grpc_clang_tidy
 
   # run clang-tidy against the checked out codebase
-  docker run -e TEST=$TEST -e CHANGED_FILES="$CHANGED_FILES" -e CLANG_TIDY_ROOT="/local-code" --rm=true -v "${REPO_ROOT}":/local-code -t grpc_clang_tidy /clang_tidy_all_the_things.sh "$@"
+  # when modifying the checked-out files, the current user will be impersonated
+  # so that the updated files don't end up being owned by "root".
+  docker run -e TEST="$TEST" -e CHANGED_FILES="$CHANGED_FILES" -e CLANG_TIDY_ROOT="/local-code" --rm=true -v "${REPO_ROOT}":/local-code --user "$(id -u):$(id -g)" -t grpc_clang_tidy /clang_tidy_all_the_things.sh "$@"
 else
   CLANG_TIDY_ROOT="${REPO_ROOT}" tools/dockerfile/grpc_clang_tidy/clang_tidy_all_the_things.sh "$@"
 fi
diff --git a/tools/dockerfile/grpc_clang_format/Dockerfile b/tools/dockerfile/grpc_clang_format/Dockerfile
index 8801315bc9..ecdc52ee31 100644
--- a/tools/dockerfile/grpc_clang_format/Dockerfile
+++ b/tools/dockerfile/grpc_clang_format/Dockerfile
@@ -23,4 +23,9 @@ RUN ln -s /clang+llvm-5.0.0-linux-x86_64-ubuntu14.04/bin/clang-tidy /usr/local/b
 ENV CLANG_TIDY=clang-tidy
 
 ADD clang_format_all_the_things.sh /
+
+# When running locally, we'll be impersonating the current user, so we need
+# to make the script runnable by everyone.
+RUN chmod a+rx /clang_format_all_the_things.sh
+
 CMD ["echo 'Run with tools/distrib/clang_format_code.sh'"]
diff --git a/tools/dockerfile/grpc_clang_tidy/Dockerfile b/tools/dockerfile/grpc_clang_tidy/Dockerfile
index 9d9d70185b..eeba455341 100644
--- a/tools/dockerfile/grpc_clang_tidy/Dockerfile
+++ b/tools/dockerfile/grpc_clang_tidy/Dockerfile
@@ -38,4 +38,9 @@ RUN pip install virtualenv
 RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.0.post1 six==1.10.0 twisted==17.5.0
 
 ADD clang_tidy_all_the_things.sh /
+
+# When running locally, we'll be impersonating the current user, so we need
+# to make the script runnable by everyone.
+RUN chmod a+rx /clang_tidy_all_the_things.sh
+
 CMD ["echo 'Run with tools/distrib/clang_tidy_code.sh'"]
diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++
index 317dd0e1eb..68c7c9205e 100644
--- a/tools/doxygen/Doxyfile.c++
+++ b/tools/doxygen/Doxyfile.c++
@@ -875,7 +875,6 @@ include/grpc++/support/time.h \
 include/grpc/byte_buffer.h \
 include/grpc/byte_buffer_reader.h \
 include/grpc/compression.h \
-include/grpc/compression_ruby.h \
 include/grpc/fork.h \
 include/grpc/grpc.h \
 include/grpc/grpc_posix.h \
diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal
index 5036068839..bbd28a731b 100644
--- a/tools/doxygen/Doxyfile.c++.internal
+++ b/tools/doxygen/Doxyfile.c++.internal
@@ -876,7 +876,6 @@ include/grpc++/support/time.h \
 include/grpc/byte_buffer.h \
 include/grpc/byte_buffer_reader.h \
 include/grpc/compression.h \
-include/grpc/compression_ruby.h \
 include/grpc/fork.h \
 include/grpc/grpc.h \
 include/grpc/grpc_posix.h \
diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core
index d61a422626..762836fedd 100644
--- a/tools/doxygen/Doxyfile.core
+++ b/tools/doxygen/Doxyfile.core
@@ -801,7 +801,6 @@ include/grpc/byte_buffer.h \
 include/grpc/byte_buffer_reader.h \
 include/grpc/census.h \
 include/grpc/compression.h \
-include/grpc/compression_ruby.h \
 include/grpc/fork.h \
 include/grpc/grpc.h \
 include/grpc/grpc_posix.h \
diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal
index 256e96edbf..1b780b3a64 100644
--- a/tools/doxygen/Doxyfile.core.internal
+++ b/tools/doxygen/Doxyfile.core.internal
@@ -801,7 +801,6 @@ include/grpc/byte_buffer.h \
 include/grpc/byte_buffer_reader.h \
 include/grpc/census.h \
 include/grpc/compression.h \
-include/grpc/compression_ruby.h \
 include/grpc/fork.h \
 include/grpc/grpc.h \
 include/grpc/grpc_posix.h \
@@ -1051,7 +1050,6 @@ src/core/lib/compression/algorithm_metadata.h \
 src/core/lib/compression/compression.cc \
 src/core/lib/compression/compression_internal.cc \
 src/core/lib/compression/compression_internal.h \
-src/core/lib/compression/compression_ruby.cc \
 src/core/lib/compression/message_compress.cc \
 src/core/lib/compression/message_compress.h \
 src/core/lib/compression/stream_compression.cc \
diff --git a/tools/internal_ci/helper_scripts/prepare_build_linux_rc b/tools/internal_ci/helper_scripts/prepare_build_linux_rc
index 8715d6c8e0..74bbc85d72 100644
--- a/tools/internal_ci/helper_scripts/prepare_build_linux_rc
+++ b/tools/internal_ci/helper_scripts/prepare_build_linux_rc
@@ -27,11 +27,6 @@ echo 'DOCKER_OPTS="${DOCKER_OPTS} --graph=/tmpfs/docker"' | sudo tee --append /e
 echo 'DOCKER_OPTS="${DOCKER_OPTS} --registry-mirror=https://mirror.gcr.io"' | sudo tee --append /etc/default/docker
 sudo service docker restart
 
-# Populate xdg-cache-home to workaround https://github.com/grpc/grpc/issues/11968
-sudo mkdir -p /tmp/xdg-cache-home
-PYTHONWARNINGS=ignore XDG_CACHE_HOME=/tmp/xdg-cache-home sudo -E pip install setuptools wheel
-PYTHONWARNINGS=ignore XDG_CACHE_HOME=/tmp/xdg-cache-home sudo -E pip install coverage==4.4 pylint==1.6.5
-
 # Download Docker images from DockerHub
 export DOCKERHUB_ORGANIZATION=grpctesting
 
diff --git a/tools/internal_ci/linux/grpc_build_submodule_at_head.sh b/tools/internal_ci/linux/grpc_build_submodule_at_head.sh
index e203a62b08..e68a2897f6 100755
--- a/tools/internal_ci/linux/grpc_build_submodule_at_head.sh
+++ b/tools/internal_ci/linux/grpc_build_submodule_at_head.sh
@@ -23,9 +23,8 @@ cd $(dirname $0)/../../..
 source tools/internal_ci/helper_scripts/prepare_build_linux_rc
 
 # Update submodule and commit it so changes are passed to Docker
-(cd third_party/$RUN_TESTS_FLAGS && git pull origin master)
+(cd third_party/$RUN_TESTS_FLAGS && git fetch --all && git checkout origin/master)
 tools/buildgen/generate_projects.sh
 git -c user.name='foo' -c user.email='foo@google.com' commit -a -m 'Update submodule'
 
 tools/run_tests/run_tests_matrix.py -f linux --inner_jobs 4 -j 4 --internal_ci --build_only
-
diff --git a/tools/internal_ci/linux/grpc_full_performance_master.sh b/tools/internal_ci/linux/grpc_full_performance_master.sh
index 18468395f6..4eddc18731 100755
--- a/tools/internal_ci/linux/grpc_full_performance_master.sh
+++ b/tools/internal_ci/linux/grpc_full_performance_master.sh
@@ -26,7 +26,7 @@ tools/run_tests/run_performance_tests.py \
     --category scalable \
     --remote_worker_host grpc-kokoro-performance-server-8core grpc-kokoro-performance-client-8core grpc-kokoro-performance-client2-8core \
     -u kbuilder \
-    --bq_result_table performance_test.kokoro_performance_experiment \
+    --bq_result_table performance_test.performance_experiment \
     --xml_report reports/8core/sponge_log.xml \
     || EXIT_CODE=1
 
@@ -40,7 +40,7 @@ tools/run_tests/run_performance_tests.py \
     --category scalable \
     --remote_worker_host grpc-kokoro-performance-server-32core grpc-kokoro-performance-client-32core grpc-kokoro-performance-client2-32core \
     -u kbuilder \
-    --bq_result_table performance_test.kokoro_performance_experiment_32core \
+    --bq_result_table performance_test.performance_experiment_32core \
     --xml_report reports/32core/sponge_log.xml \
     || EXIT_CODE=1
 
@@ -52,7 +52,7 @@ tools/run_tests/run_performance_tests.py \
     -l csharp \
     --category scalable \
     --remote_worker_host grpc-kokoro-performance-windows1 grpc-kokoro-performance-windows2 \
-    --bq_result_table performance_test.kokoro_performance_experiment_windows \
+    --bq_result_table performance_test.performance_experiment_windows \
     --xml_report reports/windows/sponge_log.xml \
     || EXIT_CODE=1
 
diff --git a/tools/internal_ci/linux/grpc_full_performance_release.cfg b/tools/internal_ci/linux/grpc_full_performance_release.cfg
new file mode 100644
index 0000000000..e9a4bcdcaf
--- /dev/null
+++ b/tools/internal_ci/linux/grpc_full_performance_release.cfg
@@ -0,0 +1,25 @@
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/linux/grpc_full_performance_release.sh"
+timeout_mins: 600
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.xml"
+    regex: "**/perf_reports/**"
+  }
+}
diff --git a/tools/internal_ci/linux/grpc_full_performance_release.sh b/tools/internal_ci/linux/grpc_full_performance_release.sh
new file mode 100755
index 0000000000..a31cea281f
--- /dev/null
+++ b/tools/internal_ci/linux/grpc_full_performance_release.sh
@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -ex
+
+# Enter the gRPC repo root
+cd $(dirname $0)/../../..
+
+source tools/internal_ci/helper_scripts/prepare_build_linux_perf_multilang_rc
+
+# run 8core client vs 8core server
+tools/run_tests/run_performance_tests.py \
+    -l c++ csharp ruby java python go php7 php7_protobuf_c \
+    --netperf \
+    --category scalable \
+    --remote_worker_host grpc-kokoro-performance-server-8core grpc-kokoro-performance-client-8core grpc-kokoro-performance-client2-8core \
+    -u kbuilder \
+    --bq_result_table performance_released.performance_experiment \
+    --xml_report reports/8core/sponge_log.xml \
+    || EXIT_CODE=1
+
+# prevent pushing leftover build files to remote hosts in the next step.
+git clean -fdxq -e reports
+
+# scalability with 32cores (and upload to a different BQ table)
+tools/run_tests/run_performance_tests.py \
+    -l c++ java csharp go \
+    --netperf \
+    --category scalable \
+    --remote_worker_host grpc-kokoro-performance-server-32core grpc-kokoro-performance-client-32core grpc-kokoro-performance-client2-32core \
+    -u kbuilder \
+    --bq_result_table performance_released.performance_experiment_32core \
+    --xml_report reports/32core/sponge_log.xml \
+    || EXIT_CODE=1
+
+# prevent pushing leftover build files to remote hosts in the next step.
+git clean -fdxq -e reports
+
+# selected scenarios on Windows
+tools/run_tests/run_performance_tests.py \
+    -l csharp \
+    --category scalable \
+    --remote_worker_host grpc-kokoro-performance-windows1 grpc-kokoro-performance-windows2 \
+    --bq_result_table performance_released.performance_experiment_windows \
+    --xml_report reports/windows/sponge_log.xml \
+    || EXIT_CODE=1
+
+exit $EXIT_CODE
diff --git a/tools/run_tests/artifacts/build_artifact_python.bat b/tools/run_tests/artifacts/build_artifact_python.bat
index 3a0564129c..d277668c94 100644
--- a/tools/run_tests/artifacts/build_artifact_python.bat
+++ b/tools/run_tests/artifacts/build_artifact_python.bat
@@ -16,7 +16,8 @@
 set PATH=C:\%1;C:\%1\scripts;C:\msys64\mingw%2\bin;C:\tools\msys64\mingw%2\bin;%PATH%
 
 pip install --upgrade six
-pip install --upgrade setuptools
+@rem some artifacts are broken for setuptools 38.5.0. See https://github.com/grpc/grpc/issues/14317
+pip install --upgrade setuptools==38.2.4
 pip install -rrequirements.txt
 
 set GRPC_PYTHON_BUILD_WITH_CYTHON=1
diff --git a/tools/run_tests/dockerize/build_docker_and_run_tests.sh b/tools/run_tests/dockerize/build_docker_and_run_tests.sh
index 8dca05ea35..21eccbaac9 100755
--- a/tools/run_tests/dockerize/build_docker_and_run_tests.sh
+++ b/tools/run_tests/dockerize/build_docker_and_run_tests.sh
@@ -25,10 +25,6 @@ cd -
 # Ensure existence of ccache directory
 mkdir -p /tmp/ccache
 
-# Ensure existence of the home directory for XDG caches (e.g. what pip uses for
-# its cache location now that --download-cache is deprecated).
-mkdir -p /tmp/xdg-cache-home
-
 # Inputs
 # DOCKERFILE_DIR - Directory in which Dockerfile file is located.
 # DOCKER_RUN_SCRIPT - Script to run under docker (relative to grpc repo root)
@@ -61,7 +57,6 @@ docker run \
   -e "config=$config" \
   -e "arch=$arch" \
   -e CCACHE_DIR=/tmp/ccache \
-  -e XDG_CACHE_HOME=/tmp/xdg-cache-home \
   -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \
   -e HOST_GIT_ROOT="$git_root" \
   -e LOCAL_GIT_ROOT=$docker_instance_git_root \
@@ -79,7 +74,6 @@ docker run \
   -v "$git_root:$docker_instance_git_root" \
   -v /tmp/ccache:/tmp/ccache \
   -v /tmp/npm-cache:/tmp/npm-cache \
-  -v /tmp/xdg-cache-home:/tmp/xdg-cache-home \
   -w /var/local/git/grpc \
   --name="$CONTAINER_NAME" \
   "$DOCKER_IMAGE_NAME" \
diff --git a/tools/run_tests/dockerize/docker_run_tests.sh b/tools/run_tests/dockerize/docker_run_tests.sh
index df9d3d1ca2..89ee315fd2 100755
--- a/tools/run_tests/dockerize/docker_run_tests.sh
+++ b/tools/run_tests/dockerize/docker_run_tests.sh
@@ -22,10 +22,6 @@ export CONFIG=$config
 export ASAN_SYMBOLIZER_PATH=/usr/bin/llvm-symbolizer
 export PATH=$PATH:/usr/bin/llvm-symbolizer
 
-# Ensure that programs depending on current-user-ownership of cache directories
-# are satisfied (it's being mounted from outside the image).
-chown "$(whoami)" "$XDG_CACHE_HOME"
-
 mkdir -p /var/local/git
 git clone  /var/local/jenkins/grpc /var/local/git/grpc
 # clone gRPC submodules, use data from locally cloned submodules where possible
diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json
index aa90753fbd..48252f24ff 100644
--- a/tools/run_tests/generated/sources_and_headers.json
+++ b/tools/run_tests/generated/sources_and_headers.json
@@ -5133,6 +5133,24 @@
     "headers": [], 
     "is_filegroup": false, 
     "language": "c", 
+    "name": "duplicate_header_bad_client_test", 
+    "src": [
+      "test/core/bad_client/tests/duplicate_header.cc"
+    ], 
+    "third_party": false, 
+    "type": "target"
+  }, 
+  {
+    "deps": [
+      "bad_client_test", 
+      "gpr", 
+      "gpr_test_util", 
+      "grpc_test_util_unsecure", 
+      "grpc_unsecure"
+    ], 
+    "headers": [], 
+    "is_filegroup": false, 
+    "language": "c", 
     "name": "head_of_line_blocking_bad_client_test", 
     "src": [
       "test/core/bad_client/tests/head_of_line_blocking.cc"
@@ -8407,7 +8425,6 @@
       "src/core/lib/channel/handshaker_registry.cc", 
       "src/core/lib/compression/compression.cc", 
       "src/core/lib/compression/compression_internal.cc", 
-      "src/core/lib/compression/compression_ruby.cc", 
       "src/core/lib/compression/message_compress.cc", 
       "src/core/lib/compression/stream_compression.cc", 
       "src/core/lib/compression/stream_compression_gzip.cc", 
@@ -8545,7 +8562,6 @@
       "include/grpc/byte_buffer.h", 
       "include/grpc/byte_buffer_reader.h", 
       "include/grpc/compression.h", 
-      "include/grpc/compression_ruby.h", 
       "include/grpc/fork.h", 
       "include/grpc/grpc.h", 
       "include/grpc/grpc_posix.h", 
@@ -8687,7 +8703,6 @@
       "include/grpc/byte_buffer.h", 
       "include/grpc/byte_buffer_reader.h", 
       "include/grpc/compression.h", 
-      "include/grpc/compression_ruby.h", 
       "include/grpc/fork.h", 
       "include/grpc/grpc.h", 
       "include/grpc/grpc_posix.h", 
diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json
index a061b4bd37..0d65cc483c 100644
--- a/tools/run_tests/generated/tests.json
+++ b/tools/run_tests/generated/tests.json
@@ -4715,6 +4715,32 @@
     "flaky": false, 
     "gtest": false, 
     "language": "c", 
+    "name": "duplicate_header_bad_client_test", 
+    "platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ], 
+    "uses_polling": true
+  }, 
+  {
+    "args": [], 
+    "benchmark": false, 
+    "ci_platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ], 
+    "cpu_cost": 1.0, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [
+      "uv"
+    ], 
+    "flaky": false, 
+    "gtest": false, 
+    "language": "c", 
     "name": "head_of_line_blocking_bad_client_test", 
     "platforms": [
       "linux", 
diff --git a/tools/run_tests/performance/OWNERS b/tools/run_tests/performance/OWNERS
index 98c81529f3..fc1d6eec9c 100644
--- a/tools/run_tests/performance/OWNERS
+++ b/tools/run_tests/performance/OWNERS
@@ -6,4 +6,4 @@ set noparent
 
 @ncteisen
 @matt-kwong
-@ctiller
+@jtattermusch