diff options
Diffstat (limited to 'tools')
20 files changed, 153 insertions, 34 deletions
diff --git a/tools/codegen/core/gen_static_metadata.py b/tools/codegen/core/gen_static_metadata.py index d4c0052b28..066f28416d 100755 --- a/tools/codegen/core/gen_static_metadata.py +++ b/tools/codegen/core/gen_static_metadata.py @@ -54,8 +54,8 @@ CONFIG = [ # well known method names '/grpc.lb.v1.LoadBalancer/BalanceLoad', # compression algorithm names - 'message/deflate', - 'message/gzip', + 'deflate', + 'gzip', 'stream/gzip', # metadata elements ('grpc-status', '0'), diff --git a/tools/distrib/clang_format_code.sh b/tools/distrib/clang_format_code.sh index 9233a230c9..cd7553e917 100755 --- a/tools/distrib/clang_format_code.sh +++ b/tools/distrib/clang_format_code.sh @@ -25,7 +25,9 @@ then docker build -t grpc_clang_format tools/dockerfile/grpc_clang_format # run clang-format against the checked out codebase - docker run -e TEST=$TEST -e CHANGED_FILES="$CHANGED_FILES" -e CLANG_FORMAT_ROOT="/local-code" --rm=true -v "${REPO_ROOT}":/local-code -t grpc_clang_format /clang_format_all_the_things.sh + # when modifying the checked-out files, the current user will be impersonated + # so that the updated files don't end up being owned by "root". + docker run -e TEST="$TEST" -e CHANGED_FILES="$CHANGED_FILES" -e CLANG_FORMAT_ROOT="/local-code" --rm=true -v "${REPO_ROOT}":/local-code --user "$(id -u):$(id -g)" -t grpc_clang_format /clang_format_all_the_things.sh else CLANG_FORMAT_ROOT="${REPO_ROOT}" tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh fi diff --git a/tools/distrib/clang_tidy_code.sh b/tools/distrib/clang_tidy_code.sh index 5da86aa277..9262b6bd3e 100755 --- a/tools/distrib/clang_tidy_code.sh +++ b/tools/distrib/clang_tidy_code.sh @@ -27,7 +27,9 @@ then docker build -t grpc_clang_tidy tools/dockerfile/grpc_clang_tidy # run clang-tidy against the checked out codebase - docker run -e TEST=$TEST -e CHANGED_FILES="$CHANGED_FILES" -e CLANG_TIDY_ROOT="/local-code" --rm=true -v "${REPO_ROOT}":/local-code -t grpc_clang_tidy /clang_tidy_all_the_things.sh "$@" + # when modifying the checked-out files, the current user will be impersonated + # so that the updated files don't end up being owned by "root". + docker run -e TEST="$TEST" -e CHANGED_FILES="$CHANGED_FILES" -e CLANG_TIDY_ROOT="/local-code" --rm=true -v "${REPO_ROOT}":/local-code --user "$(id -u):$(id -g)" -t grpc_clang_tidy /clang_tidy_all_the_things.sh "$@" else CLANG_TIDY_ROOT="${REPO_ROOT}" tools/dockerfile/grpc_clang_tidy/clang_tidy_all_the_things.sh "$@" fi diff --git a/tools/dockerfile/grpc_clang_format/Dockerfile b/tools/dockerfile/grpc_clang_format/Dockerfile index 8801315bc9..ecdc52ee31 100644 --- a/tools/dockerfile/grpc_clang_format/Dockerfile +++ b/tools/dockerfile/grpc_clang_format/Dockerfile @@ -23,4 +23,9 @@ RUN ln -s /clang+llvm-5.0.0-linux-x86_64-ubuntu14.04/bin/clang-tidy /usr/local/b ENV CLANG_TIDY=clang-tidy ADD clang_format_all_the_things.sh / + +# When running locally, we'll be impersonating the current user, so we need +# to make the script runnable by everyone. +RUN chmod a+rx /clang_format_all_the_things.sh + CMD ["echo 'Run with tools/distrib/clang_format_code.sh'"] diff --git a/tools/dockerfile/grpc_clang_tidy/Dockerfile b/tools/dockerfile/grpc_clang_tidy/Dockerfile index 9d9d70185b..eeba455341 100644 --- a/tools/dockerfile/grpc_clang_tidy/Dockerfile +++ b/tools/dockerfile/grpc_clang_tidy/Dockerfile @@ -38,4 +38,9 @@ RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.0.post1 six==1.10.0 twisted==17.5.0 ADD clang_tidy_all_the_things.sh / + +# When running locally, we'll be impersonating the current user, so we need +# to make the script runnable by everyone. +RUN chmod a+rx /clang_tidy_all_the_things.sh + CMD ["echo 'Run with tools/distrib/clang_tidy_code.sh'"] diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 317dd0e1eb..68c7c9205e 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -875,7 +875,6 @@ include/grpc++/support/time.h \ include/grpc/byte_buffer.h \ include/grpc/byte_buffer_reader.h \ include/grpc/compression.h \ -include/grpc/compression_ruby.h \ include/grpc/fork.h \ include/grpc/grpc.h \ include/grpc/grpc_posix.h \ diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 5036068839..bbd28a731b 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -876,7 +876,6 @@ include/grpc++/support/time.h \ include/grpc/byte_buffer.h \ include/grpc/byte_buffer_reader.h \ include/grpc/compression.h \ -include/grpc/compression_ruby.h \ include/grpc/fork.h \ include/grpc/grpc.h \ include/grpc/grpc_posix.h \ diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index d61a422626..762836fedd 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -801,7 +801,6 @@ include/grpc/byte_buffer.h \ include/grpc/byte_buffer_reader.h \ include/grpc/census.h \ include/grpc/compression.h \ -include/grpc/compression_ruby.h \ include/grpc/fork.h \ include/grpc/grpc.h \ include/grpc/grpc_posix.h \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 256e96edbf..1b780b3a64 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -801,7 +801,6 @@ include/grpc/byte_buffer.h \ include/grpc/byte_buffer_reader.h \ include/grpc/census.h \ include/grpc/compression.h \ -include/grpc/compression_ruby.h \ include/grpc/fork.h \ include/grpc/grpc.h \ include/grpc/grpc_posix.h \ @@ -1051,7 +1050,6 @@ src/core/lib/compression/algorithm_metadata.h \ src/core/lib/compression/compression.cc \ src/core/lib/compression/compression_internal.cc \ src/core/lib/compression/compression_internal.h \ -src/core/lib/compression/compression_ruby.cc \ src/core/lib/compression/message_compress.cc \ src/core/lib/compression/message_compress.h \ src/core/lib/compression/stream_compression.cc \ diff --git a/tools/internal_ci/helper_scripts/prepare_build_linux_rc b/tools/internal_ci/helper_scripts/prepare_build_linux_rc index 8715d6c8e0..74bbc85d72 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_linux_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_linux_rc @@ -27,11 +27,6 @@ echo 'DOCKER_OPTS="${DOCKER_OPTS} --graph=/tmpfs/docker"' | sudo tee --append /e echo 'DOCKER_OPTS="${DOCKER_OPTS} --registry-mirror=https://mirror.gcr.io"' | sudo tee --append /etc/default/docker sudo service docker restart -# Populate xdg-cache-home to workaround https://github.com/grpc/grpc/issues/11968 -sudo mkdir -p /tmp/xdg-cache-home -PYTHONWARNINGS=ignore XDG_CACHE_HOME=/tmp/xdg-cache-home sudo -E pip install setuptools wheel -PYTHONWARNINGS=ignore XDG_CACHE_HOME=/tmp/xdg-cache-home sudo -E pip install coverage==4.4 pylint==1.6.5 - # Download Docker images from DockerHub export DOCKERHUB_ORGANIZATION=grpctesting diff --git a/tools/internal_ci/linux/grpc_build_submodule_at_head.sh b/tools/internal_ci/linux/grpc_build_submodule_at_head.sh index e203a62b08..e68a2897f6 100755 --- a/tools/internal_ci/linux/grpc_build_submodule_at_head.sh +++ b/tools/internal_ci/linux/grpc_build_submodule_at_head.sh @@ -23,9 +23,8 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_rc # Update submodule and commit it so changes are passed to Docker -(cd third_party/$RUN_TESTS_FLAGS && git pull origin master) +(cd third_party/$RUN_TESTS_FLAGS && git fetch --all && git checkout origin/master) tools/buildgen/generate_projects.sh git -c user.name='foo' -c user.email='foo@google.com' commit -a -m 'Update submodule' tools/run_tests/run_tests_matrix.py -f linux --inner_jobs 4 -j 4 --internal_ci --build_only - diff --git a/tools/internal_ci/linux/grpc_full_performance_master.sh b/tools/internal_ci/linux/grpc_full_performance_master.sh index 18468395f6..4eddc18731 100755 --- a/tools/internal_ci/linux/grpc_full_performance_master.sh +++ b/tools/internal_ci/linux/grpc_full_performance_master.sh @@ -26,7 +26,7 @@ tools/run_tests/run_performance_tests.py \ --category scalable \ --remote_worker_host grpc-kokoro-performance-server-8core grpc-kokoro-performance-client-8core grpc-kokoro-performance-client2-8core \ -u kbuilder \ - --bq_result_table performance_test.kokoro_performance_experiment \ + --bq_result_table performance_test.performance_experiment \ --xml_report reports/8core/sponge_log.xml \ || EXIT_CODE=1 @@ -40,7 +40,7 @@ tools/run_tests/run_performance_tests.py \ --category scalable \ --remote_worker_host grpc-kokoro-performance-server-32core grpc-kokoro-performance-client-32core grpc-kokoro-performance-client2-32core \ -u kbuilder \ - --bq_result_table performance_test.kokoro_performance_experiment_32core \ + --bq_result_table performance_test.performance_experiment_32core \ --xml_report reports/32core/sponge_log.xml \ || EXIT_CODE=1 @@ -52,7 +52,7 @@ tools/run_tests/run_performance_tests.py \ -l csharp \ --category scalable \ --remote_worker_host grpc-kokoro-performance-windows1 grpc-kokoro-performance-windows2 \ - --bq_result_table performance_test.kokoro_performance_experiment_windows \ + --bq_result_table performance_test.performance_experiment_windows \ --xml_report reports/windows/sponge_log.xml \ || EXIT_CODE=1 diff --git a/tools/internal_ci/linux/grpc_full_performance_release.cfg b/tools/internal_ci/linux/grpc_full_performance_release.cfg new file mode 100644 index 0000000000..e9a4bcdcaf --- /dev/null +++ b/tools/internal_ci/linux/grpc_full_performance_release.cfg @@ -0,0 +1,25 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_full_performance_release.sh" +timeout_mins: 600 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "**/perf_reports/**" + } +} diff --git a/tools/internal_ci/linux/grpc_full_performance_release.sh b/tools/internal_ci/linux/grpc_full_performance_release.sh new file mode 100755 index 0000000000..a31cea281f --- /dev/null +++ b/tools/internal_ci/linux/grpc_full_performance_release.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -ex + +# Enter the gRPC repo root +cd $(dirname $0)/../../.. + +source tools/internal_ci/helper_scripts/prepare_build_linux_perf_multilang_rc + +# run 8core client vs 8core server +tools/run_tests/run_performance_tests.py \ + -l c++ csharp ruby java python go php7 php7_protobuf_c \ + --netperf \ + --category scalable \ + --remote_worker_host grpc-kokoro-performance-server-8core grpc-kokoro-performance-client-8core grpc-kokoro-performance-client2-8core \ + -u kbuilder \ + --bq_result_table performance_released.performance_experiment \ + --xml_report reports/8core/sponge_log.xml \ + || EXIT_CODE=1 + +# prevent pushing leftover build files to remote hosts in the next step. +git clean -fdxq -e reports + +# scalability with 32cores (and upload to a different BQ table) +tools/run_tests/run_performance_tests.py \ + -l c++ java csharp go \ + --netperf \ + --category scalable \ + --remote_worker_host grpc-kokoro-performance-server-32core grpc-kokoro-performance-client-32core grpc-kokoro-performance-client2-32core \ + -u kbuilder \ + --bq_result_table performance_released.performance_experiment_32core \ + --xml_report reports/32core/sponge_log.xml \ + || EXIT_CODE=1 + +# prevent pushing leftover build files to remote hosts in the next step. +git clean -fdxq -e reports + +# selected scenarios on Windows +tools/run_tests/run_performance_tests.py \ + -l csharp \ + --category scalable \ + --remote_worker_host grpc-kokoro-performance-windows1 grpc-kokoro-performance-windows2 \ + --bq_result_table performance_released.performance_experiment_windows \ + --xml_report reports/windows/sponge_log.xml \ + || EXIT_CODE=1 + +exit $EXIT_CODE diff --git a/tools/run_tests/artifacts/build_artifact_python.bat b/tools/run_tests/artifacts/build_artifact_python.bat index 3a0564129c..d277668c94 100644 --- a/tools/run_tests/artifacts/build_artifact_python.bat +++ b/tools/run_tests/artifacts/build_artifact_python.bat @@ -16,7 +16,8 @@ set PATH=C:\%1;C:\%1\scripts;C:\msys64\mingw%2\bin;C:\tools\msys64\mingw%2\bin;%PATH% pip install --upgrade six -pip install --upgrade setuptools +@rem some artifacts are broken for setuptools 38.5.0. See https://github.com/grpc/grpc/issues/14317 +pip install --upgrade setuptools==38.2.4 pip install -rrequirements.txt set GRPC_PYTHON_BUILD_WITH_CYTHON=1 diff --git a/tools/run_tests/dockerize/build_docker_and_run_tests.sh b/tools/run_tests/dockerize/build_docker_and_run_tests.sh index 8dca05ea35..21eccbaac9 100755 --- a/tools/run_tests/dockerize/build_docker_and_run_tests.sh +++ b/tools/run_tests/dockerize/build_docker_and_run_tests.sh @@ -25,10 +25,6 @@ cd - # Ensure existence of ccache directory mkdir -p /tmp/ccache -# Ensure existence of the home directory for XDG caches (e.g. what pip uses for -# its cache location now that --download-cache is deprecated). -mkdir -p /tmp/xdg-cache-home - # Inputs # DOCKERFILE_DIR - Directory in which Dockerfile file is located. # DOCKER_RUN_SCRIPT - Script to run under docker (relative to grpc repo root) @@ -61,7 +57,6 @@ docker run \ -e "config=$config" \ -e "arch=$arch" \ -e CCACHE_DIR=/tmp/ccache \ - -e XDG_CACHE_HOME=/tmp/xdg-cache-home \ -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \ -e HOST_GIT_ROOT="$git_root" \ -e LOCAL_GIT_ROOT=$docker_instance_git_root \ @@ -79,7 +74,6 @@ docker run \ -v "$git_root:$docker_instance_git_root" \ -v /tmp/ccache:/tmp/ccache \ -v /tmp/npm-cache:/tmp/npm-cache \ - -v /tmp/xdg-cache-home:/tmp/xdg-cache-home \ -w /var/local/git/grpc \ --name="$CONTAINER_NAME" \ "$DOCKER_IMAGE_NAME" \ diff --git a/tools/run_tests/dockerize/docker_run_tests.sh b/tools/run_tests/dockerize/docker_run_tests.sh index df9d3d1ca2..89ee315fd2 100755 --- a/tools/run_tests/dockerize/docker_run_tests.sh +++ b/tools/run_tests/dockerize/docker_run_tests.sh @@ -22,10 +22,6 @@ export CONFIG=$config export ASAN_SYMBOLIZER_PATH=/usr/bin/llvm-symbolizer export PATH=$PATH:/usr/bin/llvm-symbolizer -# Ensure that programs depending on current-user-ownership of cache directories -# are satisfied (it's being mounted from outside the image). -chown "$(whoami)" "$XDG_CACHE_HOME" - mkdir -p /var/local/git git clone /var/local/jenkins/grpc /var/local/git/grpc # clone gRPC submodules, use data from locally cloned submodules where possible diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index aa90753fbd..48252f24ff 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -5133,6 +5133,24 @@ "headers": [], "is_filegroup": false, "language": "c", + "name": "duplicate_header_bad_client_test", + "src": [ + "test/core/bad_client/tests/duplicate_header.cc" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "bad_client_test", + "gpr", + "gpr_test_util", + "grpc_test_util_unsecure", + "grpc_unsecure" + ], + "headers": [], + "is_filegroup": false, + "language": "c", "name": "head_of_line_blocking_bad_client_test", "src": [ "test/core/bad_client/tests/head_of_line_blocking.cc" @@ -8407,7 +8425,6 @@ "src/core/lib/channel/handshaker_registry.cc", "src/core/lib/compression/compression.cc", "src/core/lib/compression/compression_internal.cc", - "src/core/lib/compression/compression_ruby.cc", "src/core/lib/compression/message_compress.cc", "src/core/lib/compression/stream_compression.cc", "src/core/lib/compression/stream_compression_gzip.cc", @@ -8545,7 +8562,6 @@ "include/grpc/byte_buffer.h", "include/grpc/byte_buffer_reader.h", "include/grpc/compression.h", - "include/grpc/compression_ruby.h", "include/grpc/fork.h", "include/grpc/grpc.h", "include/grpc/grpc_posix.h", @@ -8687,7 +8703,6 @@ "include/grpc/byte_buffer.h", "include/grpc/byte_buffer_reader.h", "include/grpc/compression.h", - "include/grpc/compression_ruby.h", "include/grpc/fork.h", "include/grpc/grpc.h", "include/grpc/grpc_posix.h", diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index a061b4bd37..0d65cc483c 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -4715,6 +4715,32 @@ "flaky": false, "gtest": false, "language": "c", + "name": "duplicate_header_bad_client_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "uses_polling": true + }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "gtest": false, + "language": "c", "name": "head_of_line_blocking_bad_client_test", "platforms": [ "linux", diff --git a/tools/run_tests/performance/OWNERS b/tools/run_tests/performance/OWNERS index 98c81529f3..fc1d6eec9c 100644 --- a/tools/run_tests/performance/OWNERS +++ b/tools/run_tests/performance/OWNERS @@ -6,4 +6,4 @@ set noparent @ncteisen @matt-kwong -@ctiller +@jtattermusch |