From 2457a04bff9180e60570d80152af931281903826 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Mon, 14 Aug 2017 14:59:36 -0700 Subject: Add Performance profiling job to Kokoro --- .../helper_scripts/prepare_build_linux_perf_rc | 22 +++++++++++++ .../linux/grpc_performance_profile_daily.cfg | 26 +++++++++++++++ .../linux/grpc_performance_profile_daily.sh | 37 ++++++++++++++++++++++ .../linux/grpc_performance_profile_master.cfg | 26 +++++++++++++++ .../linux/grpc_performance_profile_master.sh | 32 +++++++++++++++++++ 5 files changed, 143 insertions(+) create mode 100644 tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc create mode 100644 tools/internal_ci/linux/grpc_performance_profile_daily.cfg create mode 100755 tools/internal_ci/linux/grpc_performance_profile_daily.sh create mode 100644 tools/internal_ci/linux/grpc_performance_profile_master.cfg create mode 100755 tools/internal_ci/linux/grpc_performance_profile_master.sh (limited to 'tools/internal_ci') diff --git a/tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc b/tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc new file mode 100644 index 0000000000..1b7779caa8 --- /dev/null +++ b/tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc @@ -0,0 +1,22 @@ +#!/bin/bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Source this rc script to prepare the environment for linux perf builds + +# Need to increase open files limit and size for perf test +ulimit -n 32768 +ulimit -c unlimited + +git submodule update --init diff --git a/tools/internal_ci/linux/grpc_performance_profile_daily.cfg b/tools/internal_ci/linux/grpc_performance_profile_daily.cfg new file mode 100644 index 0000000000..9831869edb --- /dev/null +++ b/tools/internal_ci/linux/grpc_performance_profile_daily.cfg @@ -0,0 +1,26 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_performance_profile_daily.sh" +timeout_mins: 1440 +action { + define_artifacts { + regex: "**" + regex: "github/grpc/reports/**" + } +} + diff --git a/tools/internal_ci/linux/grpc_performance_profile_daily.sh b/tools/internal_ci/linux/grpc_performance_profile_daily.sh new file mode 100755 index 0000000000..25523e21b8 --- /dev/null +++ b/tools/internal_ci/linux/grpc_performance_profile_daily.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# Enter the gRPC repo root +cd $(dirname $0)/../../.. + +source tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc + +CPUS=`python -c 'import multiprocessing; print multiprocessing.cpu_count()'` + +make CONFIG=opt memory_profile_test memory_profile_client memory_profile_server -j $CPUS +bins/opt/memory_profile_test +bq load microbenchmarks.memory memory_usage.csv + +tools/run_tests/run_microbenchmark.py --collect summary --bigquery_upload || FAILED="true" + +# kill port_server.py to prevent the build from hanging +ps aux | grep port_server\\.py | awk '{print $2}' | xargs kill -9 + +if [ "$FAILED" != "" ] +then + exit 1 +fi diff --git a/tools/internal_ci/linux/grpc_performance_profile_master.cfg b/tools/internal_ci/linux/grpc_performance_profile_master.cfg new file mode 100644 index 0000000000..e4cefbc3f4 --- /dev/null +++ b/tools/internal_ci/linux/grpc_performance_profile_master.cfg @@ -0,0 +1,26 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_performance_profile_master.sh" +timeout_mins: 600 +action { + define_artifacts { + regex: "**" + regex: "github/grpc/reports/**" + } +} + diff --git a/tools/internal_ci/linux/grpc_performance_profile_master.sh b/tools/internal_ci/linux/grpc_performance_profile_master.sh new file mode 100755 index 0000000000..40bbfe89dc --- /dev/null +++ b/tools/internal_ci/linux/grpc_performance_profile_master.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# Enter the gRPC repo root +cd $(dirname $0)/../../.. + +source tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc + +tools/jenkins/run_performance_profile_hourly.sh || FAILED="true" + +# kill port_server.py to prevent the build from hanging +ps aux | grep port_server\\.py | awk '{print $2}' | xargs kill -9 + +if [ "$FAILED" != "" ] +then + exit 1 +fi + -- cgit v1.2.3 From 1caed0e06ed81b525a71b2d9ba2ee23f9dbd1529 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Mon, 21 Aug 2017 15:24:17 -0700 Subject: Add max_time flag to Kokoro PR jobs --- tools/internal_ci/linux/pull_request/grpc_basictests_c_cpp_dbg.cfg | 2 +- tools/internal_ci/linux/pull_request/grpc_basictests_c_cpp_opt.cfg | 2 +- tools/internal_ci/linux/pull_request/grpc_basictests_multilang.cfg | 2 +- tools/internal_ci/macos/pull_request/grpc_basictests_dbg.cfg | 2 +- tools/internal_ci/macos/pull_request/grpc_basictests_opt.cfg | 2 +- tools/internal_ci/windows/pull_request/grpc_basictests.cfg | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) (limited to 'tools/internal_ci') diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_c_cpp_dbg.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_c_cpp_dbg.cfg index dcc6265b34..8124f5c1b3 100644 --- a/tools/internal_ci/linux/pull_request/grpc_basictests_c_cpp_dbg.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_c_cpp_dbg.cfg @@ -26,5 +26,5 @@ action { env_vars { key: "RUN_TESTS_FLAGS" - value: "-f basictests linux corelang dbg --inner_jobs 16 -j 1 --internal_ci" + value: "-f basictests linux corelang dbg --inner_jobs 16 -j 1 --internal_ci --max_time=3600" } diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_c_cpp_opt.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_c_cpp_opt.cfg index f60beaf15c..ecedc73e44 100644 --- a/tools/internal_ci/linux/pull_request/grpc_basictests_c_cpp_opt.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_c_cpp_opt.cfg @@ -26,5 +26,5 @@ action { env_vars { key: "RUN_TESTS_FLAGS" - value: "-f basictests linux corelang opt --inner_jobs 16 -j 1 --internal_ci" + value: "-f basictests linux corelang opt --inner_jobs 16 -j 1 --internal_ci --max_time=3600" } diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_multilang.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_multilang.cfg index 7c16cf6a10..f7e8d8ad33 100644 --- a/tools/internal_ci/linux/pull_request/grpc_basictests_multilang.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_multilang.cfg @@ -26,5 +26,5 @@ action { env_vars { key: "RUN_TESTS_FLAGS" - value: "-f basictests linux multilang --inner_jobs 16 -j 2 --internal_ci" + value: "-f basictests linux multilang --inner_jobs 16 -j 2 --internal_ci --max_time=3600" } diff --git a/tools/internal_ci/macos/pull_request/grpc_basictests_dbg.cfg b/tools/internal_ci/macos/pull_request/grpc_basictests_dbg.cfg index f058f0c7e4..30c01d3e2f 100644 --- a/tools/internal_ci/macos/pull_request/grpc_basictests_dbg.cfg +++ b/tools/internal_ci/macos/pull_request/grpc_basictests_dbg.cfg @@ -27,5 +27,5 @@ action { env_vars { key: "RUN_TESTS_FLAGS" - value: "-f basictests macos dbg --internal_ci -j 1 --inner_jobs 4" + value: "-f basictests macos dbg --internal_ci -j 1 --inner_jobs 4 --max_time=3600" } diff --git a/tools/internal_ci/macos/pull_request/grpc_basictests_opt.cfg b/tools/internal_ci/macos/pull_request/grpc_basictests_opt.cfg index 5048baaf48..b63ee713bc 100644 --- a/tools/internal_ci/macos/pull_request/grpc_basictests_opt.cfg +++ b/tools/internal_ci/macos/pull_request/grpc_basictests_opt.cfg @@ -27,5 +27,5 @@ action { env_vars { key: "RUN_TESTS_FLAGS" - value: "-f basictests macos opt --internal_ci -j 1 --inner_jobs 4" + value: "-f basictests macos opt --internal_ci -j 1 --inner_jobs 4 --max_time=3600" } diff --git a/tools/internal_ci/windows/pull_request/grpc_basictests.cfg b/tools/internal_ci/windows/pull_request/grpc_basictests.cfg index a116738651..678ebeb2af 100644 --- a/tools/internal_ci/windows/pull_request/grpc_basictests.cfg +++ b/tools/internal_ci/windows/pull_request/grpc_basictests.cfg @@ -26,5 +26,5 @@ action { env_vars { key: "RUN_TESTS_FLAGS" - value: "-f basictests windows -j 1 --inner_jobs 8 --internal_ci" + value: "-f basictests windows -j 1 --inner_jobs 8 --internal_ci --max_time=3600" } -- cgit v1.2.3 From 34e2d3609649142af5955f8668579f5b7ddee357 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Mon, 21 Aug 2017 11:28:32 -0700 Subject: Add test filtering to Kokoro PR jobs --- tools/internal_ci/helper_scripts/prepare_build_linux_rc | 7 +++++++ tools/internal_ci/linux/sanitizer/pull_request/grpc_c_asan.cfg | 2 +- tools/internal_ci/linux/sanitizer/pull_request/grpc_c_msan.cfg | 2 +- tools/internal_ci/linux/sanitizer/pull_request/grpc_c_tsan.cfg | 2 +- tools/internal_ci/linux/sanitizer/pull_request/grpc_c_ubsan.cfg | 2 +- tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_asan.cfg | 2 +- tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_tsan.cfg | 2 +- 7 files changed, 13 insertions(+), 6 deletions(-) (limited to 'tools/internal_ci') diff --git a/tools/internal_ci/helper_scripts/prepare_build_linux_rc b/tools/internal_ci/helper_scripts/prepare_build_linux_rc index e6f195c87d..91627d60cb 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_linux_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_linux_rc @@ -31,4 +31,11 @@ PYTHONWARNINGS=ignore XDG_CACHE_HOME=/tmp/xdg-cache-home sudo -E pip install cov # Download Docker images from DockerHub export DOCKERHUB_ORGANIZATION=grpctesting +# If this is a PR using RUN_TESTS_FLAGS var, then add flags to filter tests +if [ -n "$KOKORO_GITHUB_PULL_REQUEST_NUMBER" ] && [ -n "$RUN_TESTS_FLAGS" ]; then + sudo apt-get install -y jq + ghprbTargetBranch=$(curl -s https://api.github.com/repos/grpc/grpc/pulls/$KOKORO_GITHUB_PULL_REQUEST_NUMBER | jq -r .base.ref) + export RUN_TESTS_FLAGS="$RUN_TESTS_FLAGS --filter_pr_tests --base_branch origin/$ghprbTargetBranch" +fi + git submodule update --init diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_asan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_asan.cfg index 649d93d0a1..db9cb4f597 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_asan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_asan.cfg @@ -27,5 +27,5 @@ action { env_vars { key: "RUN_TESTS_FLAGS" - value: "-f c asan --inner_jobs 16 -j 1 --internal_ci --filter_pr_tests --base_branch origin/master --max_time=3600" + value: "-f c asan --inner_jobs 16 -j 1 --internal_ci --max_time=3600" } diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_msan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_msan.cfg index d5ba2031fc..ba651d357c 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_msan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_msan.cfg @@ -27,5 +27,5 @@ action { env_vars { key: "RUN_TESTS_FLAGS" - value: "-f c msan --inner_jobs 16 -j 1 --internal_ci --filter_pr_tests --base_branch origin/master --max_time=3600" + value: "-f c msan --inner_jobs 16 -j 1 --internal_ci --max_time=3600" } diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_tsan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_tsan.cfg index 67d4356821..ccbd95ae63 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_tsan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_tsan.cfg @@ -27,5 +27,5 @@ action { env_vars { key: "RUN_TESTS_FLAGS" - value: "-f c tsan --inner_jobs 16 -j 1 --internal_ci --filter_pr_tests --base_branch origin/master --max_time=3600" + value: "-f c tsan --inner_jobs 16 -j 1 --internal_ci --max_time=3600" } diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_ubsan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_ubsan.cfg index c162d1d1d0..c240d3a573 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_ubsan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_ubsan.cfg @@ -27,5 +27,5 @@ action { env_vars { key: "RUN_TESTS_FLAGS" - value: "-f c ubsan --inner_jobs 16 -j 1 --internal_ci --filter_pr_tests --base_branch origin/master --max_time=3600" + value: "-f c ubsan --inner_jobs 16 -j 1 --internal_ci --max_time=3600" } diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_asan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_asan.cfg index 29f1fce568..38db2f4f60 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_asan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_asan.cfg @@ -27,5 +27,5 @@ action { env_vars { key: "RUN_TESTS_FLAGS" - value: "-f c++ asan --inner_jobs 16 -j 1 --internal_ci --filter_pr_tests --base_branch origin/master --max_time=3600" + value: "-f c++ asan --inner_jobs 16 -j 1 --internal_ci --max_time=3600" } diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_tsan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_tsan.cfg index 331f73b3c7..cb15ca34fd 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_tsan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_tsan.cfg @@ -27,5 +27,5 @@ action { env_vars { key: "RUN_TESTS_FLAGS" - value: "-f c++ tsan --inner_jobs 16 -j 1 --internal_ci --filter_pr_tests --base_branch origin/master --max_time=3600" + value: "-f c++ tsan --inner_jobs 16 -j 1 --internal_ci --max_time=3600" } -- cgit v1.2.3 From 1c8e04366e8ccacf31dbe7fd14206a2603e3521f Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Wed, 23 Aug 2017 11:59:49 -0700 Subject: Fix uploading Mac results to BQ --- tools/internal_ci/helper_scripts/prepare_build_macos_rc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'tools/internal_ci') diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc index bd250957f4..6da0e1eeff 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc @@ -28,8 +28,7 @@ ulimit -n 10000 ulimit -a # Add GCP credentials for BQ access -# pip does not install google-api-python-client properly, so use easy_install -sudo easy_install --upgrade google-api-python-client +pip install google-api-python-client --user python export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json # required to build protobuf @@ -53,9 +52,10 @@ pod repo update # needed by python # python brew install coreutils # we need grealpath -sudo pip install virtualenv -sudo pip install -U six tox setuptools -export PYTHONPATH=/Library/Python/3.4/site-packages +pip install virtualenv --user python +pip install -U six tox setuptools --user python +export PYTHONPATH=/Library/Python/2.7/site-packages +source ~/.bashrc # python 3.4 wget -q https://www.python.org/ftp/python/3.4.4/python-3.4.4-macosx10.6.pkg -- cgit v1.2.3 From 1720b0b97f4d3808abf55f79819ed7fd6d526167 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Fri, 1 Sep 2017 08:02:20 -0700 Subject: Disable uploading Kokoro Mac results --- tools/internal_ci/helper_scripts/prepare_build_macos_rc | 3 +-- tools/internal_ci/macos/grpc_basictests_dbg.cfg | 2 +- tools/internal_ci/macos/grpc_basictests_opt.cfg | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) (limited to 'tools/internal_ci') diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc index 6da0e1eeff..f7fbec93ff 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc @@ -54,8 +54,7 @@ pod repo update # needed by python brew install coreutils # we need grealpath pip install virtualenv --user python pip install -U six tox setuptools --user python -export PYTHONPATH=/Library/Python/2.7/site-packages -source ~/.bashrc +export PYTHONPATH=/Library/Python/3.4/site-packages # python 3.4 wget -q https://www.python.org/ftp/python/3.4.4/python-3.4.4-macosx10.6.pkg diff --git a/tools/internal_ci/macos/grpc_basictests_dbg.cfg b/tools/internal_ci/macos/grpc_basictests_dbg.cfg index 53bda1ff0a..f058f0c7e4 100644 --- a/tools/internal_ci/macos/grpc_basictests_dbg.cfg +++ b/tools/internal_ci/macos/grpc_basictests_dbg.cfg @@ -27,5 +27,5 @@ action { env_vars { key: "RUN_TESTS_FLAGS" - value: "-f basictests macos dbg --internal_ci -j 1 --inner_jobs 4 --bq_result_table aggregate_results" + value: "-f basictests macos dbg --internal_ci -j 1 --inner_jobs 4" } diff --git a/tools/internal_ci/macos/grpc_basictests_opt.cfg b/tools/internal_ci/macos/grpc_basictests_opt.cfg index d359eb601a..5048baaf48 100644 --- a/tools/internal_ci/macos/grpc_basictests_opt.cfg +++ b/tools/internal_ci/macos/grpc_basictests_opt.cfg @@ -27,5 +27,5 @@ action { env_vars { key: "RUN_TESTS_FLAGS" - value: "-f basictests macos opt --internal_ci -j 1 --inner_jobs 4 --bq_result_table aggregate_results" + value: "-f basictests macos opt --internal_ci -j 1 --inner_jobs 4" } -- cgit v1.2.3 From aafa6994296430795ddb0e8b1eaf1fe7c7037a28 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 6 Sep 2017 17:20:15 +0200 Subject: prevent timeout for node and python windows artifact builds --- tools/internal_ci/windows/grpc_build_artifacts.bat | 2 +- tools/run_tests/artifacts/artifact_targets.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'tools/internal_ci') diff --git a/tools/internal_ci/windows/grpc_build_artifacts.bat b/tools/internal_ci/windows/grpc_build_artifacts.bat index 29c876dd35..17d9571d43 100644 --- a/tools/internal_ci/windows/grpc_build_artifacts.bat +++ b/tools/internal_ci/windows/grpc_build_artifacts.bat @@ -26,7 +26,7 @@ cd /d %~dp0\..\..\.. call tools/internal_ci/helper_scripts/prepare_build_windows.bat -python tools/run_tests/task_runner.py -f artifact windows || goto :error +python tools/run_tests/task_runner.py -f artifact windows -j 4 || goto :error goto :EOF :error diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index bd186263ca..12263282ae 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -150,6 +150,7 @@ class PythonArtifact: self.py_version, '32' if self.arch == 'x86' else '64'], environ=environ, + timeout_seconds=45*60, use_workspace=True) else: environ['PYTHON'] = self.py_version @@ -257,6 +258,7 @@ class NodeExtArtifact: ['tools\\run_tests\\artifacts\\build_artifact_node.bat', self.gyp_arch], use_workspace=True, + timeout_seconds=45*60, cpu_cost=cpu_cost) else: if self.platform == 'linux': -- cgit v1.2.3 From ee6151b70e67b98e37e09109e4b5a1027b609538 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 8 Sep 2017 13:30:02 +0200 Subject: add kokoro sanity webhook job --- .../internal_ci/linux/grpc_sanity_webhook_test.cfg | 30 ++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 tools/internal_ci/linux/grpc_sanity_webhook_test.cfg (limited to 'tools/internal_ci') diff --git a/tools/internal_ci/linux/grpc_sanity_webhook_test.cfg b/tools/internal_ci/linux/grpc_sanity_webhook_test.cfg new file mode 100644 index 0000000000..24e7984f3a --- /dev/null +++ b/tools/internal_ci/linux/grpc_sanity_webhook_test.cfg @@ -0,0 +1,30 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" +timeout_mins: 20 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + } +} + +env_vars { + key: "RUN_TESTS_FLAGS" + value: "-f basictests linux sanity opt --inner_jobs 16 -j 1 --internal_ci" +} -- cgit v1.2.3 From 4657a4818b56e11d6b11654933dcd80a434f9f93 Mon Sep 17 00:00:00 2001 From: Ken Payson Date: Fri, 8 Sep 2017 14:49:38 -0700 Subject: Warm pip cache with libraries used by virtualenv --- tools/internal_ci/helper_scripts/prepare_build_linux_rc | 1 + 1 file changed, 1 insertion(+) (limited to 'tools/internal_ci') diff --git a/tools/internal_ci/helper_scripts/prepare_build_linux_rc b/tools/internal_ci/helper_scripts/prepare_build_linux_rc index 91627d60cb..2ade8dac51 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_linux_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_linux_rc @@ -26,6 +26,7 @@ sudo service docker restart # Populate xdg-cache-home to workaround https://github.com/grpc/grpc/issues/11968 sudo mkdir -p /tmp/xdg-cache-home +PYTHONWARNINGS=ignore XDG_CACHE_HOME=/tmp/xdg-cache-home sudo -E pip install setuptools wheel PYTHONWARNINGS=ignore XDG_CACHE_HOME=/tmp/xdg-cache-home sudo -E pip install coverage==4.4 pylint==1.6.5 # Download Docker images from DockerHub -- cgit v1.2.3 From c58f84cbc11be522bbc732fe759655fdd2a96893 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Sat, 9 Sep 2017 23:39:37 +0200 Subject: more reasonable kokoro jobs timeouts --- tools/internal_ci/linux/grpc_interop_tocloud.cfg | 2 +- tools/internal_ci/windows/grpc_basictests.cfg | 2 +- tools/internal_ci/windows/grpc_portability.cfg | 2 +- tools/internal_ci/windows/grpc_portability_build_only.cfg | 2 +- tools/internal_ci/windows/pull_request/grpc_basictests.cfg | 2 +- tools/internal_ci/windows/pull_request/grpc_portability.cfg | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) (limited to 'tools/internal_ci') diff --git a/tools/internal_ci/linux/grpc_interop_tocloud.cfg b/tools/internal_ci/linux/grpc_interop_tocloud.cfg index c613f668d9..2803616007 100644 --- a/tools/internal_ci/linux/grpc_interop_tocloud.cfg +++ b/tools/internal_ci/linux/grpc_interop_tocloud.cfg @@ -17,7 +17,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/linux/grpc_interop_tocloud.sh" # grpc_interop tests can take 6+ hours to complete. -timeout_mins: 480 +timeout_mins: 60 action { define_artifacts { regex: "**/sponge_log.xml" diff --git a/tools/internal_ci/windows/grpc_basictests.cfg b/tools/internal_ci/windows/grpc_basictests.cfg index 396d29ef09..8e644e4c5e 100644 --- a/tools/internal_ci/windows/grpc_basictests.cfg +++ b/tools/internal_ci/windows/grpc_basictests.cfg @@ -16,7 +16,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" -timeout_mins: 360 +timeout_mins: 240 action { define_artifacts { regex: "**/*sponge_log.xml" diff --git a/tools/internal_ci/windows/grpc_portability.cfg b/tools/internal_ci/windows/grpc_portability.cfg index cd04d77f5d..94e71753ef 100644 --- a/tools/internal_ci/windows/grpc_portability.cfg +++ b/tools/internal_ci/windows/grpc_portability.cfg @@ -16,7 +16,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" -timeout_mins: 360 +timeout_mins: 240 action { define_artifacts { regex: "**/*sponge_log.xml" diff --git a/tools/internal_ci/windows/grpc_portability_build_only.cfg b/tools/internal_ci/windows/grpc_portability_build_only.cfg index b2b58ece2d..3bc27f1f24 100644 --- a/tools/internal_ci/windows/grpc_portability_build_only.cfg +++ b/tools/internal_ci/windows/grpc_portability_build_only.cfg @@ -16,7 +16,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" -timeout_mins: 360 +timeout_mins: 240 action { define_artifacts { regex: "**/*sponge_log.xml" diff --git a/tools/internal_ci/windows/pull_request/grpc_basictests.cfg b/tools/internal_ci/windows/pull_request/grpc_basictests.cfg index 678ebeb2af..91777cd7cb 100644 --- a/tools/internal_ci/windows/pull_request/grpc_basictests.cfg +++ b/tools/internal_ci/windows/pull_request/grpc_basictests.cfg @@ -16,7 +16,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" -timeout_mins: 360 +timeout_mins: 240 action { define_artifacts { regex: "**/*sponge_log.xml" diff --git a/tools/internal_ci/windows/pull_request/grpc_portability.cfg b/tools/internal_ci/windows/pull_request/grpc_portability.cfg index c395cb4a94..2bda487629 100644 --- a/tools/internal_ci/windows/pull_request/grpc_portability.cfg +++ b/tools/internal_ci/windows/pull_request/grpc_portability.cfg @@ -16,7 +16,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" -timeout_mins: 360 +timeout_mins: 240 action { define_artifacts { regex: "**/*sponge_log.xml" -- cgit v1.2.3 From 6d11e9092667565a181d051932e04c50ab1db099 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Wed, 23 Aug 2017 11:43:40 -0700 Subject: Add Kokoro test filtering for MacOS/Windows --- tools/internal_ci/helper_scripts/prepare_build_macos_rc | 7 +++++++ tools/internal_ci/helper_scripts/prepare_build_windows.bat | 8 ++++++++ tools/run_tests/python_utils/filter_pull_request_tests.py | 2 +- 3 files changed, 16 insertions(+), 1 deletion(-) (limited to 'tools/internal_ci') diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc index bd250957f4..2a1225683d 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc @@ -32,6 +32,13 @@ ulimit -a sudo easy_install --upgrade google-api-python-client export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json +# If this is a PR using RUN_TESTS_FLAGS var, then add flags to filter tests +if [ -n "$KOKORO_GITHUB_PULL_REQUEST_NUMBER" ] && [ -n "$RUN_TESTS_FLAGS" ]; then + brew install jq + ghprbTargetBranch=$(curl -s https://api.github.com/repos/grpc/grpc/pulls/$KOKORO_GITHUB_PULL_REQUEST_NUMBER | jq -r .base.ref) + export RUN_TESTS_FLAGS="$RUN_TESTS_FLAGS --filter_pr_tests --base_branch origin/$ghprbTargetBranch" +fi + # required to build protobuf brew install gflags diff --git a/tools/internal_ci/helper_scripts/prepare_build_windows.bat b/tools/internal_ci/helper_scripts/prepare_build_windows.bat index 69e087e1db..f625755db9 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_windows.bat +++ b/tools/internal_ci/helper_scripts/prepare_build_windows.bat @@ -18,6 +18,14 @@ set PATH=C:\tools\msys64\usr\bin;C:\Python27;%PATH% bash tools/internal_ci/helper_scripts/gen_report_index.sh +@rem If this is a PR using RUN_TESTS_FLAGS var, then add flags to filter tests +if defined KOKORO_GITHUB_PULL_REQUEST_NUMBER if defined RUN_TESTS_FLAGS ( + chocolatey install -y jq + for /f "usebackq delims=" %%x in (`curl -s https://api.github.com/repos/grpc/grpc/pulls/%KOKORO_GITHUB_PULL_REQUEST_NUMBER% ^| jq -r .base.ref`) do ( + set RUN_TESTS_FLAGS=%RUN_TESTS_FLAGS% --filter_pr_tests --base_branch origin/%%x + ) +) + @rem Update DNS settings to: @rem 1. allow resolving metadata.google.internal hostname @rem 2. make fetching default GCE credential by oauth2client work diff --git a/tools/run_tests/python_utils/filter_pull_request_tests.py b/tools/run_tests/python_utils/filter_pull_request_tests.py index 032c0701e2..4ad981237b 100644 --- a/tools/run_tests/python_utils/filter_pull_request_tests.py +++ b/tools/run_tests/python_utils/filter_pull_request_tests.py @@ -128,7 +128,7 @@ def _get_changed_files(base_branch): # Get file changes between branch and merge-base of specified branch # Not combined to be Windows friendly base_commit = check_output(["git", "merge-base", base_branch, "HEAD"]).rstrip() - return check_output(["git", "diff", base_commit, "--name-only"]).splitlines() + return check_output(["git", "diff", base_commit, "--name-only", "HEAD"]).splitlines() def _can_skip_tests(file_names, triggers): -- cgit v1.2.3 From 707145c78a7e1d65ddf14836adbbb7032172a3a0 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 13 Sep 2017 10:02:58 +0200 Subject: mingw already installed on win kokoro workers --- tools/internal_ci/windows/grpc_build_artifacts.bat | 2 -- 1 file changed, 2 deletions(-) (limited to 'tools/internal_ci') diff --git a/tools/internal_ci/windows/grpc_build_artifacts.bat b/tools/internal_ci/windows/grpc_build_artifacts.bat index 17d9571d43..c6e523697c 100644 --- a/tools/internal_ci/windows/grpc_build_artifacts.bat +++ b/tools/internal_ci/windows/grpc_build_artifacts.bat @@ -19,8 +19,6 @@ rename C:\Python34_32bit Python34_32bits rename C:\Python35_32bit Python35_32bits rename C:\Python36_32bit Python36_32bits -pacman -S --noconfirm mingw64/mingw-w64-x86_64-gcc mingw32/mingw-w64-i686-gcc - @rem enter repo root cd /d %~dp0\..\..\.. -- cgit v1.2.3 From e97bc51d3966aaa6d8bbcd6ec9238063bba8dad2 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 13 Sep 2017 12:26:32 +0200 Subject: already installed on mac workers --- tools/internal_ci/helper_scripts/prepare_build_macos_rc | 13 +------------ tools/internal_ci/macos/grpc_build_artifacts.sh | 12 ------------ 2 files changed, 1 insertion(+), 24 deletions(-) (limited to 'tools/internal_ci') diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc index f7fbec93ff..dd2741e595 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc @@ -31,23 +31,16 @@ ulimit -a pip install google-api-python-client --user python export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json -# required to build protobuf -brew install gflags - set +ex # rvm script is very verbose and exits with errorcode source $HOME/.rvm/scripts/rvm set -e # rvm commands are very verbose -rvm install ruby-2.4 +rvm use ruby-2.4 rvm osx-ssl-certs status all rvm osx-ssl-certs update all set -ex -gem install bundler - # cocoapods export LANG=en_US.UTF-8 -gem install cocoapods -gem install xcpretty pod repo update # needed by python # python @@ -56,10 +49,6 @@ pip install virtualenv --user python pip install -U six tox setuptools --user python export PYTHONPATH=/Library/Python/3.4/site-packages -# python 3.4 -wget -q https://www.python.org/ftp/python/3.4.4/python-3.4.4-macosx10.6.pkg -sudo installer -pkg python-3.4.4-macosx10.6.pkg -target / - # set xcode version for Obj-C tests sudo xcode-select -switch /Applications/Xcode_8.2.1.app/Contents/Developer diff --git a/tools/internal_ci/macos/grpc_build_artifacts.sh b/tools/internal_ci/macos/grpc_build_artifacts.sh index 603c15f210..09784e3bb4 100755 --- a/tools/internal_ci/macos/grpc_build_artifacts.sh +++ b/tools/internal_ci/macos/grpc_build_artifacts.sh @@ -20,25 +20,13 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_macos_rc -# python 3.5 -wget -q https://www.python.org/ftp/python/3.5.2/python-3.5.2-macosx10.6.pkg -sudo installer -pkg python-3.5.2-macosx10.6.pkg -target / - # install cython for all python versions python2.7 -m pip install cython setuptools wheel python3.4 -m pip install cython setuptools wheel python3.5 -m pip install cython setuptools wheel python3.6 -m pip install cython setuptools wheel -# node-gyp (needed for node artifacts) -npm install -g node-gyp - -# php dependencies: pecl -curl -O http://pear.php.net/go-pear.phar -sudo php -d detect_unicode=0 go-pear.phar - # needed to build ruby artifacts -gem install rake-compiler wget https://raw.githubusercontent.com/grpc/grpc/master/tools/distrib/build_ruby_environment_macos.sh bash build_ruby_environment_macos.sh -- cgit v1.2.3 From 5160155005c7d0f1ee9b855dfbe6aa45d8602bf1 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Tue, 12 Sep 2017 09:31:07 -0700 Subject: Upload Kokoro MacOS master results --- tools/internal_ci/macos/grpc_basictests_dbg.cfg | 2 +- tools/internal_ci/macos/grpc_basictests_opt.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'tools/internal_ci') diff --git a/tools/internal_ci/macos/grpc_basictests_dbg.cfg b/tools/internal_ci/macos/grpc_basictests_dbg.cfg index f058f0c7e4..53bda1ff0a 100644 --- a/tools/internal_ci/macos/grpc_basictests_dbg.cfg +++ b/tools/internal_ci/macos/grpc_basictests_dbg.cfg @@ -27,5 +27,5 @@ action { env_vars { key: "RUN_TESTS_FLAGS" - value: "-f basictests macos dbg --internal_ci -j 1 --inner_jobs 4" + value: "-f basictests macos dbg --internal_ci -j 1 --inner_jobs 4 --bq_result_table aggregate_results" } diff --git a/tools/internal_ci/macos/grpc_basictests_opt.cfg b/tools/internal_ci/macos/grpc_basictests_opt.cfg index 5048baaf48..d359eb601a 100644 --- a/tools/internal_ci/macos/grpc_basictests_opt.cfg +++ b/tools/internal_ci/macos/grpc_basictests_opt.cfg @@ -27,5 +27,5 @@ action { env_vars { key: "RUN_TESTS_FLAGS" - value: "-f basictests macos opt --internal_ci -j 1 --inner_jobs 4" + value: "-f basictests macos opt --internal_ci -j 1 --inner_jobs 4 --bq_result_table aggregate_results" } -- cgit v1.2.3 From 27bf05d003f48e6f95b836a5e5450e4bb787f9ef Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Mon, 7 Aug 2017 18:09:11 -0700 Subject: Add tests for c-ares wrapper using a local DNS server. --- CMakeLists.txt | 184 ++++++++++++ Makefile | 188 ++++++++++++ bazel/grpc_build_system.bzl | 16 + .../resolver/dns/c_ares/dns_resolver_ares.c | 2 +- .../naming/resolver_component_tests_defs.include | 92 ++++++ .../resolver_component_tests_runner.sh.template | 4 + templates/tools/dockerfile/apt_get_basic.include | 1 + templates/tools/dockerfile/python_deps.include | 2 +- test/cpp/naming/BUILD | 49 ++++ test/cpp/naming/gen_build_yaml.py | 99 +++++++ .../naming/generate_resolver_component_tests.bzl | 64 ++++ test/cpp/naming/resolver_component_test.cc | 323 +++++++++++++++++++++ test/cpp/naming/resolver_component_tests_runner.sh | 173 +++++++++++ .../resolver_component_tests_runner_invoker.cc | 189 ++++++++++++ test/cpp/naming/resolver_test_record_groups.yaml | 155 ++++++++++ test/cpp/naming/test_dns_server.py | 134 +++++++++ tools/buildgen/generate_build_additions.sh | 1 + .../interoptest/grpc_interop_csharp/Dockerfile | 3 +- .../interoptest/grpc_interop_cxx/Dockerfile | 3 +- .../interoptest/grpc_interop_go/Dockerfile | 2 +- .../interoptest/grpc_interop_go1.7/Dockerfile | 2 +- .../interoptest/grpc_interop_go1.8/Dockerfile | 2 +- .../interoptest/grpc_interop_http2/Dockerfile | 2 +- .../interoptest/grpc_interop_java/Dockerfile | 2 +- .../grpc_interop_java_oracle8/Dockerfile | 2 +- .../interoptest/grpc_interop_node/Dockerfile | 3 +- .../interoptest/grpc_interop_php/Dockerfile | 1 + .../interoptest/grpc_interop_python/Dockerfile | 3 +- .../interoptest/grpc_interop_ruby/Dockerfile | 3 +- tools/dockerfile/test/csharp_jessie_x64/Dockerfile | 3 +- tools/dockerfile/test/cxx_jessie_x64/Dockerfile | 3 +- tools/dockerfile/test/cxx_jessie_x86/Dockerfile | 3 +- .../dockerfile/test/cxx_ubuntu1404_x64/Dockerfile | 3 +- .../dockerfile/test/cxx_ubuntu1604_x64/Dockerfile | 3 +- tools/dockerfile/test/fuzzer/Dockerfile | 3 +- .../test/multilang_jessie_x64/Dockerfile | 3 +- tools/dockerfile/test/node_jessie_x64/Dockerfile | 3 +- tools/dockerfile/test/php7_jessie_x64/Dockerfile | 2 +- tools/dockerfile/test/php_jessie_x64/Dockerfile | 3 +- tools/dockerfile/test/python_jessie_x64/Dockerfile | 3 +- tools/dockerfile/test/python_pyenv_x64/Dockerfile | 3 +- tools/dockerfile/test/ruby_jessie_x64/Dockerfile | 3 +- tools/dockerfile/test/sanity/Dockerfile | 3 +- .../helper_scripts/prepare_build_macos_rc | 2 +- tools/run_tests/generated/sources_and_headers.json | 80 +++++ tools/run_tests/generated/tests.json | 46 +++ 46 files changed, 1845 insertions(+), 28 deletions(-) create mode 100644 templates/test/cpp/naming/resolver_component_tests_defs.include create mode 100644 templates/test/cpp/naming/resolver_component_tests_runner.sh.template create mode 100644 test/cpp/naming/BUILD create mode 100755 test/cpp/naming/gen_build_yaml.py create mode 100755 test/cpp/naming/generate_resolver_component_tests.bzl create mode 100644 test/cpp/naming/resolver_component_test.cc create mode 100755 test/cpp/naming/resolver_component_tests_runner.sh create mode 100644 test/cpp/naming/resolver_component_tests_runner_invoker.cc create mode 100644 test/cpp/naming/resolver_test_record_groups.yaml create mode 100755 test/cpp/naming/test_dns_server.py (limited to 'tools/internal_ci') diff --git a/CMakeLists.txt b/CMakeLists.txt index dd68016be8..1186e2b503 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -760,6 +760,18 @@ add_dependencies(buildtests_cxx thread_stress_test) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx writes_per_rpc_test) endif() +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) +add_dependencies(buildtests_cxx resolver_component_test_unsecure) +endif() +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) +add_dependencies(buildtests_cxx resolver_component_test) +endif() +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) +add_dependencies(buildtests_cxx resolver_component_tests_runner_invoker_unsecure) +endif() +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) +add_dependencies(buildtests_cxx resolver_component_tests_runner_invoker) +endif() add_custom_target(buildtests DEPENDS buildtests_c buildtests_cxx) @@ -14113,6 +14125,178 @@ target_link_libraries(inproc_nosec_test gpr ) +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) + +add_executable(resolver_component_test_unsecure + test/cpp/naming/resolver_component_test.cc + third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc +) + + +target_include_directories(resolver_component_test_unsecure + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CARES_INCLUDE_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include + PRIVATE third_party/googletest/googletest/include + PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(resolver_component_test_unsecure + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc++_test_util_unsecure + grpc_test_util_unsecure + gpr_test_util + grpc++_unsecure + grpc_unsecure + gpr + grpc++_test_config + ${_gRPC_GFLAGS_LIBRARIES} +) + +endif() +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) + +add_executable(resolver_component_test + test/cpp/naming/resolver_component_test.cc + third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc +) + + +target_include_directories(resolver_component_test + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CARES_INCLUDE_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include + PRIVATE third_party/googletest/googletest/include + PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(resolver_component_test + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc++_test_util + grpc_test_util + gpr_test_util + grpc++ + grpc + gpr + grpc++_test_config + ${_gRPC_GFLAGS_LIBRARIES} +) + +endif() +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) + +add_executable(resolver_component_tests_runner_invoker_unsecure + test/cpp/naming/resolver_component_tests_runner_invoker.cc + third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc +) + + +target_include_directories(resolver_component_tests_runner_invoker_unsecure + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CARES_INCLUDE_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include + PRIVATE third_party/googletest/googletest/include + PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(resolver_component_tests_runner_invoker_unsecure + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc++_test_util + grpc_test_util + gpr_test_util + grpc++ + grpc + gpr + grpc++_test_config + ${_gRPC_GFLAGS_LIBRARIES} +) + +endif() +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) + +add_executable(resolver_component_tests_runner_invoker + test/cpp/naming/resolver_component_tests_runner_invoker.cc + third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc +) + + +target_include_directories(resolver_component_tests_runner_invoker + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CARES_INCLUDE_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include + PRIVATE third_party/googletest/googletest/include + PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(resolver_component_tests_runner_invoker + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc++_test_util + grpc_test_util + gpr_test_util + grpc++ + grpc + gpr + grpc++_test_config + ${_gRPC_GFLAGS_LIBRARIES} +) + +endif() endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) diff --git a/Makefile b/Makefile index 0c3648566a..61d6cae6ab 100644 --- a/Makefile +++ b/Makefile @@ -1266,6 +1266,10 @@ h2_sockpair+trace_nosec_test: $(BINDIR)/$(CONFIG)/h2_sockpair+trace_nosec_test h2_sockpair_1byte_nosec_test: $(BINDIR)/$(CONFIG)/h2_sockpair_1byte_nosec_test h2_uds_nosec_test: $(BINDIR)/$(CONFIG)/h2_uds_nosec_test inproc_nosec_test: $(BINDIR)/$(CONFIG)/inproc_nosec_test +resolver_component_test_unsecure: $(BINDIR)/$(CONFIG)/resolver_component_test_unsecure +resolver_component_test: $(BINDIR)/$(CONFIG)/resolver_component_test +resolver_component_tests_runner_invoker_unsecure: $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker_unsecure +resolver_component_tests_runner_invoker: $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker api_fuzzer_one_entry: $(BINDIR)/$(CONFIG)/api_fuzzer_one_entry client_fuzzer_one_entry: $(BINDIR)/$(CONFIG)/client_fuzzer_one_entry hpack_parser_fuzzer_test_one_entry: $(BINDIR)/$(CONFIG)/hpack_parser_fuzzer_test_one_entry @@ -1652,6 +1656,10 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/boringssl_x509_test \ $(BINDIR)/$(CONFIG)/boringssl_tab_test \ $(BINDIR)/$(CONFIG)/boringssl_v3name_test \ + $(BINDIR)/$(CONFIG)/resolver_component_test_unsecure \ + $(BINDIR)/$(CONFIG)/resolver_component_test \ + $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker_unsecure \ + $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker \ else buildtests_cxx: privatelibs_cxx \ @@ -1730,6 +1738,10 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/thread_manager_test \ $(BINDIR)/$(CONFIG)/thread_stress_test \ $(BINDIR)/$(CONFIG)/writes_per_rpc_test \ + $(BINDIR)/$(CONFIG)/resolver_component_test_unsecure \ + $(BINDIR)/$(CONFIG)/resolver_component_test \ + $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker_unsecure \ + $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker \ endif @@ -2141,6 +2153,10 @@ test_cxx: buildtests_cxx $(Q) $(BINDIR)/$(CONFIG)/thread_stress_test || ( echo test thread_stress_test failed ; exit 1 ) $(E) "[RUN] Testing writes_per_rpc_test" $(Q) $(BINDIR)/$(CONFIG)/writes_per_rpc_test || ( echo test writes_per_rpc_test failed ; exit 1 ) + $(E) "[RUN] Testing resolver_component_tests_runner_invoker_unsecure" + $(Q) $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker_unsecure || ( echo test resolver_component_tests_runner_invoker_unsecure failed ; exit 1 ) + $(E) "[RUN] Testing resolver_component_tests_runner_invoker" + $(Q) $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker || ( echo test resolver_component_tests_runner_invoker failed ; exit 1 ) flaky_test_cxx: buildtests_cxx @@ -19480,6 +19496,178 @@ ifneq ($(NO_DEPS),true) endif +RESOLVER_COMPONENT_TEST_UNSECURE_SRC = \ + test/cpp/naming/resolver_component_test.cc \ + +RESOLVER_COMPONENT_TEST_UNSECURE_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(RESOLVER_COMPONENT_TEST_UNSECURE_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/resolver_component_test_unsecure: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. + +$(BINDIR)/$(CONFIG)/resolver_component_test_unsecure: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/resolver_component_test_unsecure: $(PROTOBUF_DEP) $(RESOLVER_COMPONENT_TEST_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(RESOLVER_COMPONENT_TEST_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/resolver_component_test_unsecure + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/cpp/naming/resolver_component_test.o: $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + +deps_resolver_component_test_unsecure: $(RESOLVER_COMPONENT_TEST_UNSECURE_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(RESOLVER_COMPONENT_TEST_UNSECURE_OBJS:.o=.dep) +endif +endif + + +RESOLVER_COMPONENT_TEST_SRC = \ + test/cpp/naming/resolver_component_test.cc \ + +RESOLVER_COMPONENT_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(RESOLVER_COMPONENT_TEST_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/resolver_component_test: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. + +$(BINDIR)/$(CONFIG)/resolver_component_test: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/resolver_component_test: $(PROTOBUF_DEP) $(RESOLVER_COMPONENT_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(RESOLVER_COMPONENT_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/resolver_component_test + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/cpp/naming/resolver_component_test.o: $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + +deps_resolver_component_test: $(RESOLVER_COMPONENT_TEST_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(RESOLVER_COMPONENT_TEST_OBJS:.o=.dep) +endif +endif + + +RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_UNSECURE_SRC = \ + test/cpp/naming/resolver_component_tests_runner_invoker.cc \ + +RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_UNSECURE_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_UNSECURE_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker_unsecure: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. + +$(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker_unsecure: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker_unsecure: $(PROTOBUF_DEP) $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker_unsecure + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/cpp/naming/resolver_component_tests_runner_invoker.o: $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + +deps_resolver_component_tests_runner_invoker_unsecure: $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_UNSECURE_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_UNSECURE_OBJS:.o=.dep) +endif +endif + + +RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_SRC = \ + test/cpp/naming/resolver_component_tests_runner_invoker.cc \ + +RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. + +$(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker: $(PROTOBUF_DEP) $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/cpp/naming/resolver_component_tests_runner_invoker.o: $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + +deps_resolver_component_tests_runner_invoker: $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_OBJS:.o=.dep) +endif +endif + + API_FUZZER_ONE_ENTRY_SRC = \ test/core/end2end/fuzzers/api_fuzzer.c \ test/core/util/one_corpus_entry_fuzzer.c \ diff --git a/bazel/grpc_build_system.bzl b/bazel/grpc_build_system.bzl index ba7ffcc9be..6cfed6b0e7 100644 --- a/bazel/grpc_build_system.bzl +++ b/bazel/grpc_build_system.bzl @@ -106,6 +106,22 @@ def grpc_sh_test(name, srcs, args = [], data = []): args = args, data = data) +def grpc_sh_binary(name, srcs, data = []): + native.sh_test( + name = name, + srcs = srcs, + data = data) + +def grpc_py_binary(name, srcs, data = [], deps = []): + if name == "test_dns_server": + # TODO: allow running test_dns_server in oss bazel test suite + deps = [] + native.py_binary( + name = name, + srcs = srcs, + data = data, + deps = deps) + def grpc_package(name, visibility = "private", features = []): if visibility == "tests": visibility = ["//test:__subpackages__"] diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c index b87a3b7082..371c59b8cf 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c @@ -204,7 +204,7 @@ static char *choose_service_config(char *service_config_choice_json) { int random_pct = rand() % 100; int percentage; if (sscanf(field->value, "%d", &percentage) != 1 || - random_pct > percentage) { + random_pct > percentage || percentage == 0) { service_config_json = NULL; break; } diff --git a/templates/test/cpp/naming/resolver_component_tests_defs.include b/templates/test/cpp/naming/resolver_component_tests_defs.include new file mode 100644 index 0000000000..6fa91c741a --- /dev/null +++ b/templates/test/cpp/naming/resolver_component_tests_defs.include @@ -0,0 +1,92 @@ +<%def name="resolver_component_tests(tests)">#!/bin/bash +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is auto-generated + +set -ex + +# all command args required in this set order +FLAGS_test_bin_path=`echo "$1" | grep '\--test_bin_path=' | cut -d "=" -f 2` +FLAGS_dns_server_bin_path=`echo "$2" | grep '\--dns_server_bin_path=' | cut -d "=" -f 2` +FLAGS_records_config_path=`echo "$3" | grep '\--records_config_path=' | cut -d "=" -f 2` +FLAGS_test_dns_server_port=`echo "$4" | grep '\--test_dns_server_port=' | cut -d "=" -f 2` + +for cmd_arg in "$FLAGS_test_bin_path" "$FLAGS_dns_server_bin_path" "$FLAGS_records_config_path" "$FLAGS_test_dns_server_port"; do + if [[ "$cmd_arg" == "" ]]; then + echo "Missing a CMD arg" && exit 1 + fi +done + +if [[ "$GRPC_DNS_RESOLVER" != "" && "$GRPC_DNS_RESOLVER" != ares ]]; then + echo "This test only works under GRPC_DNS_RESOLVER=ares. Have GRPC_DNS_RESOLVER=$GRPC_DNS_RESOLVER" && exit 1 +fi +export GRPC_DNS_RESOLVER=ares + +"$FLAGS_dns_server_bin_path" --records_config_path="$FLAGS_records_config_path" --port="$FLAGS_test_dns_server_port" 2>&1 > /dev/null & +DNS_SERVER_PID=$! +echo "Local DNS server started. PID: $DNS_SERVER_PID" + +# Health check local DNS server TCP and UDP ports +for ((i=0;i<30;i++)); +do + echo "Retry health-check DNS query to local DNS server over tcp and udp" + RETRY=0 + dig A health-check-local-dns-server-is-alive.resolver-tests.grpctestingexp. @localhost -p "$FLAGS_test_dns_server_port" +tries=1 +timeout=1 | grep '123.123.123.123' || RETRY=1 + dig A health-check-local-dns-server-is-alive.resolver-tests.grpctestingexp. @localhost -p "$FLAGS_test_dns_server_port" +tries=1 +timeout=1 +tcp | grep '123.123.123.123' || RETRY=1 + if [[ "$RETRY" == 0 ]]; then + break + fi; + sleep 0.1 +done + +if [[ $RETRY == 1 ]]; then + echo "FAILED TO START LOCAL DNS SERVER" + kill -SIGTERM $DNS_SERVER_PID + wait + exit 1 +fi + +function terminate_all { + echo "Received signal. Terminating $! and $DNS_SERVER_PID" + kill -SIGTERM $! || true + kill -SIGTERM $DNS_SERVER_PID || true + wait + exit 1 +} + +trap terminate_all SIGTERM SIGINT + +EXIT_CODE=0 +# TODO: this test should check for GCE residency and skip tests using _grpclb._tcp.* SRV records once GCE residency checks are made +# in the resolver. + +% for test in tests: +$FLAGS_test_bin_path \\ + + --target_name='${test['target_name']}' \\ + + --expected_addrs='${test['expected_addrs']}' \\ + + --expected_chosen_service_config='${test['expected_chosen_service_config']}' \\ + + --expected_lb_policy='${test['expected_lb_policy']}' \\ + + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +% endfor +kill -SIGTERM $DNS_SERVER_PID || true +wait +exit $EXIT_CODE diff --git a/templates/test/cpp/naming/resolver_component_tests_runner.sh.template b/templates/test/cpp/naming/resolver_component_tests_runner.sh.template new file mode 100644 index 0000000000..86772dd141 --- /dev/null +++ b/templates/test/cpp/naming/resolver_component_tests_runner.sh.template @@ -0,0 +1,4 @@ +%YAML 1.2 +--- | + <%namespace file="resolver_component_tests_defs.include" import="*"/>\ + ${resolver_component_tests(resolver_component_test_cases)} diff --git a/templates/tools/dockerfile/apt_get_basic.include b/templates/tools/dockerfile/apt_get_basic.include index 9237e7dace..6e19e65157 100644 --- a/templates/tools/dockerfile/apt_get_basic.include +++ b/templates/tools/dockerfile/apt_get_basic.include @@ -6,6 +6,7 @@ RUN apt-get update && apt-get install -y ${'\\'} bzip2 ${'\\'} ccache ${'\\'} curl ${'\\'} + dnsutils ${'\\'} gcc ${'\\'} gcc-multilib ${'\\'} git ${'\\'} diff --git a/templates/tools/dockerfile/python_deps.include b/templates/tools/dockerfile/python_deps.include index 2c12981418..94b854ad21 100644 --- a/templates/tools/dockerfile/python_deps.include +++ b/templates/tools/dockerfile/python_deps.include @@ -11,4 +11,4 @@ RUN apt-get update && apt-get install -y ${'\\'} # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 diff --git a/test/cpp/naming/BUILD b/test/cpp/naming/BUILD new file mode 100644 index 0000000000..24c3d1a443 --- /dev/null +++ b/test/cpp/naming/BUILD @@ -0,0 +1,49 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package( + default_visibility = ["//visibility:public"], + features = [ + "-layering_check", + "-parse_headers", + ], +) + +licenses(["notice"]) # Apache v2 + +load("//bazel:grpc_build_system.bzl", "grpc_sh_binary", "grpc_py_binary") + +load(":generate_resolver_component_tests.bzl", "generate_resolver_component_tests") + +# Meant to be invoked only through the top-level shell script driver. +grpc_sh_binary( + name = "resolver_component_tests_runner", + srcs = [ + "resolver_component_tests_runner.sh", + ], +) + +grpc_py_binary( + name = "test_dns_server", + srcs = ["test_dns_server.py"], + data = [ + "resolver_test_record_groups.yaml", + ], + deps = [ + "twisted", + "yaml", + ] +) + +generate_resolver_component_tests() diff --git a/test/cpp/naming/gen_build_yaml.py b/test/cpp/naming/gen_build_yaml.py new file mode 100755 index 0000000000..3a51fef7a0 --- /dev/null +++ b/test/cpp/naming/gen_build_yaml.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python2.7 +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Generates the appropriate build.json data for all the naming tests.""" + + +import yaml +import collections +import hashlib +import json + +_LOCAL_DNS_SERVER_ADDRESS = '127.0.0.1:15353' + +def _append_zone_name(name, zone_name): + return '%s.%s' % (name, zone_name) + +def _build_expected_addrs_cmd_arg(expected_addrs): + out = [] + for addr in expected_addrs: + out.append('%s,%s' % (addr['address'], str(addr['is_balancer']))) + return ';'.join(out) + +def main(): + resolver_component_data = '' + with open('test/cpp/naming/resolver_test_record_groups.yaml') as f: + resolver_component_data = yaml.load(f) + + json = { + 'resolver_component_test_cases': [ + { + 'target_name': _append_zone_name(test_case['record_to_resolve'], + resolver_component_data['resolver_component_tests_common_zone_name']), + 'expected_addrs': _build_expected_addrs_cmd_arg(test_case['expected_addrs']), + 'expected_chosen_service_config': (test_case['expected_chosen_service_config'] or ''), + 'expected_lb_policy': (test_case['expected_lb_policy'] or ''), + } for test_case in resolver_component_data['resolver_component_tests'] + ], + 'targets': [ + { + 'name': 'resolver_component_test' + unsecure_build_config_suffix, + 'build': 'test', + 'language': 'c++', + 'gtest': False, + 'run': False, + 'src': ['test/cpp/naming/resolver_component_test.cc'], + 'platforms': ['linux', 'posix', 'mac'], + 'deps': [ + 'grpc++_test_util' + unsecure_build_config_suffix, + 'grpc_test_util' + unsecure_build_config_suffix, + 'gpr_test_util', + 'grpc++' + unsecure_build_config_suffix, + 'grpc' + unsecure_build_config_suffix, + 'gpr', + 'grpc++_test_config', + ], + } for unsecure_build_config_suffix in ['_unsecure', ''] + ] + [ + { + 'name': 'resolver_component_tests_runner_invoker' + unsecure_build_config_suffix, + 'build': 'test', + 'language': 'c++', + 'gtest': False, + 'run': True, + 'src': ['test/cpp/naming/resolver_component_tests_runner_invoker.cc'], + 'platforms': ['linux', 'posix', 'mac'], + 'deps': [ + 'grpc++_test_util', + 'grpc_test_util', + 'gpr_test_util', + 'grpc++', + 'grpc', + 'gpr', + 'grpc++_test_config', + ], + 'args': [ + '--test_bin_name=resolver_component_test%s' % unsecure_build_config_suffix, + '--running_under_bazel=false', + ], + } for unsecure_build_config_suffix in ['_unsecure', ''] + ] + } + + print(yaml.dump(json)) + +if __name__ == '__main__': + main() diff --git a/test/cpp/naming/generate_resolver_component_tests.bzl b/test/cpp/naming/generate_resolver_component_tests.bzl new file mode 100755 index 0000000000..118d9452d9 --- /dev/null +++ b/test/cpp/naming/generate_resolver_component_tests.bzl @@ -0,0 +1,64 @@ +#!/usr/bin/env python2.7 +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("//bazel:grpc_build_system.bzl", "grpc_sh_binary", "grpc_cc_test", "grpc_cc_binary") + +def generate_resolver_component_tests(): + for unsecure_build_config_suffix in ['_unsecure', '']: + # meant to be invoked only through the top-level shell script driver + grpc_cc_binary( + name = "resolver_component_test%s" % unsecure_build_config_suffix, + testonly = 1, + srcs = [ + "resolver_component_test.cc", + ], + external_deps = [ + "gmock", + ], + deps = [ + "//test/cpp/util:test_util%s" % unsecure_build_config_suffix, + "//test/core/util:grpc_test_util%s" % unsecure_build_config_suffix, + "//test/core/util:gpr_test_util", + "//:grpc++%s" % unsecure_build_config_suffix, + "//:grpc%s" % unsecure_build_config_suffix, + "//:gpr", + "//test/cpp/util:test_config", + ], + ) + grpc_cc_test( + name = "resolver_component_tests_runner_invoker%s" % unsecure_build_config_suffix, + srcs = [ + "resolver_component_tests_runner_invoker.cc", + ], + deps = [ + "//test/cpp/util:test_util", + "//test/core/util:grpc_test_util", + "//test/core/util:gpr_test_util", + "//:grpc++", + "//:grpc", + "//:gpr", + "//test/cpp/util:test_config", + ], + data = [ + ":resolver_component_tests_runner", + ":resolver_component_test%s" % unsecure_build_config_suffix, + ":test_dns_server", + "resolver_test_record_groups.yaml", # include the transitive dependency so that the dns sever py binary can locate this + ], + args = [ + "--test_bin_name=resolver_component_test%s" % unsecure_build_config_suffix, + "--running_under_bazel=true", + ] + ) diff --git a/test/cpp/naming/resolver_component_test.cc b/test/cpp/naming/resolver_component_test.cc new file mode 100644 index 0000000000..0857fb6a32 --- /dev/null +++ b/test/cpp/naming/resolver_component_test.cc @@ -0,0 +1,323 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "test/cpp/util/subprocess.h" +#include "test/cpp/util/test_config.h" + +extern "C" { +#include "src/core/ext/filters/client_channel/client_channel.h" +#include "src/core/ext/filters/client_channel/resolver.h" +#include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h" +#include "src/core/ext/filters/client_channel/resolver_registry.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/iomgr/combiner.h" +#include "src/core/lib/iomgr/executor.h" +#include "src/core/lib/iomgr/iomgr.h" +#include "src/core/lib/iomgr/resolve_address.h" +#include "src/core/lib/iomgr/sockaddr_utils.h" +#include "src/core/lib/support/env.h" +#include "src/core/lib/support/string.h" +#include "test/core/util/port.h" +#include "test/core/util/test_config.h" +} + +using std::vector; +using grpc::SubProcess; +using testing::UnorderedElementsAreArray; + +// Hack copied from "test/cpp/end2end/server_crash_test_client.cc"! +// In some distros, gflags is in the namespace google, and in some others, +// in gflags. This hack is enabling us to find both. +namespace google {} +namespace gflags {} +using namespace google; +using namespace gflags; + +DEFINE_string(target_name, "", "Target name to resolve."); +DEFINE_string(expected_addrs, "", + "Comma-separated list of expected " + "',;,;...' " + "addresses of " + "backend and/or balancers. 'is_balancer' should be bool, i.e. " + "true or false."); +DEFINE_string(expected_chosen_service_config, "", + "Expected service config json string that gets chosen (no " + "whitespace). Empty for none."); +DEFINE_string( + local_dns_server_address, "", + "Optional. This address is placed as the uri authority if present."); +DEFINE_string(expected_lb_policy, "", + "Expected lb policy name that appears in resolver result channel " + "arg. Empty for none."); + +namespace { + +class GrpcLBAddress final { + public: + GrpcLBAddress(std::string address, bool is_balancer) + : is_balancer(is_balancer), address(address) {} + + bool operator==(const GrpcLBAddress &other) const { + return this->is_balancer == other.is_balancer && + this->address == other.address; + } + + bool operator!=(const GrpcLBAddress &other) const { + return !(*this == other); + } + + bool is_balancer; + std::string address; +}; + +vector ParseExpectedAddrs(std::string expected_addrs) { + std::vector out; + while (expected_addrs.size() != 0) { + // get the next , (v4 or v6) + size_t next_comma = expected_addrs.find(","); + if (next_comma == std::string::npos) { + gpr_log( + GPR_ERROR, + "Missing ','. Expected_addrs arg should be a semi-colon-separated " + "list of " + ", pairs. Left-to-be-parsed arg is |%s|", + expected_addrs.c_str()); + abort(); + } + std::string next_addr = expected_addrs.substr(0, next_comma); + expected_addrs = expected_addrs.substr(next_comma + 1, std::string::npos); + // get the next is_balancer 'bool' associated with this address + size_t next_semicolon = expected_addrs.find(";"); + bool is_balancer = + gpr_is_true(expected_addrs.substr(0, next_semicolon).c_str()); + out.emplace_back(GrpcLBAddress(next_addr, is_balancer)); + if (next_semicolon == std::string::npos) { + break; + } + expected_addrs = + expected_addrs.substr(next_semicolon + 1, std::string::npos); + } + if (out.size() == 0) { + gpr_log(GPR_ERROR, + "expected_addrs arg should be a comma-separated list of " + ", pairs"); + abort(); + } + return out; +} + +gpr_timespec TestDeadline(void) { + return grpc_timeout_seconds_to_deadline(100); +} + +struct ArgsStruct { + gpr_event ev; + gpr_atm done_atm; + gpr_mu *mu; + grpc_pollset *pollset; + grpc_pollset_set *pollset_set; + grpc_combiner *lock; + grpc_channel_args *channel_args; + vector expected_addrs; + std::string expected_service_config_string; + std::string expected_lb_policy; +}; + +void ArgsInit(grpc_exec_ctx *exec_ctx, ArgsStruct *args) { + gpr_event_init(&args->ev); + args->pollset = (grpc_pollset *)gpr_zalloc(grpc_pollset_size()); + grpc_pollset_init(args->pollset, &args->mu); + args->pollset_set = grpc_pollset_set_create(); + grpc_pollset_set_add_pollset(exec_ctx, args->pollset_set, args->pollset); + args->lock = grpc_combiner_create(); + gpr_atm_rel_store(&args->done_atm, 0); + args->channel_args = NULL; +} + +void DoNothing(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {} + +void ArgsFinish(grpc_exec_ctx *exec_ctx, ArgsStruct *args) { + GPR_ASSERT(gpr_event_wait(&args->ev, TestDeadline())); + grpc_pollset_set_del_pollset(exec_ctx, args->pollset_set, args->pollset); + grpc_pollset_set_destroy(exec_ctx, args->pollset_set); + grpc_closure DoNothing_cb; + GRPC_CLOSURE_INIT(&DoNothing_cb, DoNothing, NULL, grpc_schedule_on_exec_ctx); + grpc_pollset_shutdown(exec_ctx, args->pollset, &DoNothing_cb); + // exec_ctx needs to be flushed before calling grpc_pollset_destroy() + grpc_channel_args_destroy(exec_ctx, args->channel_args); + grpc_exec_ctx_flush(exec_ctx); + grpc_pollset_destroy(exec_ctx, args->pollset); + gpr_free(args->pollset); + GRPC_COMBINER_UNREF(exec_ctx, args->lock, NULL); +} + +gpr_timespec NSecondDeadline(int seconds) { + return gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), + gpr_time_from_seconds(seconds, GPR_TIMESPAN)); +} + +void PollPollsetUntilRequestDone(ArgsStruct *args) { + gpr_timespec deadline = NSecondDeadline(10); + while (true) { + bool done = gpr_atm_acq_load(&args->done_atm) != 0; + if (done) { + break; + } + gpr_timespec time_left = + gpr_time_sub(deadline, gpr_now(GPR_CLOCK_REALTIME)); + gpr_log(GPR_DEBUG, "done=%d, time_left=%" PRId64 ".%09d", done, + time_left.tv_sec, time_left.tv_nsec); + GPR_ASSERT(gpr_time_cmp(time_left, gpr_time_0(GPR_TIMESPAN)) >= 0); + grpc_pollset_worker *worker = NULL; + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + gpr_mu_lock(args->mu); + GRPC_LOG_IF_ERROR( + "pollset_work", + grpc_pollset_work(&exec_ctx, args->pollset, &worker, + gpr_now(GPR_CLOCK_REALTIME), NSecondDeadline(1))); + gpr_mu_unlock(args->mu); + grpc_exec_ctx_finish(&exec_ctx); + } + gpr_event_set(&args->ev, (void *)1); +} + +void CheckServiceConfigResultLocked(grpc_channel_args *channel_args, + ArgsStruct *args) { + const grpc_arg *service_config_arg = + grpc_channel_args_find(channel_args, GRPC_ARG_SERVICE_CONFIG); + if (args->expected_service_config_string != "") { + GPR_ASSERT(service_config_arg != NULL); + GPR_ASSERT(service_config_arg->type == GRPC_ARG_STRING); + EXPECT_EQ(service_config_arg->value.string, + args->expected_service_config_string); + } else { + GPR_ASSERT(service_config_arg == NULL); + } +} + +void CheckLBPolicyResultLocked(grpc_channel_args *channel_args, + ArgsStruct *args) { + const grpc_arg *lb_policy_arg = + grpc_channel_args_find(channel_args, GRPC_ARG_LB_POLICY_NAME); + if (args->expected_lb_policy != "") { + GPR_ASSERT(lb_policy_arg != NULL); + GPR_ASSERT(lb_policy_arg->type == GRPC_ARG_STRING); + EXPECT_EQ(lb_policy_arg->value.string, args->expected_lb_policy); + } else { + GPR_ASSERT(lb_policy_arg == NULL); + } +} + +void CheckResolverResultLocked(grpc_exec_ctx *exec_ctx, void *argsp, + grpc_error *err) { + ArgsStruct *args = (ArgsStruct *)argsp; + grpc_channel_args *channel_args = args->channel_args; + const grpc_arg *channel_arg = + grpc_channel_args_find(channel_args, GRPC_ARG_LB_ADDRESSES); + GPR_ASSERT(channel_arg != NULL); + GPR_ASSERT(channel_arg->type == GRPC_ARG_POINTER); + grpc_lb_addresses *addresses = + (grpc_lb_addresses *)channel_arg->value.pointer.p; + gpr_log(GPR_INFO, "num addrs found: %" PRIdPTR ". expected %" PRIdPTR, + addresses->num_addresses, args->expected_addrs.size()); + GPR_ASSERT(addresses->num_addresses == args->expected_addrs.size()); + std::vector found_lb_addrs; + for (size_t i = 0; i < addresses->num_addresses; i++) { + grpc_lb_address addr = addresses->addresses[i]; + char *str; + grpc_sockaddr_to_string(&str, &addr.address, 1 /* normalize */); + gpr_log(GPR_INFO, "%s", str); + found_lb_addrs.emplace_back( + GrpcLBAddress(std::string(str), addr.is_balancer)); + gpr_free(str); + } + if (args->expected_addrs.size() != found_lb_addrs.size()) { + gpr_log(GPR_DEBUG, "found lb addrs size is: %" PRIdPTR + ". expected addrs size is %" PRIdPTR, + found_lb_addrs.size(), args->expected_addrs.size()); + abort(); + } + EXPECT_THAT(args->expected_addrs, UnorderedElementsAreArray(found_lb_addrs)); + CheckServiceConfigResultLocked(channel_args, args); + CheckLBPolicyResultLocked(channel_args, args); + gpr_atm_rel_store(&args->done_atm, 1); + gpr_mu_lock(args->mu); + GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(args->pollset, NULL)); + gpr_mu_unlock(args->mu); +} + +TEST(ResolverComponentTest, TestResolvesRelevantRecords) { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + ArgsStruct args; + ArgsInit(&exec_ctx, &args); + args.expected_addrs = ParseExpectedAddrs(FLAGS_expected_addrs); + args.expected_service_config_string = FLAGS_expected_chosen_service_config; + args.expected_lb_policy = FLAGS_expected_lb_policy; + // maybe build the address with an authority + char *whole_uri = NULL; + GPR_ASSERT(asprintf(&whole_uri, "dns://%s/%s", + FLAGS_local_dns_server_address.c_str(), + FLAGS_target_name.c_str())); + // create resolver and resolve + grpc_resolver *resolver = grpc_resolver_create(&exec_ctx, whole_uri, NULL, + args.pollset_set, args.lock); + gpr_free(whole_uri); + grpc_closure on_resolver_result_changed; + GRPC_CLOSURE_INIT(&on_resolver_result_changed, CheckResolverResultLocked, + (void *)&args, grpc_combiner_scheduler(args.lock)); + grpc_resolver_next_locked(&exec_ctx, resolver, &args.channel_args, + &on_resolver_result_changed); + grpc_exec_ctx_flush(&exec_ctx); + PollPollsetUntilRequestDone(&args); + GRPC_RESOLVER_UNREF(&exec_ctx, resolver, NULL); + ArgsFinish(&exec_ctx, &args); + grpc_exec_ctx_finish(&exec_ctx); +} + +} // namespace + +int main(int argc, char **argv) { + grpc_init(); + grpc_test_init(argc, argv); + ::testing::InitGoogleTest(&argc, argv); + ParseCommandLineFlags(&argc, &argv, true); + if (FLAGS_target_name == "") { + gpr_log(GPR_ERROR, "Missing target_name param."); + abort(); + } + if (FLAGS_local_dns_server_address != "") { + gpr_log(GPR_INFO, "Specifying authority in uris to: %s", + FLAGS_local_dns_server_address.c_str()); + } + auto result = RUN_ALL_TESTS(); + grpc_shutdown(); + return result; +} diff --git a/test/cpp/naming/resolver_component_tests_runner.sh b/test/cpp/naming/resolver_component_tests_runner.sh new file mode 100755 index 0000000000..83b03b67a3 --- /dev/null +++ b/test/cpp/naming/resolver_component_tests_runner.sh @@ -0,0 +1,173 @@ +#!/bin/bash +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is auto-generated + +set -ex + +# all command args required in this set order +FLAGS_test_bin_path=`echo "$1" | grep '\--test_bin_path=' | cut -d "=" -f 2` +FLAGS_dns_server_bin_path=`echo "$2" | grep '\--dns_server_bin_path=' | cut -d "=" -f 2` +FLAGS_records_config_path=`echo "$3" | grep '\--records_config_path=' | cut -d "=" -f 2` +FLAGS_test_dns_server_port=`echo "$4" | grep '\--test_dns_server_port=' | cut -d "=" -f 2` + +for cmd_arg in "$FLAGS_test_bin_path" "$FLAGS_dns_server_bin_path" "$FLAGS_records_config_path" "$FLAGS_test_dns_server_port"; do + if [[ "$cmd_arg" == "" ]]; then + echo "Missing a CMD arg" && exit 1 + fi +done + +if [[ "$GRPC_DNS_RESOLVER" != "" && "$GRPC_DNS_RESOLVER" != ares ]]; then + echo "This test only works under GRPC_DNS_RESOLVER=ares. Have GRPC_DNS_RESOLVER=$GRPC_DNS_RESOLVER" && exit 1 +fi +export GRPC_DNS_RESOLVER=ares + +"$FLAGS_dns_server_bin_path" --records_config_path="$FLAGS_records_config_path" --port="$FLAGS_test_dns_server_port" 2>&1 > /dev/null & +DNS_SERVER_PID=$! +echo "Local DNS server started. PID: $DNS_SERVER_PID" + +# Health check local DNS server TCP and UDP ports +for ((i=0;i<30;i++)); +do + echo "Retry health-check DNS query to local DNS server over tcp and udp" + RETRY=0 + dig A health-check-local-dns-server-is-alive.resolver-tests.grpctestingexp. @localhost -p "$FLAGS_test_dns_server_port" +tries=1 +timeout=1 | grep '123.123.123.123' || RETRY=1 + dig A health-check-local-dns-server-is-alive.resolver-tests.grpctestingexp. @localhost -p "$FLAGS_test_dns_server_port" +tries=1 +timeout=1 +tcp | grep '123.123.123.123' || RETRY=1 + if [[ "$RETRY" == 0 ]]; then + break + fi; + sleep 0.1 +done + +if [[ $RETRY == 1 ]]; then + echo "FAILED TO START LOCAL DNS SERVER" + kill -SIGTERM $DNS_SERVER_PID + wait + exit 1 +fi + +function terminate_all { + echo "Received signal. Terminating $! and $DNS_SERVER_PID" + kill -SIGTERM $! || true + kill -SIGTERM $DNS_SERVER_PID || true + wait + exit 1 +} + +trap terminate_all SIGTERM SIGINT + +EXIT_CODE=0 +# TODO: this test should check for GCE residency and skip tests using _grpclb._tcp.* SRV records once GCE residency checks are made +# in the resolver. + +$FLAGS_test_bin_path \ + --target_name='srv-ipv4-single-target.resolver-tests.grpctestingexp.' \ + --expected_addrs='1.2.3.4:1234,True' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='srv-ipv4-multi-target.resolver-tests.grpctestingexp.' \ + --expected_addrs='1.2.3.5:1234,True;1.2.3.6:1234,True;1.2.3.7:1234,True' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='srv-ipv6-single-target.resolver-tests.grpctestingexp.' \ + --expected_addrs='[2607:f8b0:400a:801::1001]:1234,True' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='srv-ipv6-multi-target.resolver-tests.grpctestingexp.' \ + --expected_addrs='[2607:f8b0:400a:801::1002]:1234,True;[2607:f8b0:400a:801::1003]:1234,True;[2607:f8b0:400a:801::1004]:1234,True' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='srv-ipv4-simple-service-config.resolver-tests.grpctestingexp.' \ + --expected_addrs='1.2.3.4:1234,True' \ + --expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]}]}' \ + --expected_lb_policy='round_robin' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='ipv4-no-srv-simple-service-config.resolver-tests.grpctestingexp.' \ + --expected_addrs='1.2.3.4:443,False' \ + --expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"NoSrvSimpleService","waitForReady":true}]}]}' \ + --expected_lb_policy='round_robin' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='ipv4-no-config-for-cpp.resolver-tests.grpctestingexp.' \ + --expected_addrs='1.2.3.4:443,False' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='ipv4-cpp-config-has-zero-percentage.resolver-tests.grpctestingexp.' \ + --expected_addrs='1.2.3.4:443,False' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='ipv4-second-language-is-cpp.resolver-tests.grpctestingexp.' \ + --expected_addrs='1.2.3.4:443,False' \ + --expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"CppService","waitForReady":true}]}]}' \ + --expected_lb_policy='round_robin' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='ipv4-config-with-percentages.resolver-tests.grpctestingexp.' \ + --expected_addrs='1.2.3.4:443,False' \ + --expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"AlwaysPickedService","waitForReady":true}]}]}' \ + --expected_lb_policy='round_robin' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='srv-ipv4-target-has-backend-and-balancer.resolver-tests.grpctestingexp.' \ + --expected_addrs='1.2.3.4:1234,True;1.2.3.4:443,False' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='srv-ipv6-target-has-backend-and-balancer.resolver-tests.grpctestingexp.' \ + --expected_addrs='[2607:f8b0:400a:801::1002]:1234,True;[2607:f8b0:400a:801::1002]:443,False' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +kill -SIGTERM $DNS_SERVER_PID || true +wait +exit $EXIT_CODE diff --git a/test/cpp/naming/resolver_component_tests_runner_invoker.cc b/test/cpp/naming/resolver_component_tests_runner_invoker.cc new file mode 100644 index 0000000000..b14391284d --- /dev/null +++ b/test/cpp/naming/resolver_component_tests_runner_invoker.cc @@ -0,0 +1,189 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "test/cpp/util/subprocess.h" +#include "test/cpp/util/test_config.h" + +extern "C" { +#include "src/core/lib/support/env.h" +#include "test/core/util/port.h" +} + +DEFINE_bool( + running_under_bazel, false, + "True if this test is running under bazel. " + "False indicates that this test is running under run_tests.py. " + "Child process test binaries are located differently based on this flag. "); + +DEFINE_string(test_bin_name, "", + "Name, without the preceding path, of the test binary"); + +DEFINE_string(grpc_test_directory_relative_to_test_srcdir, "/__main__", + "This flag only applies if runner_under_bazel is true. This " + "flag is ignored if runner_under_bazel is false. " + "Directory of the /test directory relative to bazel's " + "TEST_SRCDIR environment variable"); + +using grpc::SubProcess; + +static volatile sig_atomic_t abort_wait_for_child = 0; + +static void sighandler(int sig) { abort_wait_for_child = 1; } + +static void register_sighandler() { + struct sigaction act; + memset(&act, 0, sizeof(act)); + act.sa_handler = sighandler; + sigaction(SIGINT, &act, NULL); + sigaction(SIGTERM, &act, NULL); +} + +namespace { + +const int kTestTimeoutSeconds = 60 * 2; + +void RunSigHandlingThread(SubProcess *test_driver, gpr_mu *test_driver_mu, + gpr_cv *test_driver_cv, int *test_driver_done) { + gpr_timespec overall_deadline = + gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), + gpr_time_from_seconds(kTestTimeoutSeconds, GPR_TIMESPAN)); + while (true) { + gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); + if (gpr_time_cmp(now, overall_deadline) > 0 || abort_wait_for_child) break; + gpr_mu_lock(test_driver_mu); + if (*test_driver_done) { + gpr_mu_unlock(test_driver_mu); + return; + } + gpr_timespec wait_deadline = gpr_time_add( + gpr_now(GPR_CLOCK_MONOTONIC), gpr_time_from_seconds(1, GPR_TIMESPAN)); + gpr_cv_wait(test_driver_cv, test_driver_mu, wait_deadline); + gpr_mu_unlock(test_driver_mu); + } + gpr_log(GPR_DEBUG, + "Test timeout reached or received signal. Interrupting test driver " + "child process."); + test_driver->Interrupt(); + return; +} +} + +namespace grpc { + +namespace testing { + +void InvokeResolverComponentTestsRunner(std::string test_runner_bin_path, + std::string test_bin_path, + std::string dns_server_bin_path, + std::string records_config_path) { + int test_dns_server_port = grpc_pick_unused_port_or_die(); + + SubProcess *test_driver = new SubProcess( + {test_runner_bin_path, "--test_bin_path=" + test_bin_path, + "--dns_server_bin_path=" + dns_server_bin_path, + "--records_config_path=" + records_config_path, + "--test_dns_server_port=" + std::to_string(test_dns_server_port)}); + gpr_mu test_driver_mu; + gpr_mu_init(&test_driver_mu); + gpr_cv test_driver_cv; + gpr_cv_init(&test_driver_cv); + int test_driver_done = 0; + register_sighandler(); + std::thread sig_handling_thread(RunSigHandlingThread, test_driver, + &test_driver_mu, &test_driver_cv, + &test_driver_done); + int status = test_driver->Join(); + if (WIFEXITED(status)) { + if (WEXITSTATUS(status)) { + gpr_log(GPR_INFO, + "Resolver component test test-runner exited with code %d", + WEXITSTATUS(status)); + abort(); + } + } else if (WIFSIGNALED(status)) { + gpr_log(GPR_INFO, + "Resolver component test test-runner ended from signal %d", + WTERMSIG(status)); + abort(); + } else { + gpr_log(GPR_INFO, + "Resolver component test test-runner ended with unknown status %d", + status); + abort(); + } + gpr_mu_lock(&test_driver_mu); + test_driver_done = 1; + gpr_cv_signal(&test_driver_cv); + gpr_mu_unlock(&test_driver_mu); + sig_handling_thread.join(); + delete test_driver; + gpr_mu_destroy(&test_driver_mu); + gpr_cv_destroy(&test_driver_cv); +} + +} // namespace testing + +} // namespace grpc + +int main(int argc, char **argv) { + grpc::testing::InitTest(&argc, &argv, true); + grpc_init(); + GPR_ASSERT(FLAGS_test_bin_name != ""); + std::string my_bin = argv[0]; + if (FLAGS_running_under_bazel) { + GPR_ASSERT(FLAGS_grpc_test_directory_relative_to_test_srcdir != ""); + // Use bazel's TEST_SRCDIR environment variable to locate the "test data" + // binaries. + std::string const bin_dir = + gpr_getenv("TEST_SRCDIR") + + FLAGS_grpc_test_directory_relative_to_test_srcdir + + std::string("/test/cpp/naming"); + // Invoke bazel's executeable links to the .sh and .py scripts (don't use + // the .sh and .py suffixes) to make + // sure that we're using bazel's test environment. + grpc::testing::InvokeResolverComponentTestsRunner( + bin_dir + "/resolver_component_tests_runner", + bin_dir + "/" + FLAGS_test_bin_name, bin_dir + "/test_dns_server", + bin_dir + "/resolver_test_record_groups.yaml"); + } else { + // Get the current binary's directory relative to repo root to invoke the + // correct build config (asan/tsan/dbg, etc.). + std::string const bin_dir = my_bin.substr(0, my_bin.rfind('/')); + // Invoke the .sh and .py scripts directly where they are in source code. + grpc::testing::InvokeResolverComponentTestsRunner( + "test/cpp/naming/resolver_component_tests_runner.sh", + bin_dir + "/" + FLAGS_test_bin_name, + "test/cpp/naming/test_dns_server.py", + "test/cpp/naming/resolver_test_record_groups.yaml"); + } + grpc_shutdown(); + return 0; +} diff --git a/test/cpp/naming/resolver_test_record_groups.yaml b/test/cpp/naming/resolver_test_record_groups.yaml new file mode 100644 index 0000000000..67c611d831 --- /dev/null +++ b/test/cpp/naming/resolver_test_record_groups.yaml @@ -0,0 +1,155 @@ +resolver_component_tests_common_zone_name: resolver-tests.grpctestingexp. +resolver_component_tests: +- expected_addrs: + - {address: '1.2.3.4:1234', is_balancer: true} + expected_chosen_service_config: null + expected_lb_policy: null + record_to_resolve: srv-ipv4-single-target + records: + _grpclb._tcp.srv-ipv4-single-target: + - {TTL: '2100', data: 0 0 1234 ipv4-single-target, type: SRV} + ipv4-single-target: + - {TTL: '2100', data: 1.2.3.4, type: A} +- expected_addrs: + - {address: '1.2.3.5:1234', is_balancer: true} + - {address: '1.2.3.6:1234', is_balancer: true} + - {address: '1.2.3.7:1234', is_balancer: true} + expected_chosen_service_config: null + expected_lb_policy: null + record_to_resolve: srv-ipv4-multi-target + records: + _grpclb._tcp.srv-ipv4-multi-target: + - {TTL: '2100', data: 0 0 1234 ipv4-multi-target, type: SRV} + ipv4-multi-target: + - {TTL: '2100', data: 1.2.3.5, type: A} + - {TTL: '2100', data: 1.2.3.6, type: A} + - {TTL: '2100', data: 1.2.3.7, type: A} +- expected_addrs: + - {address: '[2607:f8b0:400a:801::1001]:1234', is_balancer: true} + expected_chosen_service_config: null + expected_lb_policy: null + record_to_resolve: srv-ipv6-single-target + records: + _grpclb._tcp.srv-ipv6-single-target: + - {TTL: '2100', data: 0 0 1234 ipv6-single-target, type: SRV} + ipv6-single-target: + - {TTL: '2100', data: '2607:f8b0:400a:801::1001', type: AAAA} +- expected_addrs: + - {address: '[2607:f8b0:400a:801::1002]:1234', is_balancer: true} + - {address: '[2607:f8b0:400a:801::1003]:1234', is_balancer: true} + - {address: '[2607:f8b0:400a:801::1004]:1234', is_balancer: true} + expected_chosen_service_config: null + expected_lb_policy: null + record_to_resolve: srv-ipv6-multi-target + records: + _grpclb._tcp.srv-ipv6-multi-target: + - {TTL: '2100', data: 0 0 1234 ipv6-multi-target, type: SRV} + ipv6-multi-target: + - {TTL: '2100', data: '2607:f8b0:400a:801::1002', type: AAAA} + - {TTL: '2100', data: '2607:f8b0:400a:801::1003', type: AAAA} + - {TTL: '2100', data: '2607:f8b0:400a:801::1004', type: AAAA} +- expected_addrs: + - {address: '1.2.3.4:1234', is_balancer: true} + expected_chosen_service_config: '{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]}]}' + expected_lb_policy: round_robin + record_to_resolve: srv-ipv4-simple-service-config + records: + _grpclb._tcp.srv-ipv4-simple-service-config: + - {TTL: '2100', data: 0 0 1234 ipv4-simple-service-config, type: SRV} + ipv4-simple-service-config: + - {TTL: '2100', data: 1.2.3.4, type: A} + srv-ipv4-simple-service-config: + - {TTL: '2100', data: 'grpc_config=[{"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]}]}}]', + type: TXT} +- expected_addrs: + - {address: '1.2.3.4:443', is_balancer: false} + expected_chosen_service_config: '{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"NoSrvSimpleService","waitForReady":true}]}]}' + expected_lb_policy: round_robin + record_to_resolve: ipv4-no-srv-simple-service-config + records: + ipv4-no-srv-simple-service-config: + - {TTL: '2100', data: 1.2.3.4, type: A} + - {TTL: '2100', data: 'grpc_config=[{"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"NoSrvSimpleService","waitForReady":true}]}]}}]', + type: TXT} +- expected_addrs: + - {address: '1.2.3.4:443', is_balancer: false} + expected_chosen_service_config: null + expected_lb_policy: null + record_to_resolve: ipv4-no-config-for-cpp + records: + ipv4-no-config-for-cpp: + - {TTL: '2100', data: 1.2.3.4, type: A} + - {TTL: '2100', data: 'grpc_config=[{"clientLanguage":["python"],"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"PythonService","waitForReady":true}]}]}}]', + type: TXT} +- expected_addrs: + - {address: '1.2.3.4:443', is_balancer: false} + expected_chosen_service_config: null + expected_lb_policy: null + record_to_resolve: ipv4-cpp-config-has-zero-percentage + records: + ipv4-cpp-config-has-zero-percentage: + - {TTL: '2100', data: 1.2.3.4, type: A} + - {TTL: '2100', data: 'grpc_config=[{"percentage":0,"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"CppService","waitForReady":true}]}]}}]', + type: TXT} +- expected_addrs: + - {address: '1.2.3.4:443', is_balancer: false} + expected_chosen_service_config: '{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"CppService","waitForReady":true}]}]}' + expected_lb_policy: round_robin + record_to_resolve: ipv4-second-language-is-cpp + records: + ipv4-second-language-is-cpp: + - {TTL: '2100', data: 1.2.3.4, type: A} + - {TTL: '2100', data: 'grpc_config=[{"clientLanguage":["go"],"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"GoService","waitForReady":true}]}]}},{"clientLanguage":["c++"],"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"CppService","waitForReady":true}]}]}}]', + type: TXT} +- expected_addrs: + - {address: '1.2.3.4:443', is_balancer: false} + expected_chosen_service_config: '{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"AlwaysPickedService","waitForReady":true}]}]}' + expected_lb_policy: round_robin + record_to_resolve: ipv4-config-with-percentages + records: + ipv4-config-with-percentages: + - {TTL: '2100', data: 1.2.3.4, type: A} + - {TTL: '2100', data: 'grpc_config=[{"percentage":0,"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"NeverPickedService","waitForReady":true}]}]}},{"percentage":100,"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"AlwaysPickedService","waitForReady":true}]}]}}]', + type: TXT} +- expected_addrs: + - {address: '1.2.3.4:1234', is_balancer: true} + - {address: '1.2.3.4:443', is_balancer: false} + expected_chosen_service_config: null + expected_lb_policy: null + record_to_resolve: srv-ipv4-target-has-backend-and-balancer + records: + _grpclb._tcp.srv-ipv4-target-has-backend-and-balancer: + - {TTL: '2100', data: 0 0 1234 balancer-for-ipv4-has-backend-and-balancer, type: SRV} + balancer-for-ipv4-has-backend-and-balancer: + - {TTL: '2100', data: 1.2.3.4, type: A} + srv-ipv4-target-has-backend-and-balancer: + - {TTL: '2100', data: 1.2.3.4, type: A} +- expected_addrs: + - {address: '[2607:f8b0:400a:801::1002]:1234', is_balancer: true} + - {address: '[2607:f8b0:400a:801::1002]:443', is_balancer: false} + expected_chosen_service_config: null + expected_lb_policy: null + record_to_resolve: srv-ipv6-target-has-backend-and-balancer + records: + _grpclb._tcp.srv-ipv6-target-has-backend-and-balancer: + - {TTL: '2100', data: 0 0 1234 balancer-for-ipv6-has-backend-and-balancer, type: SRV} + balancer-for-ipv6-has-backend-and-balancer: + - {TTL: '2100', data: '2607:f8b0:400a:801::1002', type: AAAA} + srv-ipv6-target-has-backend-and-balancer: + - {TTL: '2100', data: '2607:f8b0:400a:801::1002', type: AAAA} + +resolver_component_tests_TODO: +- 'TODO: enable this large-txt-record test once working. (it is much longer than 512 + bytes, likely to cause use of TCP even if max payload for UDP is changed somehow, + e.g. via notes in RFC 2671)' +- expected_addrs: + - {address: '1.2.3.4:443', is_balancer: false} + expected_chosen_service_config: '{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooThree","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFour","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFive","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSix","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSeven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEight","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooNine","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTen","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEleven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]}]}' + expected_lb_policy: null + record_to_resolve: srv-ipv6-target-has-backend-and-balancer + record_to_resolve: ipv4-config-causing-fallback-to-tcp + records: + ipv4-config-causing-fallback-to-tcp: + - {TTL: '2100', data: 1.2.3.4, type: A} + - {TTL: '2100', data: 'grpc_config=[{"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooThree","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFour","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFive","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSix","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSeven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEight","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooNine","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTen","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEleven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]}]}}]', + type: TXT} diff --git a/test/cpp/naming/test_dns_server.py b/test/cpp/naming/test_dns_server.py new file mode 100755 index 0000000000..9d4b89cffb --- /dev/null +++ b/test/cpp/naming/test_dns_server.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python2.7 +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Starts a local DNS server for use in tests""" + +import argparse +import sys +import yaml +import signal +import os + +import twisted +import twisted.internet +import twisted.internet.reactor +import twisted.internet.threads +import twisted.internet.defer +import twisted.internet.protocol +import twisted.names +import twisted.names.client +import twisted.names.dns +import twisted.names.server +from twisted.names import client, server, common, authority, dns +import argparse + +_SERVER_HEALTH_CHECK_RECORD_NAME = 'health-check-local-dns-server-is-alive.resolver-tests.grpctestingexp' # missing end '.' for twisted syntax +_SERVER_HEALTH_CHECK_RECORD_DATA = '123.123.123.123' + +class NoFileAuthority(authority.FileAuthority): + def __init__(self, soa, records): + # skip FileAuthority + common.ResolverBase.__init__(self) + self.soa = soa + self.records = records + +def start_local_dns_server(args): + all_records = {} + def _push_record(name, r): + print('pushing record: |%s|' % name) + if all_records.get(name) is not None: + all_records[name].append(r) + return + all_records[name] = [r] + + def _maybe_split_up_txt_data(name, txt_data, r_ttl): + start = 0 + txt_data_list = [] + while len(txt_data[start:]) > 0: + next_read = len(txt_data[start:]) + if next_read > 255: + next_read = 255 + txt_data_list.append(txt_data[start:start+next_read]) + start += next_read + _push_record(name, dns.Record_TXT(*txt_data_list, ttl=r_ttl)) + + with open(args.records_config_path) as config: + test_records_config = yaml.load(config) + common_zone_name = test_records_config['resolver_component_tests_common_zone_name'] + for group in test_records_config['resolver_component_tests']: + for name in group['records'].keys(): + for record in group['records'][name]: + r_type = record['type'] + r_data = record['data'] + r_ttl = int(record['TTL']) + record_full_name = '%s.%s' % (name, common_zone_name) + assert record_full_name[-1] == '.' + record_full_name = record_full_name[:-1] + if r_type == 'A': + _push_record(record_full_name, dns.Record_A(r_data, ttl=r_ttl)) + if r_type == 'AAAA': + _push_record(record_full_name, dns.Record_AAAA(r_data, ttl=r_ttl)) + if r_type == 'SRV': + p, w, port, target = r_data.split(' ') + p = int(p) + w = int(w) + port = int(port) + target_full_name = '%s.%s' % (target, common_zone_name) + r_data = '%s %s %s %s' % (p, w, port, target_full_name) + _push_record(record_full_name, dns.Record_SRV(p, w, port, target_full_name, ttl=r_ttl)) + if r_type == 'TXT': + _maybe_split_up_txt_data(record_full_name, r_data, r_ttl) + # Server health check record + _push_record(_SERVER_HEALTH_CHECK_RECORD_NAME, dns.Record_A(_SERVER_HEALTH_CHECK_RECORD_DATA, ttl=0)) + soa_record = dns.Record_SOA(mname = common_zone_name) + test_domain_com = NoFileAuthority( + soa = (common_zone_name, soa_record), + records = all_records, + ) + server = twisted.names.server.DNSServerFactory( + authorities=[test_domain_com], verbose=2) + server.noisy = 2 + twisted.internet.reactor.listenTCP(args.port, server) + dns_proto = twisted.names.dns.DNSDatagramProtocol(server) + dns_proto.noisy = 2 + twisted.internet.reactor.listenUDP(args.port, dns_proto) + print('starting local dns server on 127.0.0.1:%s' % args.port) + print('starting twisted.internet.reactor') + twisted.internet.reactor.suggestThreadPoolSize(1) + twisted.internet.reactor.run() + +def _quit_on_signal(signum, _frame): + print('Received SIGNAL %d. Quitting with exit code 0' % signum) + twisted.internet.reactor.stop() + sys.stdout.flush() + sys.exit(0) + +def main(): + argp = argparse.ArgumentParser(description='Local DNS Server for resolver tests') + argp.add_argument('-p', '--port', default=None, type=int, + help='Port for DNS server to listen on for TCP and UDP.') + argp.add_argument('-r', '--records_config_path', default=None, type=str, + help=('Directory of resolver_test_record_groups.yaml file. ' + 'Defauls to path needed when the test is invoked as part of run_tests.py.')) + args = argp.parse_args() + signal.signal(signal.SIGALRM, _quit_on_signal) + signal.signal(signal.SIGTERM, _quit_on_signal) + signal.signal(signal.SIGINT, _quit_on_signal) + # Prevent zombies. Tests that use this server are short-lived. + signal.alarm(2 * 60) + start_local_dns_server(args) + +if __name__ == '__main__': + main() diff --git a/tools/buildgen/generate_build_additions.sh b/tools/buildgen/generate_build_additions.sh index 7649145f86..693c02fdb2 100644 --- a/tools/buildgen/generate_build_additions.sh +++ b/tools/buildgen/generate_build_additions.sh @@ -24,6 +24,7 @@ gen_build_yaml_dirs=" \ test/core/bad_client \ test/core/bad_ssl \ test/core/end2end \ + test/cpp/naming \ test/cpp/qps" gen_build_files="" for gen_build_yaml in $gen_build_yaml_dirs diff --git a/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile index 4ccfbc43c3..dbf58023c5 100644 --- a/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -61,7 +62,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================ # C# dependencies diff --git a/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile index 824ecf4604..7cfe98cbc8 100644 --- a/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -61,7 +62,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================= # C++ dependencies diff --git a/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile index 5f7bedde73..febe2fa251 100644 --- a/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile @@ -30,7 +30,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Define the default command. CMD ["bash"] diff --git a/tools/dockerfile/interoptest/grpc_interop_go1.7/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_go1.7/Dockerfile index 2bfa87c4ea..3a516cbb62 100644 --- a/tools/dockerfile/interoptest/grpc_interop_go1.7/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_go1.7/Dockerfile @@ -30,7 +30,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Define the default command. CMD ["bash"] diff --git a/tools/dockerfile/interoptest/grpc_interop_go1.8/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_go1.8/Dockerfile index 96391372ca..acb640a81d 100644 --- a/tools/dockerfile/interoptest/grpc_interop_go1.8/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_go1.8/Dockerfile @@ -30,7 +30,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Define the default command. CMD ["bash"] diff --git a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile index 27a924343f..354b7bfdb5 100644 --- a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile @@ -30,7 +30,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 RUN pip install twisted h2==2.6.1 hyper diff --git a/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile index 264a0f9271..92a542ff76 100644 --- a/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile @@ -45,7 +45,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Trigger download of as many Gradle artifacts as possible. diff --git a/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile index 264a0f9271..92a542ff76 100644 --- a/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile @@ -45,7 +45,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Trigger download of as many Gradle artifacts as possible. diff --git a/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile index c59b12d155..4343d56cb4 100644 --- a/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -61,7 +62,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================== # Node dependencies diff --git a/tools/dockerfile/interoptest/grpc_interop_php/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_php/Dockerfile index f859a46514..d6e229f84a 100644 --- a/tools/dockerfile/interoptest/grpc_interop_php/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_php/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ diff --git a/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile index 42b1107c11..271c6e75e9 100644 --- a/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -61,7 +62,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Prepare ccache RUN ln -s /usr/bin/ccache /usr/local/bin/gcc diff --git a/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile index 3c744b434e..7bcada68e0 100644 --- a/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -61,7 +62,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================== # Ruby dependencies diff --git a/tools/dockerfile/test/csharp_jessie_x64/Dockerfile b/tools/dockerfile/test/csharp_jessie_x64/Dockerfile index 60dc7eb21a..40d46fcf58 100644 --- a/tools/dockerfile/test/csharp_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/csharp_jessie_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================ # C# dependencies diff --git a/tools/dockerfile/test/cxx_jessie_x64/Dockerfile b/tools/dockerfile/test/cxx_jessie_x64/Dockerfile index f9622dd757..888a37baca 100644 --- a/tools/dockerfile/test/cxx_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_jessie_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================= # C++ dependencies diff --git a/tools/dockerfile/test/cxx_jessie_x86/Dockerfile b/tools/dockerfile/test/cxx_jessie_x86/Dockerfile index 2d396ce2ff..319f1e1889 100644 --- a/tools/dockerfile/test/cxx_jessie_x86/Dockerfile +++ b/tools/dockerfile/test/cxx_jessie_x86/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================= # C++ dependencies diff --git a/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile b/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile index 349a6eff83..61f005d9da 100644 --- a/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================= # C++ dependencies diff --git a/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile b/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile index bccfdba7cd..f35247eccb 100644 --- a/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================= # C++ dependencies diff --git a/tools/dockerfile/test/fuzzer/Dockerfile b/tools/dockerfile/test/fuzzer/Dockerfile index 94c37b3f63..ce1badfeb1 100644 --- a/tools/dockerfile/test/fuzzer/Dockerfile +++ b/tools/dockerfile/test/fuzzer/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================= # C++ dependencies diff --git a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile index b7373b5d9c..59fe4d8f93 100644 --- a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -122,7 +123,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Install coverage for Python test coverage reporting RUN pip install coverage diff --git a/tools/dockerfile/test/node_jessie_x64/Dockerfile b/tools/dockerfile/test/node_jessie_x64/Dockerfile index 31602e594e..103be8412b 100644 --- a/tools/dockerfile/test/node_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/node_jessie_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -76,7 +77,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================== # Node dependencies diff --git a/tools/dockerfile/test/php7_jessie_x64/Dockerfile b/tools/dockerfile/test/php7_jessie_x64/Dockerfile index e79305bb27..f6d426bcd6 100644 --- a/tools/dockerfile/test/php7_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/php7_jessie_x64/Dockerfile @@ -77,7 +77,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Prepare ccache RUN ln -s /usr/bin/ccache /usr/local/bin/gcc diff --git a/tools/dockerfile/test/php_jessie_x64/Dockerfile b/tools/dockerfile/test/php_jessie_x64/Dockerfile index 245a642367..ae82a8d99f 100644 --- a/tools/dockerfile/test/php_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/php_jessie_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================= # PHP dependencies diff --git a/tools/dockerfile/test/python_jessie_x64/Dockerfile b/tools/dockerfile/test/python_jessie_x64/Dockerfile index 5093b793b0..d5d781cd1a 100644 --- a/tools/dockerfile/test/python_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/python_jessie_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Prepare ccache RUN ln -s /usr/bin/ccache /usr/local/bin/gcc diff --git a/tools/dockerfile/test/python_pyenv_x64/Dockerfile b/tools/dockerfile/test/python_pyenv_x64/Dockerfile index ff8f0231cb..3b4ad12b6d 100644 --- a/tools/dockerfile/test/python_pyenv_x64/Dockerfile +++ b/tools/dockerfile/test/python_pyenv_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Install dependencies for pyenv RUN apt-get update && apt-get install -y \ diff --git a/tools/dockerfile/test/ruby_jessie_x64/Dockerfile b/tools/dockerfile/test/ruby_jessie_x64/Dockerfile index d77917c87f..3d879bb0c9 100644 --- a/tools/dockerfile/test/ruby_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/ruby_jessie_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================== # Ruby dependencies diff --git a/tools/dockerfile/test/sanity/Dockerfile b/tools/dockerfile/test/sanity/Dockerfile index 0f2f9ede58..44732a5ae4 100644 --- a/tools/dockerfile/test/sanity/Dockerfile +++ b/tools/dockerfile/test/sanity/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #======================== # Sanity test dependencies diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc index dd2741e595..3319aa4394 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc @@ -46,7 +46,7 @@ pod repo update # needed by python # python brew install coreutils # we need grealpath pip install virtualenv --user python -pip install -U six tox setuptools --user python +pip install -U six tox setuptools twisted pyyaml --user python export PYTHONPATH=/Library/Python/3.4/site-packages # set xcode version for Obj-C tests diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 2f6e34bfb3..8d42c3e432 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -5627,6 +5627,86 @@ "third_party": false, "type": "target" }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc++_test_config", + "grpc++_test_util_unsecure", + "grpc++_unsecure", + "grpc_test_util_unsecure", + "grpc_unsecure" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "resolver_component_test_unsecure", + "src": [ + "test/cpp/naming/resolver_component_test.cc" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc++", + "grpc++_test_config", + "grpc++_test_util", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "resolver_component_test", + "src": [ + "test/cpp/naming/resolver_component_test.cc" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc++", + "grpc++_test_config", + "grpc++_test_util", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "resolver_component_tests_runner_invoker_unsecure", + "src": [ + "test/cpp/naming/resolver_component_tests_runner_invoker.cc" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc++", + "grpc++_test_config", + "grpc++_test_util", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "resolver_component_tests_runner_invoker", + "src": [ + "test/cpp/naming/resolver_component_tests_runner_invoker.cc" + ], + "third_party": false, + "type": "target" + }, { "deps": [ "gpr", diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 45d7145e8e..72dfbc0b98 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -4404,6 +4404,52 @@ "posix" ] }, + { + "args": [ + "--test_bin_name=resolver_component_test_unsecure", + "--running_under_bazel=false" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": false, + "language": "c++", + "name": "resolver_component_tests_runner_invoker_unsecure", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, + { + "args": [ + "--test_bin_name=resolver_component_test", + "--running_under_bazel=false" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": false, + "language": "c++", + "name": "resolver_component_tests_runner_invoker", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, { "args": [ "third_party/boringssl/crypto/aes/aes_tests.txt" -- cgit v1.2.3 From 05eadf21e05d521d73a05babf7c8f45218728355 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 11 Sep 2017 11:46:59 +0200 Subject: reveal leftover processes --- tools/internal_ci/linux/grpc_run_tests_matrix.sh | 12 +++++++++++- tools/internal_ci/macos/grpc_run_tests_matrix.sh | 5 +++++ tools/internal_ci/windows/grpc_run_tests_matrix.bat | 10 ++++++---- 3 files changed, 22 insertions(+), 5 deletions(-) (limited to 'tools/internal_ci') diff --git a/tools/internal_ci/linux/grpc_run_tests_matrix.sh b/tools/internal_ci/linux/grpc_run_tests_matrix.sh index 028704b3ff..bd1430b741 100755 --- a/tools/internal_ci/linux/grpc_run_tests_matrix.sh +++ b/tools/internal_ci/linux/grpc_run_tests_matrix.sh @@ -20,4 +20,14 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_rc -tools/run_tests/run_tests_matrix.py $RUN_TESTS_FLAGS +tools/run_tests/run_tests_matrix.py $RUN_TESTS_FLAGS || FAILED="true" + +# Reveal leftover processes that might be left behind by the build +ps aux | grep -i kbuilder + +echo 'Exiting gRPC main test script.' + +if [ "$FAILED" != "" ] +then + exit 1 +fi diff --git a/tools/internal_ci/macos/grpc_run_tests_matrix.sh b/tools/internal_ci/macos/grpc_run_tests_matrix.sh index 9a43e4869b..8e7fd54a62 100755 --- a/tools/internal_ci/macos/grpc_run_tests_matrix.sh +++ b/tools/internal_ci/macos/grpc_run_tests_matrix.sh @@ -25,6 +25,11 @@ tools/run_tests/run_tests_matrix.py $RUN_TESTS_FLAGS || FAILED="true" # kill port_server.py to prevent the build from hanging ps aux | grep port_server\\.py | awk '{print $2}' | xargs kill -9 +# Reveal leftover processes that might be left behind by the build +ps aux | grep -i kbuilder + +echo 'Exiting gRPC main test script.' + if [ "$FAILED" != "" ] then exit 1 diff --git a/tools/internal_ci/windows/grpc_run_tests_matrix.bat b/tools/internal_ci/windows/grpc_run_tests_matrix.bat index 08d834f8b0..10627add25 100644 --- a/tools/internal_ci/windows/grpc_run_tests_matrix.bat +++ b/tools/internal_ci/windows/grpc_run_tests_matrix.bat @@ -17,8 +17,10 @@ cd /d %~dp0\..\..\.. call tools/internal_ci/helper_scripts/prepare_build_windows.bat -python tools/run_tests/run_tests_matrix.py %RUN_TESTS_FLAGS% || goto :error -goto :EOF +python tools/run_tests/run_tests_matrix.py %RUN_TESTS_FLAGS% +set RUNTESTS_EXITCODE=%errorlevel% -:error -exit /b %errorlevel% +@rem Reveal leftover processes that might be left behind by the build +tasklist /V + +exit /b %RUNTESTS_EXITCODE% -- cgit v1.2.3 From f68858464afd15402e09f480938e5efc6c7ff02f Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Mon, 21 Aug 2017 08:09:06 -0700 Subject: Add Kokoro perf PR jobs --- .../helper_scripts/prepare_build_linux_perf_rc | 15 ++++++++ .../internal_ci/linux/grpc_microbenchmark_diff.sh | 38 ++++++++++++++++++++ tools/internal_ci/linux/grpc_trickle_diff.sh | 42 ++++++++++++++++++++++ .../pull_request/grpc_microbenchmark_diff.cfg | 25 +++++++++++++ .../linux/pull_request/grpc_trickle_diff.cfg | 25 +++++++++++++ 5 files changed, 145 insertions(+) create mode 100755 tools/internal_ci/linux/grpc_microbenchmark_diff.sh create mode 100755 tools/internal_ci/linux/grpc_trickle_diff.sh create mode 100644 tools/internal_ci/linux/pull_request/grpc_microbenchmark_diff.cfg create mode 100644 tools/internal_ci/linux/pull_request/grpc_trickle_diff.cfg (limited to 'tools/internal_ci') diff --git a/tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc b/tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc index 1b7779caa8..dc3c75a6a1 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc @@ -19,4 +19,19 @@ ulimit -n 32768 ulimit -c unlimited +# Performance PR testing needs GH API key and PR metadata to comment results +if [ -n "$KOKORO_GITHUB_PULL_REQUEST_NUMBER" ]; then + set +x + sudo apt-get install -y jq + export ghprbTargetBranch=$(curl -s https://api.github.com/repos/grpc/grpc/pulls/$KOKORO_GITHUB_PULL_REQUEST_NUMBER | jq -r .base.ref) + + gsutil cp gs://grpc-testing-secrets/github_credentials/oauth_token.txt ~/ + # TODO(matt-kwong): rename this to GITHUB_OAUTH_TOKEN after Jenkins deprecation + export JENKINS_OAUTH_TOKEN=$(cat ~/oauth_token.txt) + export ghprbPullId=$KOKORO_GITHUB_PULL_REQUEST_NUMBER + set -x +fi + +sudo pip install tabulate + git submodule update --init diff --git a/tools/internal_ci/linux/grpc_microbenchmark_diff.sh b/tools/internal_ci/linux/grpc_microbenchmark_diff.sh new file mode 100755 index 0000000000..58ffcf336b --- /dev/null +++ b/tools/internal_ci/linux/grpc_microbenchmark_diff.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This script is invoked by Jenkins and runs a diff on the microbenchmarks +set -ex + +# List of benchmarks that provide good signal for analyzing performance changes in pull requests +BENCHMARKS_TO_RUN="bm_fullstack_unary_ping_pong bm_fullstack_streaming_ping_pong bm_fullstack_streaming_pump bm_closure bm_cq bm_call_create bm_error bm_chttp2_hpack bm_chttp2_transport bm_pollset bm_metadata" + +# Enter the gRPC repo root +cd $(dirname $0)/../../.. + +source tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc + +tools/run_tests/start_port_server.py +tools/jenkins/run_c_cpp_test.sh tools/profiling/microbenchmarks/bm_diff/bm_main.py \ + -d origin/$ghprbTargetBranch \ + -b $BENCHMARKS_TO_RUN || FAILED="true" + +# kill port_server.py to prevent the build from hanging +ps aux | grep port_server\\.py | awk '{print $2}' | xargs kill -9 + +if [ "$FAILED" != "" ] +then + exit 1 +fi diff --git a/tools/internal_ci/linux/grpc_trickle_diff.sh b/tools/internal_ci/linux/grpc_trickle_diff.sh new file mode 100755 index 0000000000..624031afa7 --- /dev/null +++ b/tools/internal_ci/linux/grpc_trickle_diff.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This script is invoked by Jenkins and runs a diff on the microbenchmarks +set -ex + +# List of benchmarks that provide good signal for analyzing performance changes in pull requests +BENCHMARKS_TO_RUN="cli_transport_stalls_per_iteration cli_stream_stalls_per_iteration svr_transport_stalls_per_iteration svr_stream_stalls_per_iteration" + +# Enter the gRPC repo root +cd $(dirname $0)/../../.. + +source tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc + +tools/run_tests/start_port_server.py +tools/jenkins/run_c_cpp_test.sh tools/profiling/microbenchmarks/bm_diff/bm_main.py \ + -d origin/$ghprbTargetBranch \ + -b bm_fullstack_trickle \ + -l 4 \ + -t $BENCHMARKS_TO_RUN \ + --no-counters \ + --pr_comment_name trickle || FAILED="true" + +# kill port_server.py to prevent the build from hanging +ps aux | grep port_server\\.py | awk '{print $2}' | xargs kill -9 + +if [ "$FAILED" != "" ] +then + exit 1 +fi diff --git a/tools/internal_ci/linux/pull_request/grpc_microbenchmark_diff.cfg b/tools/internal_ci/linux/pull_request/grpc_microbenchmark_diff.cfg new file mode 100644 index 0000000000..9269c345f0 --- /dev/null +++ b/tools/internal_ci/linux/pull_request/grpc_microbenchmark_diff.cfg @@ -0,0 +1,25 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_microbenchmark_diff.sh" +timeout_mins: 120 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + } +} diff --git a/tools/internal_ci/linux/pull_request/grpc_trickle_diff.cfg b/tools/internal_ci/linux/pull_request/grpc_trickle_diff.cfg new file mode 100644 index 0000000000..e86b3ab475 --- /dev/null +++ b/tools/internal_ci/linux/pull_request/grpc_trickle_diff.cfg @@ -0,0 +1,25 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_trickle_diff.sh" +timeout_mins: 120 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + } +} -- cgit v1.2.3 From 4cd56837465234b7384bd3cb98abebced881f104 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Mon, 18 Sep 2017 11:22:03 -0700 Subject: Split C and C++ basic tests --- .../linux/pull_request/grpc_basictests_c_dbg.cfg | 30 ++++++++++++++++++++++ .../linux/pull_request/grpc_basictests_c_opt.cfg | 30 ++++++++++++++++++++++ .../linux/pull_request/grpc_basictests_cpp_dbg.cfg | 30 ++++++++++++++++++++++ .../linux/pull_request/grpc_basictests_cpp_opt.cfg | 30 ++++++++++++++++++++++ 4 files changed, 120 insertions(+) create mode 100644 tools/internal_ci/linux/pull_request/grpc_basictests_c_dbg.cfg create mode 100644 tools/internal_ci/linux/pull_request/grpc_basictests_c_opt.cfg create mode 100644 tools/internal_ci/linux/pull_request/grpc_basictests_cpp_dbg.cfg create mode 100644 tools/internal_ci/linux/pull_request/grpc_basictests_cpp_opt.cfg (limited to 'tools/internal_ci') diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_c_dbg.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_c_dbg.cfg new file mode 100644 index 0000000000..577cb28ae5 --- /dev/null +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_c_dbg.cfg @@ -0,0 +1,30 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + } +} + +env_vars { + key: "RUN_TESTS_FLAGS" + value: "-f basictests linux c dbg --inner_jobs 16 -j 1 --internal_ci --max_time=3600" +} diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_c_opt.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_c_opt.cfg new file mode 100644 index 0000000000..9e0b724b2e --- /dev/null +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_c_opt.cfg @@ -0,0 +1,30 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + } +} + +env_vars { + key: "RUN_TESTS_FLAGS" + value: "-f basictests linux c opt --inner_jobs 16 -j 1 --internal_ci --max_time=3600" +} diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_dbg.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_dbg.cfg new file mode 100644 index 0000000000..0fda74cf44 --- /dev/null +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_dbg.cfg @@ -0,0 +1,30 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + } +} + +env_vars { + key: "RUN_TESTS_FLAGS" + value: "-f basictests linux c++ dbg --inner_jobs 16 -j 1 --internal_ci --max_time=3600" +} diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_opt.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_opt.cfg new file mode 100644 index 0000000000..199a8905d9 --- /dev/null +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_opt.cfg @@ -0,0 +1,30 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + } +} + +env_vars { + key: "RUN_TESTS_FLAGS" + value: "-f basictests linux c++ opt --inner_jobs 16 -j 1 --internal_ci --max_time=3600" +} -- cgit v1.2.3 From 67cd21e6c7e1e9347760349c29f32d0d5b42c843 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 19 Sep 2017 16:21:20 +0200 Subject: adjust sanitizer timeouts --- .../internal_ci/linux/sanitizer/pull_request/grpc_c_asan.cfg | 2 +- .../internal_ci/linux/sanitizer/pull_request/grpc_c_msan.cfg | 2 +- .../internal_ci/linux/sanitizer/pull_request/grpc_c_tsan.cfg | 2 +- .../linux/sanitizer/pull_request/grpc_c_ubsan.cfg | 2 +- .../linux/sanitizer/pull_request/grpc_cpp_asan.cfg | 2 +- tools/run_tests/run_tests_matrix.py | 12 +++++++++++- 6 files changed, 16 insertions(+), 6 deletions(-) (limited to 'tools/internal_ci') diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_asan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_asan.cfg index db9cb4f597..1daf7a514e 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_asan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_asan.cfg @@ -17,7 +17,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" -timeout_mins: 1440 +timeout_mins: 240 action { define_artifacts { regex: "**/*sponge_log.xml" diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_msan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_msan.cfg index ba651d357c..a8503b7bcb 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_msan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_msan.cfg @@ -17,7 +17,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" -timeout_mins: 1440 +timeout_mins: 240 action { define_artifacts { regex: "**/*sponge_log.xml" diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_tsan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_tsan.cfg index ccbd95ae63..12af4581eb 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_tsan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_tsan.cfg @@ -17,7 +17,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" -timeout_mins: 1440 +timeout_mins: 240 action { define_artifacts { regex: "**/*sponge_log.xml" diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_ubsan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_ubsan.cfg index c240d3a573..0d3803bf23 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_ubsan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_ubsan.cfg @@ -17,7 +17,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" -timeout_mins: 1440 +timeout_mins: 240 action { define_artifacts { regex: "**/*sponge_log.xml" diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_asan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_asan.cfg index 38db2f4f60..557561810b 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_asan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_asan.cfg @@ -17,7 +17,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" -timeout_mins: 1440 +timeout_mins: 240 action { define_artifacts { regex: "**/*sponge_log.xml" diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py index 957e7b569e..2417c5ae55 100755 --- a/tools/run_tests/run_tests_matrix.py +++ b/tools/run_tests/run_tests_matrix.py @@ -35,6 +35,9 @@ _DEFAULT_RUNTESTS_TIMEOUT = 1*60*60 # clang docker. _CPP_RUNTESTS_TIMEOUT = 4*60*60 +# C++ TSAN takes longer than other sanitizers +_CPP_TSAN_RUNTESTS_TIMEOUT = 8*60*60 + # Number of jobs assigned to each run_tests.py instance _DEFAULT_INNER_JOBS = 2 @@ -190,12 +193,19 @@ def _create_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS): inner_jobs=inner_jobs, timeout_seconds=_CPP_RUNTESTS_TIMEOUT) test_jobs += _generate_jobs(languages=['c++'], - configs=['asan', 'tsan'], + configs=['asan'], platforms=['linux'], labels=['sanitizers', 'corelang'], extra_args=extra_args, inner_jobs=inner_jobs, timeout_seconds=_CPP_RUNTESTS_TIMEOUT) + test_jobs += _generate_jobs(languages=['c++'], + configs=['tsan'], + platforms=['linux'], + labels=['sanitizers', 'corelang'], + extra_args=extra_args, + inner_jobs=inner_jobs, + timeout_seconds=_CPP_TSAN_RUNTESTS_TIMEOUT) return test_jobs -- cgit v1.2.3 From 8b255c7e4764d316adbdc90e49ec86f2336222f6 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Wed, 20 Sep 2017 15:47:29 -0700 Subject: Add line count job to Kokoro --- tools/internal_ci/linux/grpc_line_count.cfg | 19 +++++++++++++++++++ tools/internal_ci/linux/grpc_line_count.sh | 29 +++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 tools/internal_ci/linux/grpc_line_count.cfg create mode 100755 tools/internal_ci/linux/grpc_line_count.sh (limited to 'tools/internal_ci') diff --git a/tools/internal_ci/linux/grpc_line_count.cfg b/tools/internal_ci/linux/grpc_line_count.cfg new file mode 100644 index 0000000000..5854d9c9c2 --- /dev/null +++ b/tools/internal_ci/linux/grpc_line_count.cfg @@ -0,0 +1,19 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_line_count.sh" +timeout_mins: 30 diff --git a/tools/internal_ci/linux/grpc_line_count.sh b/tools/internal_ci/linux/grpc_line_count.sh new file mode 100755 index 0000000000..22e484ba2f --- /dev/null +++ b/tools/internal_ci/linux/grpc_line_count.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This script counts the numbers of line in gRPC's repo and uploads to BQ +set -ex + +# Enter the gRPC repo root +cd $(dirname $0)/../../.. + +git submodule update --init + +# Install cloc +git clone -b v1.72 https://github.com/AlDanial/cloc/ ~/cloc +PERL_MM_USE_DEFAULT=1 sudo perl -MCPAN -e 'install Regexp::Common; install Algorithm::Diff' +sudo make install -C ~/cloc/Unix + +./tools/line_count/collect-now.sh -- cgit v1.2.3 From f1a25c7df1f4f18686fdb8de7bc13f103be34f57 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 21 Sep 2017 14:07:02 +0200 Subject: split windows PR basictests into dbg and opt --- .../windows/pull_request/grpc_basictests_dbg.cfg | 30 ++++++++++++++++++++++ .../windows/pull_request/grpc_basictests_opt.cfg | 30 ++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 tools/internal_ci/windows/pull_request/grpc_basictests_dbg.cfg create mode 100644 tools/internal_ci/windows/pull_request/grpc_basictests_opt.cfg (limited to 'tools/internal_ci') diff --git a/tools/internal_ci/windows/pull_request/grpc_basictests_dbg.cfg b/tools/internal_ci/windows/pull_request/grpc_basictests_dbg.cfg new file mode 100644 index 0000000000..81a1e9d3c8 --- /dev/null +++ b/tools/internal_ci/windows/pull_request/grpc_basictests_dbg.cfg @@ -0,0 +1,30 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + } +} + +env_vars { + key: "RUN_TESTS_FLAGS" + value: "-f basictests windows dbg -j 1 --inner_jobs 8 --internal_ci --max_time=3600" +} diff --git a/tools/internal_ci/windows/pull_request/grpc_basictests_opt.cfg b/tools/internal_ci/windows/pull_request/grpc_basictests_opt.cfg new file mode 100644 index 0000000000..3bb6510ca7 --- /dev/null +++ b/tools/internal_ci/windows/pull_request/grpc_basictests_opt.cfg @@ -0,0 +1,30 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + } +} + +env_vars { + key: "RUN_TESTS_FLAGS" + value: "-f basictests windows opt -j 1 --inner_jobs 8 --internal_ci --max_time=3600" +} -- cgit v1.2.3 From d987d95423e7a410190157248f42537657f0b359 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Thu, 21 Sep 2017 12:21:17 -0700 Subject: Don't collect everything after running a build --- tools/internal_ci/linux/grpc_performance_profile_daily.cfg | 1 - tools/internal_ci/linux/grpc_performance_profile_master.cfg | 1 - 2 files changed, 2 deletions(-) (limited to 'tools/internal_ci') diff --git a/tools/internal_ci/linux/grpc_performance_profile_daily.cfg b/tools/internal_ci/linux/grpc_performance_profile_daily.cfg index 9831869edb..7320a19a76 100644 --- a/tools/internal_ci/linux/grpc_performance_profile_daily.cfg +++ b/tools/internal_ci/linux/grpc_performance_profile_daily.cfg @@ -19,7 +19,6 @@ build_file: "grpc/tools/internal_ci/linux/grpc_performance_profile_daily.sh" timeout_mins: 1440 action { define_artifacts { - regex: "**" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_performance_profile_master.cfg b/tools/internal_ci/linux/grpc_performance_profile_master.cfg index e4cefbc3f4..ccb8aabb38 100644 --- a/tools/internal_ci/linux/grpc_performance_profile_master.cfg +++ b/tools/internal_ci/linux/grpc_performance_profile_master.cfg @@ -19,7 +19,6 @@ build_file: "grpc/tools/internal_ci/linux/grpc_performance_profile_master.sh" timeout_mins: 600 action { define_artifacts { - regex: "**" regex: "github/grpc/reports/**" } } -- cgit v1.2.3