diff options
Diffstat (limited to 'tools/internal_ci')
29 files changed, 266 insertions, 43 deletions
diff --git a/tools/internal_ci/helper_scripts/delete_nonartifacts.sh b/tools/internal_ci/helper_scripts/delete_nonartifacts.sh new file mode 100755 index 0000000000..6c49b7377e --- /dev/null +++ b/tools/internal_ci/helper_scripts/delete_nonartifacts.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# change to grpc repo root +cd "$(dirname "$0")/../../.." + +# After kokoro build finishes, the workspace gets rsync'ed to another machine, +# from where the artifacts and reports are processed. +# Especially on Windows, the rsync can take long time, so we cleanup the workspace +# after finishing each build. We only leave files we want to keep: +# - reports and artifacts +# - directory containing the kokoro scripts to prevent deleting a script while being executed. +time find . -type f -not -iname "*sponge_log.xml" -not -path "./reports/*" -not -path "./artifacts/*" -not -path "./tools/internal_ci/*" -exec rm -f {} +
\ No newline at end of file diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc index d84bebd4ea..d2b77691d4 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc @@ -69,6 +69,11 @@ export PYTHONPATH=/Library/Python/3.4/site-packages # set xcode version for Obj-C tests sudo xcode-select -switch /Applications/Xcode_9.2.app/Contents/Developer/ +# Disable some unwanted dotnet options +export NUGET_XMLDOC_MODE=skip +export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true +export DOTNET_CLI_TELEMETRY_OPTOUT=true + # TODO(jtattermusch): better debugging of clock skew, remove once not needed date diff --git a/tools/internal_ci/helper_scripts/prepare_build_windows.bat b/tools/internal_ci/helper_scripts/prepare_build_windows.bat index 9fa6860226..f987f8a8cb 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_windows.bat +++ b/tools/internal_ci/helper_scripts/prepare_build_windows.bat @@ -34,4 +34,9 @@ netsh interface ip add dnsservers "Local Area Connection 8" 8.8.4.4 index=3 @rem Needed for big_query_utils python -m pip install google-api-python-client +@rem Disable some unwanted dotnet options +set NUGET_XMLDOC_MODE=skip +set DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true +set DOTNET_CLI_TELEMETRY_OPTOUT=true + git submodule update --init diff --git a/tools/internal_ci/linux/grpc_asan_on_foundry.sh b/tools/internal_ci/linux/grpc_asan_on_foundry.sh index 2aebb65552..791c56c086 100644 --- a/tools/internal_ci/linux/grpc_asan_on_foundry.sh +++ b/tools/internal_ci/linux/grpc_asan_on_foundry.sh @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +export UPLOAD_TEST_RESULTS=true EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=address --linkopt=-fsanitize=address --test_timeout=3600" github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" diff --git a/tools/internal_ci/linux/grpc_bazel_build.cfg b/tools/internal_ci/linux/grpc_bazel_build.cfg index 4d3a2ade5d..59f5da9a67 100644 --- a/tools/internal_ci/linux/grpc_bazel_build.cfg +++ b/tools/internal_ci/linux/grpc_bazel_build.cfg @@ -19,5 +19,5 @@ build_file: "grpc/tools/internal_ci/linux/grpc_bazel.sh" timeout_mins: 60 env_vars { key: "BAZEL_SCRIPT" - value: "tools/jenkins/run_bazel_basic_in_docker.sh" + value: "tools/internal_ci/linux/grpc_bazel_build_in_docker.sh" } diff --git a/tools/internal_ci/linux/grpc_bazel_build_in_docker.sh b/tools/internal_ci/linux/grpc_bazel_build_in_docker.sh new file mode 100755 index 0000000000..24598f43f0 --- /dev/null +++ b/tools/internal_ci/linux/grpc_bazel_build_in_docker.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Test basic Bazel features +# +# NOTE: No empty lines should appear in this file before igncr is set! +set -ex -o igncr || set -ex + +mkdir -p /var/local/git +git clone /var/local/jenkins/grpc /var/local/git/grpc +(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \ +&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \ +${name}') +cd /var/local/git/grpc +bazel build --spawn_strategy=standalone --genrule_strategy=standalone :all test/... examples/... diff --git a/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh b/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh index 3102992bef..7881e3a7fb 100755 --- a/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh +++ b/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh @@ -22,8 +22,8 @@ mkdir -p ${KOKORO_KEYSTORE_DIR} cp ${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json ${KOKORO_KEYSTORE_DIR}/4321_grpc-testing-service temp_dir=$(mktemp -d) -ln -f "${KOKORO_GFILE_DIR}/bazel-canary" ${temp_dir}/bazel -chmod 755 "${KOKORO_GFILE_DIR}/bazel-canary" +ln -f "${KOKORO_GFILE_DIR}/bazel-release-0.12.0" ${temp_dir}/bazel +chmod 755 "${KOKORO_GFILE_DIR}/bazel-release-0.12.0" export PATH="${temp_dir}:${PATH}" # This should show ${temp_dir}/bazel which bazel @@ -53,4 +53,16 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_rc --crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/debian8_clang/0.3.0/bazel_0.10.0:toolchain \ --define GRPC_PORT_ISOLATED_RUNTIME=1 \ $1 \ - -- //test/... + -- //test/... || FAILED="true" + +if [ "$UPLOAD_TEST_RESULTS" != "" ] +then + # Sleep to let ResultStore finish writing results before querying + sleep 60 + python ./tools/run_tests/python_utils/upload_rbe_results.py +fi + +if [ "$FAILED" != "" ] +then + exit 1 +fi diff --git a/tools/internal_ci/linux/grpc_bazel_on_foundry_dbg.sh b/tools/internal_ci/linux/grpc_bazel_on_foundry_dbg.sh index 882e4df21b..192d9d1b2d 100644 --- a/tools/internal_ci/linux/grpc_bazel_on_foundry_dbg.sh +++ b/tools/internal_ci/linux/grpc_bazel_on_foundry_dbg.sh @@ -13,6 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -EXTRA_FLAGS="-c dbg --test_timeout=300,450,1200,3600 --runs_per_test_detects_flakes --runs_per_test=1" +export UPLOAD_TEST_RESULTS=true +EXTRA_FLAGS="-c dbg --test_timeout=300,450,1200,3600 --runs_per_test_detects_flakes --runs_per_test=2" github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" - diff --git a/tools/internal_ci/linux/grpc_bazel_on_foundry_opt.sh b/tools/internal_ci/linux/grpc_bazel_on_foundry_opt.sh index 85f69537bd..6fb3c77892 100644 --- a/tools/internal_ci/linux/grpc_bazel_on_foundry_opt.sh +++ b/tools/internal_ci/linux/grpc_bazel_on_foundry_opt.sh @@ -13,5 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -EXTRA_FLAGS="-c opt --test_timeout=300,450,1200,3600 --runs_per_test_detects_flakes --runs_per_test=1" +export UPLOAD_TEST_RESULTS=true +EXTRA_FLAGS="-c opt --test_timeout=300,450,1200,3600 --runs_per_test_detects_flakes --runs_per_test=2" github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" diff --git a/tools/internal_ci/linux/grpc_bazel_test.cfg b/tools/internal_ci/linux/grpc_bazel_test.cfg index 46198b9600..6b03be9518 100644 --- a/tools/internal_ci/linux/grpc_bazel_test.cfg +++ b/tools/internal_ci/linux/grpc_bazel_test.cfg @@ -19,5 +19,5 @@ build_file: "grpc/tools/internal_ci/linux/grpc_bazel.sh" timeout_mins: 240 env_vars { key: "BAZEL_SCRIPT" - value: "tools/jenkins/run_bazel_full_in_docker.sh" + value: "tools/internal_ci/linux/grpc_bazel_test_in_docker.sh" } diff --git a/tools/internal_ci/linux/grpc_bazel_test_in_docker.sh b/tools/internal_ci/linux/grpc_bazel_test_in_docker.sh new file mode 100755 index 0000000000..9663debe69 --- /dev/null +++ b/tools/internal_ci/linux/grpc_bazel_test_in_docker.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Test full Bazel +# +# NOTE: No empty lines should appear in this file before igncr is set! +set -ex -o igncr || set -ex + +mkdir -p /var/local/git +git clone /var/local/jenkins/grpc /var/local/git/grpc +(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \ +&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \ +${name}') +cd /var/local/git/grpc/test +bazel test --spawn_strategy=standalone --genrule_strategy=standalone ... diff --git a/tools/internal_ci/linux/grpc_interop_matrix.cfg b/tools/internal_ci/linux/grpc_interop_matrix.cfg index 71e930e0b0..ae59e930f7 100644 --- a/tools/internal_ci/linux/grpc_interop_matrix.cfg +++ b/tools/internal_ci/linux/grpc_interop_matrix.cfg @@ -17,7 +17,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/linux/grpc_interop_matrix.sh" # grpc_interop tests can take 1 hours to complete. -timeout_mins: 120 +timeout_mins: 300 action { define_artifacts { regex: "**/sponge_log.xml" diff --git a/tools/internal_ci/linux/grpc_interop_toprod.cfg b/tools/internal_ci/linux/grpc_interop_toprod.cfg index ff7a98f44e..8d025c4f60 100644 --- a/tools/internal_ci/linux/grpc_interop_toprod.cfg +++ b/tools/internal_ci/linux/grpc_interop_toprod.cfg @@ -26,5 +26,5 @@ action { env_vars { key: "RUN_TESTS_FLAGS" - value: "-l all --cloud_to_prod --cloud_to_prod_auth --prod_servers default gateway_v4 cloud_gateway cloud_gateway_v4 --use_docker --internal_ci -t -j 12 --bq_result_table interop_results" + value: "-l all --cloud_to_prod --cloud_to_prod_auth --prod_servers default gateway_v4 --use_docker --internal_ci -t -j 12 --bq_result_table interop_results" } diff --git a/tools/internal_ci/linux/grpc_microbenchmark_diff.sh b/tools/internal_ci/linux/grpc_microbenchmark_diff.sh index 45add1b02d..9834aaa053 100755 --- a/tools/internal_ci/linux/grpc_microbenchmark_diff.sh +++ b/tools/internal_ci/linux/grpc_microbenchmark_diff.sh @@ -25,9 +25,9 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc tools/run_tests/start_port_server.py -tools/jenkins/run_c_cpp_test.sh tools/profiling/bloat/bloat_diff.py \ +tools/internal_ci/linux/run_if_c_cpp_modified.sh tools/profiling/bloat/bloat_diff.py \ -d origin/$ghprbTargetBranch || FAILED="true" -tools/jenkins/run_c_cpp_test.sh tools/profiling/microbenchmarks/bm_diff/bm_main.py \ +tools/internal_ci/linux/run_if_c_cpp_modified.sh tools/profiling/microbenchmarks/bm_diff/bm_main.py \ -d origin/$ghprbTargetBranch \ -b $BENCHMARKS_TO_RUN || FAILED="true" diff --git a/tools/internal_ci/linux/grpc_msan_on_foundry.sh b/tools/internal_ci/linux/grpc_msan_on_foundry.sh index 6858d971cb..5e644793ad 100644 --- a/tools/internal_ci/linux/grpc_msan_on_foundry.sh +++ b/tools/internal_ci/linux/grpc_msan_on_foundry.sh @@ -23,8 +23,8 @@ mkdir -p ${KOKORO_KEYSTORE_DIR} cp ${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json ${KOKORO_KEYSTORE_DIR}/4321_grpc-testing-service temp_dir=$(mktemp -d) -ln -f "${KOKORO_GFILE_DIR}/bazel-canary" ${temp_dir}/bazel -chmod 755 "${KOKORO_GFILE_DIR}/bazel-canary" +ln -f "${KOKORO_GFILE_DIR}/bazel-release-0.12.0" ${temp_dir}/bazel +chmod 755 "${KOKORO_GFILE_DIR}/bazel-release-0.12.0" export PATH="${temp_dir}:${PATH}" # This should show ${temp_dir}/bazel which bazel @@ -61,4 +61,13 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_rc --action_env=LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH \ --host_crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/debian8_clang/0.3.0/bazel_0.10.0:toolchain \ --crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/experimental/debian8_clang/0.3.0/bazel_0.10.0/msan:msan_experimental_toolchain \ - -- //test/... + -- //test/... || FAILED="true" + +# Sleep to let ResultStore finish writing results before querying +sleep 60 +python ./tools/run_tests/python_utils/upload_rbe_results.py + +if [ "$FAILED" != "" ] +then + exit 1 +fi diff --git a/tools/internal_ci/linux/grpc_performance_profile_master.sh b/tools/internal_ci/linux/grpc_performance_profile_master.sh index 40bbfe89dc..fbff17466b 100755 --- a/tools/internal_ci/linux/grpc_performance_profile_master.sh +++ b/tools/internal_ci/linux/grpc_performance_profile_master.sh @@ -20,7 +20,7 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc -tools/jenkins/run_performance_profile_hourly.sh || FAILED="true" +tools/internal_ci/linux/run_performance_profile_hourly.sh || FAILED="true" # kill port_server.py to prevent the build from hanging ps aux | grep port_server\\.py | awk '{print $2}' | xargs kill -9 diff --git a/tools/internal_ci/linux/grpc_trickle_diff.sh b/tools/internal_ci/linux/grpc_trickle_diff.sh index 624031afa7..4ed1b73f9e 100755 --- a/tools/internal_ci/linux/grpc_trickle_diff.sh +++ b/tools/internal_ci/linux/grpc_trickle_diff.sh @@ -25,7 +25,7 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc tools/run_tests/start_port_server.py -tools/jenkins/run_c_cpp_test.sh tools/profiling/microbenchmarks/bm_diff/bm_main.py \ +tools/internal_ci/linux/run_if_c_cpp_modified.sh tools/profiling/microbenchmarks/bm_diff/bm_main.py \ -d origin/$ghprbTargetBranch \ -b bm_fullstack_trickle \ -l 4 \ diff --git a/tools/internal_ci/linux/grpc_tsan_on_foundry.sh b/tools/internal_ci/linux/grpc_tsan_on_foundry.sh index aacb1ad496..fafa1ceecb 100644 --- a/tools/internal_ci/linux/grpc_tsan_on_foundry.sh +++ b/tools/internal_ci/linux/grpc_tsan_on_foundry.sh @@ -13,6 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. +export UPLOAD_TEST_RESULTS=true EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=thread --linkopt=-fsanitize=thread --test_timeout=3600" github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" - diff --git a/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh b/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh index 0f0c12db12..5c0e2df09d 100644 --- a/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh +++ b/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh @@ -23,8 +23,8 @@ mkdir -p ${KOKORO_KEYSTORE_DIR} cp ${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json ${KOKORO_KEYSTORE_DIR}/4321_grpc-testing-service temp_dir=$(mktemp -d) -ln -f "${KOKORO_GFILE_DIR}/bazel-canary" ${temp_dir}/bazel -chmod 755 "${KOKORO_GFILE_DIR}/bazel-canary" +ln -f "${KOKORO_GFILE_DIR}/bazel-release-0.12.0" ${temp_dir}/bazel +chmod 755 "${KOKORO_GFILE_DIR}/bazel-release-0.12.0" export PATH="${temp_dir}:${PATH}" # This should show ${temp_dir}/bazel which bazel @@ -50,11 +50,20 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_rc --strategy=Closure=remote \ --genrule_strategy=remote \ --experimental_strict_action_env=true \ - --experimental_remote_platform_override='properties:{name:"container-image" value:"docker://gcr.io/cloud-marketplace/google/rbe-debian8@sha256:b2d946c1ddc20af250fe85cf98bd648ac5519131659f7c36e64184b433175a33" }' \ + --experimental_remote_platform_override='properties:{name:"container-image" value:"docker://gcr.io/cloud-marketplace/google/rbe-debian8@sha256:1ede2a929b44d629ec5abe86eee6d7ffea1d5a4d247489a8867d46cfde3e38bd" }' \ --define GRPC_PORT_ISOLATED_RUNTIME=1 \ --copt=-gmlt \ --strip=never \ --copt=-fsanitize=undefined \ --linkopt=-fsanitize=undefined \ - --crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/experimental/debian8_clang/0.3.0/bazel_0.10.0/ubsan:ubsan_experimental_toolchain \ - -- //test/... + --crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/experimental/debian8_clang/0.3.0/bazel_0.12.0/ubsan:toolchain \ + -- //test/... || FAILED="true" + +# Sleep to let ResultStore finish writing results before querying +sleep 60 +python ./tools/run_tests/python_utils/upload_rbe_results.py + +if [ "$FAILED" != "" ] +then + exit 1 +fi diff --git a/tools/internal_ci/linux/pull_request/grpc_bazel_on_foundry_dbg.sh b/tools/internal_ci/linux/pull_request/grpc_bazel_on_foundry_dbg.sh index 3ff569df47..8e2aaebaee 100644 --- a/tools/internal_ci/linux/pull_request/grpc_bazel_on_foundry_dbg.sh +++ b/tools/internal_ci/linux/pull_request/grpc_bazel_on_foundry_dbg.sh @@ -15,4 +15,3 @@ EXTRA_FLAGS="-c dbg --test_timeout=300,450,1200,3600" github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" - diff --git a/tools/internal_ci/linux/pull_request/grpc_interop_toprod.cfg b/tools/internal_ci/linux/pull_request/grpc_interop_toprod.cfg index e141d9f648..d14c79a1f6 100644 --- a/tools/internal_ci/linux/pull_request/grpc_interop_toprod.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_interop_toprod.cfg @@ -26,5 +26,5 @@ action { env_vars { key: "RUN_TESTS_FLAGS" - value: "-l all --allow_flakes --cloud_to_prod --cloud_to_prod_auth --prod_servers default gateway_v4 --use_docker --internal_ci -t -j 12" + value: "-l all --cloud_to_prod --cloud_to_prod_auth --prod_servers default gateway_v4 --use_docker --internal_ci -t -j 12" } diff --git a/tools/internal_ci/linux/run_if_c_cpp_modified.sh b/tools/internal_ci/linux/run_if_c_cpp_modified.sh new file mode 100755 index 0000000000..736d759423 --- /dev/null +++ b/tools/internal_ci/linux/run_if_c_cpp_modified.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This script is invoked by a pull request job and executes all +# args passed to this script if the pull request affect C/C++ code +set -ex + +# Enter the gRPC repo root +cd $(dirname $0)/../../.. + +# TODO(jtattermusch): the "ghprbTargetBranch" is Jenkins specific and probably +# does not work on kokoro? +AFFECTS_C_CPP=`python -c 'import os; \ + import sys; \ + sys.path.insert(0, "tools/run_tests/python_utils"); \ + import filter_pull_request_tests as filter; \ + github_target_branch = os.environ.get("ghprbTargetBranch"); \ + print(filter.affects_c_cpp("origin/%s" % github_target_branch))'` + +if [ $AFFECTS_C_CPP == "False" ] ; then + echo "This pull request does not affect C/C++. Tests do not need to be run." +else + $@ +fi diff --git a/tools/internal_ci/linux/run_performance_profile_daily.sh b/tools/internal_ci/linux/run_performance_profile_daily.sh new file mode 100755 index 0000000000..45c7a99805 --- /dev/null +++ b/tools/internal_ci/linux/run_performance_profile_daily.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Copyright 2016 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +cd $(dirname $0)/../../.. + +# try to use pypy for generating reports +# each trace dumps 7-8gig of text to disk, and processing this into a report is +# heavyweight - so any speed boost is worthwhile +# TODO(ctiller): consider rewriting report generation in C++ for performance +if which pypy >/dev/null; then + PYTHON=pypy +else + PYTHON=python2.7 +fi + +BENCHMARKS_TO_RUN="bm_fullstack_unary_ping_pong bm_fullstack_streaming_ping_pong bm_fullstack_streaming_pump bm_closure bm_cq bm_call_create bm_error bm_chttp2_hpack bm_chttp2_transport bm_pollset bm_metadata" + +./tools/run_tests/start_port_server.py || true + +$PYTHON tools/run_tests/run_microbenchmark.py --collect summary perf latency -b $BENCHMARKS_TO_RUN diff --git a/tools/internal_ci/linux/run_performance_profile_hourly.sh b/tools/internal_ci/linux/run_performance_profile_hourly.sh new file mode 100755 index 0000000000..edf85c2e2c --- /dev/null +++ b/tools/internal_ci/linux/run_performance_profile_hourly.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2016 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +cd $(dirname $0)/../../.. + +./tools/run_tests/start_port_server.py || true + +CPUS=`python -c 'import multiprocessing; print multiprocessing.cpu_count()'` + +make CONFIG=opt memory_profile_test memory_profile_client memory_profile_server -j $CPUS +bins/opt/memory_profile_test +bq load microbenchmarks.memory memory_usage.csv + +tools/run_tests/run_microbenchmark.py --collect summary --bigquery_upload diff --git a/tools/internal_ci/windows/grpc_build_artifacts.bat b/tools/internal_ci/windows/grpc_build_artifacts.bat index c6e523697c..4d528e0128 100644 --- a/tools/internal_ci/windows/grpc_build_artifacts.bat +++ b/tools/internal_ci/windows/grpc_build_artifacts.bat @@ -24,8 +24,9 @@ cd /d %~dp0\..\..\.. call tools/internal_ci/helper_scripts/prepare_build_windows.bat -python tools/run_tests/task_runner.py -f artifact windows -j 4 || goto :error -goto :EOF +python tools/run_tests/task_runner.py -f artifact windows -j 4 +set RUNTESTS_EXITCODE=%errorlevel% -:error -exit /b %errorlevel% +bash tools/internal_ci/helper_scripts/delete_nonartifacts.sh + +exit /b %RUNTESTS_EXITCODE% diff --git a/tools/internal_ci/windows/grpc_build_packages.bat b/tools/internal_ci/windows/grpc_build_packages.bat index 23ba365e40..61a636f607 100644 --- a/tools/internal_ci/windows/grpc_build_packages.bat +++ b/tools/internal_ci/windows/grpc_build_packages.bat @@ -28,8 +28,9 @@ call tools/internal_ci/helper_scripts/prepare_build_windows.bat powershell -Command "mv %KOKORO_GFILE_DIR%\github\grpc\artifacts input_artifacts" dir input_artifacts -python tools/run_tests/task_runner.py -f package windows -j 4 || goto :error -goto :EOF +python tools/run_tests/task_runner.py -f package windows -j 4 +set RUNTESTS_EXITCODE=%errorlevel% -:error -exit /b %errorlevel% +bash tools/internal_ci/helper_scripts/delete_nonartifacts.sh + +exit /b %RUNTESTS_EXITCODE% diff --git a/tools/internal_ci/windows/grpc_distribtests.bat b/tools/internal_ci/windows/grpc_distribtests.bat index d6d50b684f..65fd01ef72 100644 --- a/tools/internal_ci/windows/grpc_distribtests.bat +++ b/tools/internal_ci/windows/grpc_distribtests.bat @@ -28,8 +28,9 @@ call tools/internal_ci/helper_scripts/prepare_build_windows.bat powershell -Command "mv %KOKORO_GFILE_DIR%\github\grpc\artifacts input_artifacts" dir input_artifacts -python tools/run_tests/task_runner.py -f distribtest windows -j 4 || goto :error -goto :EOF +python tools/run_tests/task_runner.py -f distribtest windows -j 4 +set RUNTESTS_EXITCODE=%errorlevel% -:error -exit /b %errorlevel% +bash tools/internal_ci/helper_scripts/delete_nonartifacts.sh + +exit /b %RUNTESTS_EXITCODE% diff --git a/tools/internal_ci/windows/grpc_distribtests_standalone.bat b/tools/internal_ci/windows/grpc_distribtests_standalone.bat index 3eb33b1548..ea4c0d7a97 100644 --- a/tools/internal_ci/windows/grpc_distribtests_standalone.bat +++ b/tools/internal_ci/windows/grpc_distribtests_standalone.bat @@ -24,8 +24,9 @@ cd /d %~dp0\..\..\.. call tools/internal_ci/helper_scripts/prepare_build_windows.bat -python tools/run_tests/task_runner.py -f distribtest windows cpp -j 4 || goto :error -goto :EOF +python tools/run_tests/task_runner.py -f distribtest windows cpp -j 4 +set RUNTESTS_EXITCODE=%errorlevel% -:error -exit /b %errorlevel% +bash tools/internal_ci/helper_scripts/delete_nonartifacts.sh + +exit /b %RUNTESTS_EXITCODE% diff --git a/tools/internal_ci/windows/grpc_run_tests_matrix.bat b/tools/internal_ci/windows/grpc_run_tests_matrix.bat index 10627add25..c6277d0929 100644 --- a/tools/internal_ci/windows/grpc_run_tests_matrix.bat +++ b/tools/internal_ci/windows/grpc_run_tests_matrix.bat @@ -20,7 +20,6 @@ call tools/internal_ci/helper_scripts/prepare_build_windows.bat python tools/run_tests/run_tests_matrix.py %RUN_TESTS_FLAGS% set RUNTESTS_EXITCODE=%errorlevel% -@rem Reveal leftover processes that might be left behind by the build -tasklist /V +bash tools/internal_ci/helper_scripts/delete_nonartifacts.sh exit /b %RUNTESTS_EXITCODE% |