diff options
Diffstat (limited to 'tools/internal_ci')
113 files changed, 910 insertions, 153 deletions
diff --git a/tools/internal_ci/helper_scripts/delete_nonartifacts.sh b/tools/internal_ci/helper_scripts/delete_nonartifacts.sh new file mode 100755 index 0000000000..01e9427e1c --- /dev/null +++ b/tools/internal_ci/helper_scripts/delete_nonartifacts.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# change to grpc repo root +cd "$(dirname "$0")/../../.." + +# After kokoro build finishes, the workspace gets rsync'ed to another machine, +# from where the artifacts and reports are processed. +# Especially on Windows, the rsync can take long time, so we cleanup the workspace +# after finishing each build. We only leave files we want to keep: +# - reports and artifacts +# - directory containing the kokoro scripts to prevent deleting a script while being executed. +time find . -type f -not -iname "*sponge_log.*" -not -path "./reports/*" -not -path "./artifacts/*" -not -path "./tools/internal_ci/*" -exec rm -f {} + diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc index b0feeef363..43bc9609c7 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc @@ -17,17 +17,6 @@ # builds. This rc script must be used in the root directory of gRPC # and is expected to be used before prepare_build_macos_rc -export CONFIG=opt - -# Move gRPC repo to directory that Docker for Mac has drive access to -mkdir /Users/kbuilder/workspace -cp -R ./ /Users/kbuilder/workspace/grpc -cd /Users/kbuilder/workspace/grpc - -# Needed for identifying Docker image sha1 -brew update -brew install md5sha1sum - # Set up gRPC-Go and gRPC-Java to test git clone --recursive https://github.com/grpc/grpc-go ./../grpc-go git clone --recursive https://github.com/grpc/grpc-java ./../grpc-java diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc index d2b77691d4..3212e80854 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc @@ -34,15 +34,22 @@ sudo systemsetup -setusingnetworktime on date # Add GCP credentials for BQ access -pip install google-api-python-client --user python +# pin google-api-python-client to avoid https://github.com/grpc/grpc/issues/15600 +pip install google-api-python-client==1.6.7 --user python export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json # If this is a PR using RUN_TESTS_FLAGS var, then add flags to filter tests -if [ -n "$KOKORO_GITHUB_PULL_REQUEST_NUMBER" ] && [ -n "$RUN_TESTS_FLAGS" ]; then +if [ -n "$KOKORO_GITHUB_PULL_REQUEST_NUMBER" ]; then + set +x brew update brew install jq ghprbTargetBranch=$(curl -s https://api.github.com/repos/grpc/grpc/pulls/$KOKORO_GITHUB_PULL_REQUEST_NUMBER | jq -r .base.ref) export RUN_TESTS_FLAGS="$RUN_TESTS_FLAGS --filter_pr_tests --base_branch origin/$ghprbTargetBranch" + + # TODO(matt-kwong): rename this to GITHUB_OAUTH_TOKEN after Jenkins deprecation + export JENKINS_OAUTH_TOKEN=$(cat ${KOKORO_GFILE_DIR}/oauth_token.txt) + export ghprbPullId=$KOKORO_GITHUB_PULL_REQUEST_NUMBER + set -x fi set +ex # rvm script is very verbose and exits with errorcode @@ -66,6 +73,10 @@ pip install virtualenv --user python pip install -U Mako six tox setuptools twisted pyyaml --user python export PYTHONPATH=/Library/Python/3.4/site-packages +# Install Python 3.7 +curl -O https://www.python.org/ftp/python/3.7.0/python-3.7.0-macosx10.9.pkg +sudo installer -pkg ./python-3.7.0-macosx10.9.pkg -target / + # set xcode version for Obj-C tests sudo xcode-select -switch /Applications/Xcode_9.2.app/Contents/Developer/ @@ -78,3 +89,10 @@ export DOTNET_CLI_TELEMETRY_OPTOUT=true date git submodule update --init + +# Store intermediate build files of ObjC tests into /tmpfs +mkdir /tmpfs/Build-ios-binary-size +ln -s /tmpfs/Build-ios-binary-size src/objective-c/examples/Sample/Build +mkdir /tmpfs/DerivedData +rm -rf ~/Library/Developer/Xcode/DerivedData +ln -s /tmpfs/DerivedData ~/Library/Developer/Xcode/DerivedData diff --git a/tools/internal_ci/linux/grpc_android.sh b/tools/internal_ci/linux/grpc_android.sh index 2220145dac..42c7f5fb04 100755 --- a/tools/internal_ci/linux/grpc_android.sh +++ b/tools/internal_ci/linux/grpc_android.sh @@ -25,13 +25,6 @@ git submodule update --init # Build protoc and grpc_cpp_plugin. Codegen is not cross-compiled to Android make HAS_SYSTEM_PROTOBUF=false -# TODO(ericgribkoff) Remove when this commit (already in master) is included in -# next protobuf release -cd third_party/protobuf -git fetch -git cherry-pick 7daa320065f3bea2b54bf983337d1724f153422d -m 1 - - # Build and run interop instrumentation tests on Firebase Test Lab cd "${REPO_ROOT}/src/android/test/interop/" diff --git a/tools/internal_ci/linux/grpc_asan_on_foundry.sh b/tools/internal_ci/linux/grpc_asan_on_foundry.sh index 791c56c086..dfef004a60 100644..100755 --- a/tools/internal_ci/linux/grpc_asan_on_foundry.sh +++ b/tools/internal_ci/linux/grpc_asan_on_foundry.sh @@ -14,6 +14,7 @@ # limitations under the License. export UPLOAD_TEST_RESULTS=true -EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=address --linkopt=-fsanitize=address --test_timeout=3600" -github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" +EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=address --linkopt=-fsanitize=address --test_timeout=3600 --cache_test_results=no" +EXCLUDE_TESTS="--test_tag_filters=-qps_json_driver,-json_run_localhost" +github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" "${EXCLUDE_TESTS}" diff --git a/tools/internal_ci/linux/grpc_basictests_c_cpp_dbg.cfg b/tools/internal_ci/linux/grpc_basictests_c_cpp_dbg.cfg index 4a0badf43b..07f7f0c659 100644 --- a/tools/internal_ci/linux/grpc_basictests_c_cpp_dbg.cfg +++ b/tools/internal_ci/linux/grpc_basictests_c_cpp_dbg.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_basictests_c_cpp_opt.cfg b/tools/internal_ci/linux/grpc_basictests_c_cpp_opt.cfg index a2cfe021e1..8f2813febf 100644 --- a/tools/internal_ci/linux/grpc_basictests_c_cpp_opt.cfg +++ b/tools/internal_ci/linux/grpc_basictests_c_cpp_opt.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_basictests_multilang.cfg b/tools/internal_ci/linux/grpc_basictests_multilang.cfg index 4433d14cd7..f8a5a4aea5 100644 --- a/tools/internal_ci/linux/grpc_basictests_multilang.cfg +++ b/tools/internal_ci/linux/grpc_basictests_multilang.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_bazel_build.cfg b/tools/internal_ci/linux/grpc_bazel_build.cfg index 4d3a2ade5d..59f5da9a67 100644 --- a/tools/internal_ci/linux/grpc_bazel_build.cfg +++ b/tools/internal_ci/linux/grpc_bazel_build.cfg @@ -19,5 +19,5 @@ build_file: "grpc/tools/internal_ci/linux/grpc_bazel.sh" timeout_mins: 60 env_vars { key: "BAZEL_SCRIPT" - value: "tools/jenkins/run_bazel_basic_in_docker.sh" + value: "tools/internal_ci/linux/grpc_bazel_build_in_docker.sh" } diff --git a/tools/internal_ci/linux/grpc_bazel_build_in_docker.sh b/tools/internal_ci/linux/grpc_bazel_build_in_docker.sh new file mode 100755 index 0000000000..24598f43f0 --- /dev/null +++ b/tools/internal_ci/linux/grpc_bazel_build_in_docker.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Test basic Bazel features +# +# NOTE: No empty lines should appear in this file before igncr is set! +set -ex -o igncr || set -ex + +mkdir -p /var/local/git +git clone /var/local/jenkins/grpc /var/local/git/grpc +(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \ +&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \ +${name}') +cd /var/local/git/grpc +bazel build --spawn_strategy=standalone --genrule_strategy=standalone :all test/... examples/... diff --git a/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh b/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh index 7881e3a7fb..bb2a85138c 100755 --- a/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh +++ b/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh @@ -22,8 +22,8 @@ mkdir -p ${KOKORO_KEYSTORE_DIR} cp ${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json ${KOKORO_KEYSTORE_DIR}/4321_grpc-testing-service temp_dir=$(mktemp -d) -ln -f "${KOKORO_GFILE_DIR}/bazel-release-0.12.0" ${temp_dir}/bazel -chmod 755 "${KOKORO_GFILE_DIR}/bazel-release-0.12.0" +ln -f "${KOKORO_GFILE_DIR}/bazel-latest-release" ${temp_dir}/bazel +chmod 755 "${KOKORO_GFILE_DIR}/bazel-latest-release" export PATH="${temp_dir}:${PATH}" # This should show ${temp_dir}/bazel which bazel @@ -34,6 +34,8 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_rc +export KOKORO_FOUNDRY_PROJECT_ID="projects/grpc-testing/instances/default_instance" + # TODO(adelez): implement size for test targets and change test_timeout back "${KOKORO_GFILE_DIR}/bazel_wrapper.py" \ --host_jvm_args=-Dbazel.DigestFunction=SHA256 \ @@ -49,10 +51,16 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_rc --strategy=Closure=remote \ --genrule_strategy=remote \ --experimental_strict_action_env=true \ - --experimental_remote_platform_override='properties:{name:"container-image" value:"docker://gcr.io/cloud-marketplace/google/rbe-debian8@sha256:1ede2a929b44d629ec5abe86eee6d7ffea1d5a4d247489a8867d46cfde3e38bd" }' \ - --crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/debian8_clang/0.3.0/bazel_0.10.0:toolchain \ + --crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.0/bazel_0.16.1/default:toolchain \ --define GRPC_PORT_ISOLATED_RUNTIME=1 \ - $1 \ + --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 \ + --extra_toolchains=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.0/bazel_0.16.1/cpp:cc-toolchain-clang-x86_64-default \ + --extra_execution_platforms=//third_party/toolchains:rbe_ubuntu1604 \ + --host_platform=//third_party/toolchains:rbe_ubuntu1604 \ + --platforms=//third_party/toolchains:rbe_ubuntu1604 \ + --test_env=GRPC_VERBOSITY=debug \ + --remote_instance_name=projects/grpc-testing/instances/default_instance \ + $@ \ -- //test/... || FAILED="true" if [ "$UPLOAD_TEST_RESULTS" != "" ] diff --git a/tools/internal_ci/linux/grpc_bazel_on_foundry_dbg.sh b/tools/internal_ci/linux/grpc_bazel_on_foundry_dbg.sh index 192d9d1b2d..51cb66f5b8 100644 --- a/tools/internal_ci/linux/grpc_bazel_on_foundry_dbg.sh +++ b/tools/internal_ci/linux/grpc_bazel_on_foundry_dbg.sh @@ -14,5 +14,5 @@ # limitations under the License. export UPLOAD_TEST_RESULTS=true -EXTRA_FLAGS="-c dbg --test_timeout=300,450,1200,3600 --runs_per_test_detects_flakes --runs_per_test=2" +EXTRA_FLAGS="-c dbg --test_timeout=300,450,1200,3600 --cache_test_results=no" github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" diff --git a/tools/internal_ci/linux/grpc_bazel_on_foundry_opt.sh b/tools/internal_ci/linux/grpc_bazel_on_foundry_opt.sh index 6fb3c77892..cbba9067ad 100644 --- a/tools/internal_ci/linux/grpc_bazel_on_foundry_opt.sh +++ b/tools/internal_ci/linux/grpc_bazel_on_foundry_opt.sh @@ -14,5 +14,5 @@ # limitations under the License. export UPLOAD_TEST_RESULTS=true -EXTRA_FLAGS="-c opt --test_timeout=300,450,1200,3600 --runs_per_test_detects_flakes --runs_per_test=2" +EXTRA_FLAGS="-c opt --test_timeout=300,450,1200,3600 --cache_test_results=no" github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" diff --git a/tools/internal_ci/linux/grpc_bazel_test.cfg b/tools/internal_ci/linux/grpc_bazel_test.cfg index 46198b9600..6b03be9518 100644 --- a/tools/internal_ci/linux/grpc_bazel_test.cfg +++ b/tools/internal_ci/linux/grpc_bazel_test.cfg @@ -19,5 +19,5 @@ build_file: "grpc/tools/internal_ci/linux/grpc_bazel.sh" timeout_mins: 240 env_vars { key: "BAZEL_SCRIPT" - value: "tools/jenkins/run_bazel_full_in_docker.sh" + value: "tools/internal_ci/linux/grpc_bazel_test_in_docker.sh" } diff --git a/tools/internal_ci/linux/grpc_bazel_test_in_docker.sh b/tools/internal_ci/linux/grpc_bazel_test_in_docker.sh new file mode 100755 index 0000000000..9663debe69 --- /dev/null +++ b/tools/internal_ci/linux/grpc_bazel_test_in_docker.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Test full Bazel +# +# NOTE: No empty lines should appear in this file before igncr is set! +set -ex -o igncr || set -ex + +mkdir -p /var/local/git +git clone /var/local/jenkins/grpc /var/local/git/grpc +(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \ +&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \ +${name}') +cd /var/local/git/grpc/test +bazel test --spawn_strategy=standalone --genrule_strategy=standalone ... diff --git a/tools/internal_ci/linux/grpc_build_artifacts.cfg b/tools/internal_ci/linux/grpc_build_artifacts.cfg index 88fc6b7b35..1e04a1e788 100644 --- a/tools/internal_ci/linux/grpc_build_artifacts.cfg +++ b/tools/internal_ci/linux/grpc_build_artifacts.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_build_artifacts.sh" timeout_mins: 120 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" regex: "github/grpc/artifacts/**" } diff --git a/tools/internal_ci/linux/grpc_build_artifacts_extra.cfg b/tools/internal_ci/linux/grpc_build_artifacts_extra.cfg index 619e3ea3a9..2737e2f345 100644 --- a/tools/internal_ci/linux/grpc_build_artifacts_extra.cfg +++ b/tools/internal_ci/linux/grpc_build_artifacts_extra.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_build_artifacts_extra.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" regex: "github/grpc/artifacts/**" } diff --git a/tools/internal_ci/linux/grpc_build_artifacts_extra_release.cfg b/tools/internal_ci/linux/grpc_build_artifacts_extra_release.cfg new file mode 100644 index 0000000000..2737e2f345 --- /dev/null +++ b/tools/internal_ci/linux/grpc_build_artifacts_extra_release.cfg @@ -0,0 +1,26 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_build_artifacts_extra.sh" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} diff --git a/tools/internal_ci/linux/grpc_build_boringssl_at_head.cfg b/tools/internal_ci/linux/grpc_build_boringssl_at_head.cfg index 11c211fd2b..9a430db0f9 100644 --- a/tools/internal_ci/linux/grpc_build_boringssl_at_head.cfg +++ b/tools/internal_ci/linux/grpc_build_boringssl_at_head.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_build_submodule_at_head.sh" timeout_mins: 180 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_build_packages.cfg b/tools/internal_ci/linux/grpc_build_packages.cfg index 6a4a163dfc..23a676cf72 100644 --- a/tools/internal_ci/linux/grpc_build_packages.cfg +++ b/tools/internal_ci/linux/grpc_build_packages.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_build_packages.sh" timeout_mins: 120 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" regex: "github/grpc/artifacts/**" } diff --git a/tools/internal_ci/linux/grpc_build_protobuf_at_head.cfg b/tools/internal_ci/linux/grpc_build_protobuf_at_head.cfg index 2f08e15e63..aef6a7a1dc 100644 --- a/tools/internal_ci/linux/grpc_build_protobuf_at_head.cfg +++ b/tools/internal_ci/linux/grpc_build_protobuf_at_head.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_build_submodule_at_head.sh" timeout_mins: 180 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_build_submodule_at_head.sh b/tools/internal_ci/linux/grpc_build_submodule_at_head.sh index e68a2897f6..f4600c5593 100755 --- a/tools/internal_ci/linux/grpc_build_submodule_at_head.sh +++ b/tools/internal_ci/linux/grpc_build_submodule_at_head.sh @@ -22,9 +22,24 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_rc -# Update submodule and commit it so changes are passed to Docker +# Update submodule to be tested at HEAD (cd third_party/$RUN_TESTS_FLAGS && git fetch --all && git checkout origin/master) tools/buildgen/generate_projects.sh + +if [ "$RUN_TESTS_FLAGS" == "protobuf" ] +then + # Upgrade bazel. + # make_grpcio_tools.py requires bazel >=0.13.1 to run (Kokoro workers only have bazel 0.9) + curl -fSsL -O https://github.com/bazelbuild/bazel/releases/download/0.13.1/bazel-0.13.1-installer-linux-x86_64.sh + chmod +x ./bazel-0.13.1-installer-linux-x86_64.sh + ./bazel-0.13.1-installer-linux-x86_64.sh --user + rm -f ./bazel-0.13.1-installer-linux-x86_64.sh + export PATH="$PATH:$HOME/bin" + + tools/distrib/python/make_grpcio_tools.py +fi + +# commit so that changes are passed to Docker git -c user.name='foo' -c user.email='foo@google.com' commit -a -m 'Update submodule' tools/run_tests/run_tests_matrix.py -f linux --inner_jobs 4 -j 4 --internal_ci --build_only diff --git a/tools/internal_ci/linux/grpc_coverage.cfg b/tools/internal_ci/linux/grpc_coverage.cfg index 794a51d3f1..6eb37b7dec 100644 --- a/tools/internal_ci/linux/grpc_coverage.cfg +++ b/tools/internal_ci/linux/grpc_coverage.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_coverage.sh" timeout_mins: 420 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_coverage.sh b/tools/internal_ci/linux/grpc_coverage.sh index fd927a85fd..97166372ab 100755 --- a/tools/internal_ci/linux/grpc_coverage.sh +++ b/tools/internal_ci/linux/grpc_coverage.sh @@ -26,4 +26,16 @@ python tools/run_tests/run_tests.py \ -l all \ -c gcov \ -x sponge_log.xml \ - -j 16 + -j 16 || FAILED="true" + +# HTML reports can't be easily displayed in GCS, so create a zip archive +# and put it under reports directory to get it uploaded as an artifact. +zip -q -r coverage_report.zip reports || true +rm -rf reports || true +mkdir reports || true +mv coverage_report.zip reports || true + +if [ "$FAILED" != "" ] +then + exit 1 +fi diff --git a/tools/internal_ci/linux/grpc_distribtests.cfg b/tools/internal_ci/linux/grpc_distribtests.cfg index 0f1d79355a..848d571333 100644 --- a/tools/internal_ci/linux/grpc_distribtests.cfg +++ b/tools/internal_ci/linux/grpc_distribtests.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_distribtests.sh" timeout_mins: 120 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" regex: "github/grpc/artifacts/**" } diff --git a/tools/internal_ci/linux/grpc_distribtests_standalone.cfg b/tools/internal_ci/linux/grpc_distribtests_standalone.cfg index bc6c8e8f80..734bdfd78a 100644 --- a/tools/internal_ci/linux/grpc_distribtests_standalone.cfg +++ b/tools/internal_ci/linux/grpc_distribtests_standalone.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_standalone.sh" timeout_mins: 120 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" regex: "github/grpc/artifacts/**" } diff --git a/tools/internal_ci/linux/grpc_full_performance_master.cfg b/tools/internal_ci/linux/grpc_full_performance_master.cfg index 8852130a13..8ad93ee052 100644 --- a/tools/internal_ci/linux/grpc_full_performance_master.cfg +++ b/tools/internal_ci/linux/grpc_full_performance_master.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_full_performance_master.sh" timeout_mins: 600 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "**/perf_reports/**" } } diff --git a/tools/internal_ci/linux/grpc_full_performance_master.sh b/tools/internal_ci/linux/grpc_full_performance_master.sh index 4eddc18731..24ee71edd1 100755 --- a/tools/internal_ci/linux/grpc_full_performance_master.sh +++ b/tools/internal_ci/linux/grpc_full_performance_master.sh @@ -21,7 +21,7 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_perf_multilang_rc # run 8core client vs 8core server tools/run_tests/run_performance_tests.py \ - -l c++ csharp ruby java python go php7 php7_protobuf_c \ + -l c++ csharp ruby java python go php7 php7_protobuf_c node node_purejs \ --netperf \ --category scalable \ --remote_worker_host grpc-kokoro-performance-server-8core grpc-kokoro-performance-client-8core grpc-kokoro-performance-client2-8core \ diff --git a/tools/internal_ci/linux/grpc_full_performance_release.cfg b/tools/internal_ci/linux/grpc_full_performance_release.cfg index e9a4bcdcaf..11a95e889f 100644 --- a/tools/internal_ci/linux/grpc_full_performance_release.cfg +++ b/tools/internal_ci/linux/grpc_full_performance_release.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_full_performance_release.sh" timeout_mins: 600 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "**/perf_reports/**" } } diff --git a/tools/internal_ci/linux/grpc_interop_alts.cfg b/tools/internal_ci/linux/grpc_interop_alts.cfg index bda76faf44..4684aba96b 100644 --- a/tools/internal_ci/linux/grpc_interop_alts.cfg +++ b/tools/internal_ci/linux/grpc_interop_alts.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_interop_tests.sh" timeout_mins: 60 action { define_artifacts { - regex: "**/sponge_log.xml" + regex: "**/sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_interop_matrix.cfg b/tools/internal_ci/linux/grpc_interop_matrix.cfg index 71e930e0b0..696a55c0df 100644 --- a/tools/internal_ci/linux/grpc_interop_matrix.cfg +++ b/tools/internal_ci/linux/grpc_interop_matrix.cfg @@ -17,10 +17,10 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/linux/grpc_interop_matrix.sh" # grpc_interop tests can take 1 hours to complete. -timeout_mins: 120 +timeout_mins: 300 action { define_artifacts { - regex: "**/sponge_log.xml" + regex: "**/sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_interop_tocloud.cfg b/tools/internal_ci/linux/grpc_interop_tocloud.cfg index 13aec15770..9b35adcece 100644 --- a/tools/internal_ci/linux/grpc_interop_tocloud.cfg +++ b/tools/internal_ci/linux/grpc_interop_tocloud.cfg @@ -19,12 +19,12 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_interop_tests.sh" timeout_mins: 60 action { define_artifacts { - regex: "**/sponge_log.xml" + regex: "**/sponge_log.*" regex: "github/grpc/reports/**" } } env_vars { key: "RUN_TESTS_FLAGS" - value: "-l all -s all --use_docker --http2_interop --internal_ci -t -j 12 --bq_result_table interop_results" + value: "-l all -s all --use_docker --http2_interop --internal_ci -t -j 8 --bq_result_table interop_results" } diff --git a/tools/internal_ci/linux/grpc_interop_toprod.cfg b/tools/internal_ci/linux/grpc_interop_toprod.cfg index 8d025c4f60..de4db81e6b 100644 --- a/tools/internal_ci/linux/grpc_interop_toprod.cfg +++ b/tools/internal_ci/linux/grpc_interop_toprod.cfg @@ -19,12 +19,12 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_interop_tests.sh" timeout_mins: 60 action { define_artifacts { - regex: "**/sponge_log.xml" + regex: "**/sponge_log.*" regex: "github/grpc/reports/**" } } env_vars { key: "RUN_TESTS_FLAGS" - value: "-l all --cloud_to_prod --cloud_to_prod_auth --prod_servers default gateway_v4 --use_docker --internal_ci -t -j 12 --bq_result_table interop_results" + value: "-l all --cloud_to_prod --cloud_to_prod_auth --prod_servers default gateway_v4 --use_docker --internal_ci -t -j 8 --bq_result_table interop_results" } diff --git a/tools/internal_ci/linux/grpc_microbenchmark_diff.sh b/tools/internal_ci/linux/grpc_microbenchmark_diff.sh index 45add1b02d..9834aaa053 100755 --- a/tools/internal_ci/linux/grpc_microbenchmark_diff.sh +++ b/tools/internal_ci/linux/grpc_microbenchmark_diff.sh @@ -25,9 +25,9 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc tools/run_tests/start_port_server.py -tools/jenkins/run_c_cpp_test.sh tools/profiling/bloat/bloat_diff.py \ +tools/internal_ci/linux/run_if_c_cpp_modified.sh tools/profiling/bloat/bloat_diff.py \ -d origin/$ghprbTargetBranch || FAILED="true" -tools/jenkins/run_c_cpp_test.sh tools/profiling/microbenchmarks/bm_diff/bm_main.py \ +tools/internal_ci/linux/run_if_c_cpp_modified.sh tools/profiling/microbenchmarks/bm_diff/bm_main.py \ -d origin/$ghprbTargetBranch \ -b $BENCHMARKS_TO_RUN || FAILED="true" diff --git a/tools/internal_ci/linux/grpc_msan_on_foundry.sh b/tools/internal_ci/linux/grpc_msan_on_foundry.sh index 5e644793ad..1ef13ef0d4 100644 --- a/tools/internal_ci/linux/grpc_msan_on_foundry.sh +++ b/tools/internal_ci/linux/grpc_msan_on_foundry.sh @@ -23,8 +23,8 @@ mkdir -p ${KOKORO_KEYSTORE_DIR} cp ${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json ${KOKORO_KEYSTORE_DIR}/4321_grpc-testing-service temp_dir=$(mktemp -d) -ln -f "${KOKORO_GFILE_DIR}/bazel-release-0.12.0" ${temp_dir}/bazel -chmod 755 "${KOKORO_GFILE_DIR}/bazel-release-0.12.0" +ln -f "${KOKORO_GFILE_DIR}/bazel-latest-release" ${temp_dir}/bazel +chmod 755 "${KOKORO_GFILE_DIR}/bazel-latest-release" export PATH="${temp_dir}:${PATH}" # This should show ${temp_dir}/bazel which bazel @@ -35,6 +35,8 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_rc +export KOKORO_FOUNDRY_PROJECT_ID="projects/grpc-testing/instances/default_instance" + "${KOKORO_GFILE_DIR}/bazel_wrapper.py" \ --host_jvm_args=-Dbazel.DigestFunction=SHA256 \ test --jobs="200" \ @@ -50,7 +52,6 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_rc --strategy=Closure=remote \ --genrule_strategy=remote \ --experimental_strict_action_env=true \ - --experimental_remote_platform_override='properties:{name:"container-image" value:"docker://gcr.io/asci-toolchain/nosla-debian8-clang-msan@sha256:8f381d55c0456fb65821c90ada902c2584977e03a1eaca8fba8ce77e644c775b" }' \ --define GRPC_PORT_ISOLATED_RUNTIME=1 \ --copt=-gmlt \ --strip=never \ @@ -59,8 +60,15 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_rc --linkopt=-fsanitize=memory \ --copt=-fsanitize-memory-track-origins \ --action_env=LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH \ - --host_crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/debian8_clang/0.3.0/bazel_0.10.0:toolchain \ - --crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/experimental/debian8_clang/0.3.0/bazel_0.10.0/msan:msan_experimental_toolchain \ + --host_crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.0/bazel_0.16.1/default:toolchain \ + --crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.0/bazel_0.16.1/msan:toolchain \ + --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 \ + --extra_toolchains=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.0/bazel_0.16.1/cpp:cc-toolchain-clang-x86_64-default \ + --extra_execution_platforms=//third_party/toolchains:rbe_ubuntu1604 \ + --host_platform=//third_party/toolchains:rbe_ubuntu1604 \ + --platforms=//third_party/toolchains:rbe_ubuntu1604 \ + --test_env=GRPC_VERBOSITY=debug \ + --remote_instance_name=projects/grpc-testing/instances/default_instance \ -- //test/... || FAILED="true" # Sleep to let ResultStore finish writing results before querying diff --git a/tools/internal_ci/linux/grpc_performance_profile_master.sh b/tools/internal_ci/linux/grpc_performance_profile_master.sh index 40bbfe89dc..fbff17466b 100755 --- a/tools/internal_ci/linux/grpc_performance_profile_master.sh +++ b/tools/internal_ci/linux/grpc_performance_profile_master.sh @@ -20,7 +20,7 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc -tools/jenkins/run_performance_profile_hourly.sh || FAILED="true" +tools/internal_ci/linux/run_performance_profile_hourly.sh || FAILED="true" # kill port_server.py to prevent the build from hanging ps aux | grep port_server\\.py | awk '{print $2}' | xargs kill -9 diff --git a/tools/internal_ci/linux/grpc_portability.cfg b/tools/internal_ci/linux/grpc_portability.cfg index 76e5028477..f417f24bb1 100644 --- a/tools/internal_ci/linux/grpc_portability.cfg +++ b/tools/internal_ci/linux/grpc_portability.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 1440 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_portability_build_only.cfg b/tools/internal_ci/linux/grpc_portability_build_only.cfg index 4acd9353fb..fab9dde3e5 100644 --- a/tools/internal_ci/linux/grpc_portability_build_only.cfg +++ b/tools/internal_ci/linux/grpc_portability_build_only.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 180 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_publish_packages.cfg b/tools/internal_ci/linux/grpc_publish_packages.cfg new file mode 100644 index 0000000000..dc9fe7d0a7 --- /dev/null +++ b/tools/internal_ci/linux/grpc_publish_packages.cfg @@ -0,0 +1,26 @@ +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_publish_packages.sh" +timeout_mins: 120 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} diff --git a/tools/internal_ci/linux/grpc_publish_packages.sh b/tools/internal_ci/linux/grpc_publish_packages.sh new file mode 100755 index 0000000000..14492301cc --- /dev/null +++ b/tools/internal_ci/linux/grpc_publish_packages.sh @@ -0,0 +1,235 @@ +#!/bin/bash +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +shopt -s nullglob + +cd "$(dirname "$0")/../../.." + +GRPC_VERSION=$(grep -e "^ *version: " build.yaml | head -n 1 | sed 's/.*: //') + +INPUT_ARTIFACTS=$KOKORO_GFILE_DIR/github/grpc/artifacts +INDEX_FILENAME=index.xml + +BUILD_ID=${KOKORO_BUILD_ID:-$(uuidgen)} +BUILD_BRANCH_NAME=master +BUILD_GIT_COMMIT=${KOKORO_GIT_COMMIT:-unknown} +BUILD_TIMESTAMP=$(date -Iseconds) +BUILD_RELPATH=$(date "+%Y/%m")/$BUILD_GIT_COMMIT-$BUILD_ID/ + +GCS_ROOT=gs://packages.grpc.io/ +GCS_ARCHIVE_PREFIX=archive/ +GCS_ARCHIVE_ROOT=$GCS_ROOT$GCS_ARCHIVE_PREFIX +GCS_INDEX=$GCS_ROOT$INDEX_FILENAME + +LOCAL_STAGING_TEMPDIR=$(mktemp -d) +LOCAL_BUILD_ROOT=$LOCAL_STAGING_TEMPDIR/$BUILD_RELPATH +LOCAL_BUILD_INDEX=$LOCAL_BUILD_ROOT$INDEX_FILENAME + +mkdir -p "$LOCAL_BUILD_ROOT" + +find "$INPUT_ARTIFACTS" -type f + +# protoc Plugins +PROTOC_PLUGINS_ZIPPED_PACKAGES=$(mktemp -d) +for zip_dir in protoc_windows_{x86,x64} +do + zip -jr "$PROTOC_PLUGINS_ZIPPED_PACKAGES/grpc-$zip_dir-$GRPC_VERSION.zip" "$INPUT_ARTIFACTS/$zip_dir/"* +done +for tar_dir in protoc_{linux,macos}_{x86,x64} +do + chmod +x "$INPUT_ARTIFACTS/$tar_dir"/* + tar -cvzf "$PROTOC_PLUGINS_ZIPPED_PACKAGES/grpc-$tar_dir-$GRPC_VERSION.tar.gz" -C "$INPUT_ARTIFACTS/$tar_dir" . +done + +PROTOC_PACKAGES=( + "$PROTOC_PLUGINS_ZIPPED_PACKAGES"/grpc-protoc_windows_{x86,x64}-"$GRPC_VERSION.zip" + "$PROTOC_PLUGINS_ZIPPED_PACKAGES"/grpc-protoc_{linux,macos}_{x86,x64}-"$GRPC_VERSION.tar.gz" +) + +# C# +UNZIPPED_CSHARP_PACKAGES=$(mktemp -d) +unzip "$INPUT_ARTIFACTS/csharp_nugets_windows_dotnetcli.zip" -d "$UNZIPPED_CSHARP_PACKAGES" +CSHARP_PACKAGES=( + "$UNZIPPED_CSHARP_PACKAGES"/* + "$INPUT_ARTIFACTS"/grpc_unity_package.[0-9]*.zip +) + +# Python +PYTHON_GRPCIO_PACKAGES=( + "$INPUT_ARTIFACTS"/grpcio-[0-9]*.tar.gz + "$INPUT_ARTIFACTS"/grpcio-[0-9]*.whl + "$INPUT_ARTIFACTS"/python_linux_extra_arm*/grpcio-[0-9]*.whl +) +PYTHON_GRPCIO_TOOLS_PACKAGES=( + "$INPUT_ARTIFACTS"/grpcio-tools-[0-9]*.tar.gz + "$INPUT_ARTIFACTS"/grpcio_tools-[0-9]*.whl + "$INPUT_ARTIFACTS"/python_linux_extra_arm*/grpcio_tools-[0-9]*.whl +) +PYTHON_GRPCIO_HEALTH_CHECKING_PACKAGES=( + "$INPUT_ARTIFACTS"/grpcio-health-checking-[0-9]*.tar.gz +) +PYTHON_GRPCIO_REFLECTION_PACKAGES=( + "$INPUT_ARTIFACTS"/grpcio-reflection-[0-9]*.tar.gz +) +PYTHON_GRPCIO_TESTING_PACKAGES=( + "$INPUT_ARTIFACTS"/grpcio-testing-[0-9]*.tar.gz +) + +# PHP +PHP_PACKAGES=( + "$INPUT_ARTIFACTS"/grpc-[0-9]*.tgz +) + +# Ruby +RUBY_PACKAGES=( + "$INPUT_ARTIFACTS"/grpc-[0-9]*.gem + "$INPUT_ARTIFACTS"/grpc-tools-[0-9]*.gem +) + +function add_to_manifest() { + local artifact_type=$1 + local artifact_file=$2 + local artifact_prefix=$3 + local artifact_name + artifact_name=$(basename "$artifact_file") + local artifact_size + artifact_size=$(stat -c%s "$artifact_file") + local artifact_sha256 + artifact_sha256=$(openssl sha256 -r "$artifact_file" | cut -d " " -f 1) + local artifact_target=$LOCAL_BUILD_ROOT/$artifact_type/$artifact_prefix + mkdir -p "$artifact_target" + cp "$artifact_file" "$artifact_target" + cat <<EOF + <artifact name='$artifact_name' + type='$artifact_type' + path='$artifact_type/$artifact_prefix$artifact_name' + size='$artifact_size' + sha256='$artifact_sha256' /> +EOF +} + +{ + cat <<EOF +<?xml version="1.0"?> +<?xml-stylesheet href="/web-assets/build-201807.xsl" type="text/xsl"?> +<build id='$BUILD_ID' timestamp='$BUILD_TIMESTAMP' version="201807"> + <metadata> + <project>gRPC</project> + <repository>https://github.com/grpc/grpc</repository> + <branch>$BUILD_BRANCH_NAME</branch> + <commit>$BUILD_GIT_COMMIT</commit> + </metadata> + <artifacts> +EOF + + for pkg in "${PROTOC_PACKAGES[@]}"; do add_to_manifest protoc "$pkg"; done + for pkg in "${CSHARP_PACKAGES[@]}"; do add_to_manifest csharp "$pkg"; done + for pkg in "${PHP_PACKAGES[@]}"; do add_to_manifest php "$pkg"; done + for pkg in "${PYTHON_GRPCIO_PACKAGES[@]}"; do add_to_manifest python "$pkg" grpcio/; done + for pkg in "${PYTHON_GRPCIO_TOOLS_PACKAGES[@]}"; do add_to_manifest python "$pkg" grpcio-tools/; done + for pkg in "${PYTHON_GRPCIO_HEALTH_CHECKING_PACKAGES[@]}"; do add_to_manifest python "$pkg" grpcio-health-checking/; done + for pkg in "${PYTHON_GRPCIO_REFLECTION_PACKAGES[@]}"; do add_to_manifest python "$pkg" grpcio-reflection/; done + for pkg in "${PYTHON_GRPCIO_TESTING_PACKAGES[@]}"; do add_to_manifest python "$pkg" grpcio-testing/; done + for pkg in "${RUBY_PACKAGES[@]}"; do add_to_manifest ruby "$pkg"; done + + cat <<EOF + </artifacts> +</build> +EOF +}> "$LOCAL_BUILD_INDEX" + +LOCAL_BUILD_INDEX_SIZE=$(stat -c%s "$LOCAL_BUILD_INDEX") +LOCAL_BUILD_INDEX_SHA256=$(openssl sha256 -r "$LOCAL_BUILD_INDEX" | cut -d " " -f 1) + +OLD_INDEX=$(mktemp) +NEW_INDEX=$(mktemp) + +# Download the current /index.xml into $OLD_INDEX +gsutil cp "$GCS_INDEX" "$OLD_INDEX" + +{ + # we want to add an entry as the first child under <builds> tag + # we can get by without a real XML parser by rewriting the header, + # injecting our new tag, and then dumping the rest of the file as is. + cat <<EOF +<?xml version="1.0"?> +<?xml-stylesheet href="/web-assets/home.xsl" type="text/xsl"?> +<packages> + <builds> + <build id='$BUILD_ID' + timestamp='$BUILD_TIMESTAMP' + branch='$BUILD_BRANCH_NAME' + commit='$BUILD_GIT_COMMIT' + path='$GCS_ARCHIVE_PREFIX$BUILD_RELPATH$INDEX_FILENAME' + size='$LOCAL_BUILD_INDEX_SIZE' + sha256='$LOCAL_BUILD_INDEX_SHA256' /> +EOF + tail --lines=+5 "$OLD_INDEX" +}> "$NEW_INDEX" + + +function generate_directory_index() +{ + local target_dir=$1 + local current_directory_name + current_directory_name=$(basename "$target_dir") + cat <<EOF +<!DOCTYPE html> +<html xmlns="http://www.w3.org/1999/xhtml" lang="" xml:lang=""> + <head> + <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /> + <title>Index of $current_directory_name - packages.grpc.io</title> + <link rel="stylesheet" type="text/css" href="/web-assets/dirindex.css" /> + </head> + <body> + <h1>Index of <a href="#"><code>$current_directory_name</code></a></h1> + <ul> + <li><a href="#">.</a></li> + <li><a href="..">..</a></li> +EOF + +( + cd "$target_dir" + find * -maxdepth 0 -type d -print | sort | while read -r line + do + echo " <li><a href='$line/'>$line/</a></li>" + done + find * -maxdepth 0 -type f -print | sort | while read -r line + do + echo " <li><a href='$line'>$line</a></li>" + done +) + +cat <<EOF + </ul> + </body> +</html> +EOF +} + +# Upload the current build artifacts +gsutil -m cp -r "$LOCAL_STAGING_TEMPDIR/${BUILD_RELPATH%%/*}" "$GCS_ARCHIVE_ROOT" +# Upload directory indicies for subdirectories +( + cd "$LOCAL_BUILD_ROOT" + find * -type d | while read -r directory + do + generate_directory_index "$directory" | gsutil -h 'Content-Type:text/html' cp - "$GCS_ARCHIVE_ROOT$BUILD_RELPATH$directory/$INDEX_FILENAME" + done +) +# Upload the new /index.xml +gsutil -h "Content-Type:application/xml" cp "$NEW_INDEX" "$GCS_INDEX" diff --git a/tools/internal_ci/linux/grpc_pull_request_sanity.cfg b/tools/internal_ci/linux/grpc_pull_request_sanity.cfg index b20d2ffba8..704d5c6cbc 100644 --- a/tools/internal_ci/linux/grpc_pull_request_sanity.cfg +++ b/tools/internal_ci/linux/grpc_pull_request_sanity.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 30 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_python_bazel_test.cfg b/tools/internal_ci/linux/grpc_python_bazel_test.cfg new file mode 100644 index 0000000000..feae924330 --- /dev/null +++ b/tools/internal_ci/linux/grpc_python_bazel_test.cfg @@ -0,0 +1,23 @@ +# Copyright 2018 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_bazel.sh" +timeout_mins: 240 +env_vars { + key: "BAZEL_SCRIPT" + value: "tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh" +} diff --git a/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh b/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh new file mode 100755 index 0000000000..4f98d0a93a --- /dev/null +++ b/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Copyright 2018 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Test full Bazel +# +# NOTE: No empty lines should appear in this file before igncr is set! +set -ex -o igncr || set -ex + +mkdir -p /var/local/git +git clone /var/local/jenkins/grpc /var/local/git/grpc +(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \ +&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \ +${name}') +cd /var/local/git/grpc/test +bazel test --spawn_strategy=standalone --genrule_strategy=standalone //src/python/... diff --git a/tools/internal_ci/linux/grpc_run_tests_matrix.sh b/tools/internal_ci/linux/grpc_run_tests_matrix.sh index 1018708f96..4e7515227b 100755 --- a/tools/internal_ci/linux/grpc_run_tests_matrix.sh +++ b/tools/internal_ci/linux/grpc_run_tests_matrix.sh @@ -29,9 +29,6 @@ fi tools/run_tests/run_tests_matrix.py $RUN_TESTS_FLAGS || FAILED="true" -# Reveal leftover processes that might be left behind by the build -ps aux | grep -i kbuilder - echo 'Exiting gRPC main test script.' if [ "$FAILED" != "" ] diff --git a/tools/internal_ci/linux/grpc_sanity.cfg b/tools/internal_ci/linux/grpc_sanity.cfg index 9f65918e23..341471bbb5 100644 --- a/tools/internal_ci/linux/grpc_sanity.cfg +++ b/tools/internal_ci/linux/grpc_sanity.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 40 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_trickle_diff.sh b/tools/internal_ci/linux/grpc_trickle_diff.sh index 624031afa7..4ed1b73f9e 100755 --- a/tools/internal_ci/linux/grpc_trickle_diff.sh +++ b/tools/internal_ci/linux/grpc_trickle_diff.sh @@ -25,7 +25,7 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc tools/run_tests/start_port_server.py -tools/jenkins/run_c_cpp_test.sh tools/profiling/microbenchmarks/bm_diff/bm_main.py \ +tools/internal_ci/linux/run_if_c_cpp_modified.sh tools/profiling/microbenchmarks/bm_diff/bm_main.py \ -d origin/$ghprbTargetBranch \ -b bm_fullstack_trickle \ -l 4 \ diff --git a/tools/internal_ci/linux/grpc_tsan_on_foundry.sh b/tools/internal_ci/linux/grpc_tsan_on_foundry.sh index fafa1ceecb..366b5cbe34 100644 --- a/tools/internal_ci/linux/grpc_tsan_on_foundry.sh +++ b/tools/internal_ci/linux/grpc_tsan_on_foundry.sh @@ -14,5 +14,6 @@ # limitations under the License. export UPLOAD_TEST_RESULTS=true -EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=thread --linkopt=-fsanitize=thread --test_timeout=3600" -github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" +EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=thread --linkopt=-fsanitize=thread --test_timeout=3600 --action_env=TSAN_OPTIONS=suppressions=test/core/util/tsan_suppressions.txt:halt_on_error=1:second_deadlock_stack=1 --cache_test_results=no" +EXCLUDE_TESTS="--test_tag_filters=-qps_json_driver,-json_run_localhost" +github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" "${EXCLUDE_TESTS}" diff --git a/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh b/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh index 0f71ff27cf..e0ae9103c4 100644 --- a/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh +++ b/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh @@ -23,8 +23,8 @@ mkdir -p ${KOKORO_KEYSTORE_DIR} cp ${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json ${KOKORO_KEYSTORE_DIR}/4321_grpc-testing-service temp_dir=$(mktemp -d) -ln -f "${KOKORO_GFILE_DIR}/bazel-release-0.12.0" ${temp_dir}/bazel -chmod 755 "${KOKORO_GFILE_DIR}/bazel-release-0.12.0" +ln -f "${KOKORO_GFILE_DIR}/bazel-latest-release" ${temp_dir}/bazel +chmod 755 "${KOKORO_GFILE_DIR}/bazel-latest-release" export PATH="${temp_dir}:${PATH}" # This should show ${temp_dir}/bazel which bazel @@ -35,6 +35,8 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_rc +export KOKORO_FOUNDRY_PROJECT_ID="projects/grpc-testing/instances/default_instance" + "${KOKORO_GFILE_DIR}/bazel_wrapper.py" \ --host_jvm_args=-Dbazel.DigestFunction=SHA256 \ test --jobs="200" \ @@ -50,13 +52,20 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_rc --strategy=Closure=remote \ --genrule_strategy=remote \ --experimental_strict_action_env=true \ - --experimental_remote_platform_override='properties:{name:"container-image" value:"docker://gcr.io/cloud-marketplace/google/rbe-debian8@sha256:b2d946c1ddc20af250fe85cf98bd648ac5519131659f7c36e64184b433175a33" }' \ --define GRPC_PORT_ISOLATED_RUNTIME=1 \ --copt=-gmlt \ --strip=never \ --copt=-fsanitize=undefined \ --linkopt=-fsanitize=undefined \ - --crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/experimental/debian8_clang/0.3.0/bazel_0.10.0/ubsan:ubsan_experimental_toolchain \ + --crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/experimental/ubuntu16_04_clang/1.0/bazel_0.15.0/ubsan:toolchain \ + --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 \ + --extra_toolchains=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.0/bazel_0.16.1/cpp:cc-toolchain-clang-x86_64-default \ + --extra_execution_platforms=//third_party/toolchains:rbe_ubuntu1604 \ + --host_platform=//third_party/toolchains:rbe_ubuntu1604 \ + --platforms=//third_party/toolchains:rbe_ubuntu1604 \ + --cache_test_results=no \ + --test_env=GRPC_VERBOSITY=debug \ + --remote_instance_name=projects/grpc-testing/instances/default_instance \ -- //test/... || FAILED="true" # Sleep to let ResultStore finish writing results before querying diff --git a/tools/internal_ci/linux/pull_request/grpc_asan_on_foundry.sh b/tools/internal_ci/linux/pull_request/grpc_asan_on_foundry.sh new file mode 100644 index 0000000000..39c991f291 --- /dev/null +++ b/tools/internal_ci/linux/pull_request/grpc_asan_on_foundry.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=address --linkopt=-fsanitize=address --test_timeout=3600" +EXCLUDE_TESTS="--test_tag_filters=-qps_json_driver,-json_run_localhost" +github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" "${EXCLUDE_TESTS}" + diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_c_cpp_dbg.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_c_cpp_dbg.cfg index 8124f5c1b3..8a67d28ce4 100644 --- a/tools/internal_ci/linux/pull_request/grpc_basictests_c_cpp_dbg.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_c_cpp_dbg.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_c_cpp_opt.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_c_cpp_opt.cfg index ecedc73e44..a681978b6e 100644 --- a/tools/internal_ci/linux/pull_request/grpc_basictests_c_cpp_opt.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_c_cpp_opt.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_c_dbg.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_c_dbg.cfg index 577cb28ae5..249ecc99ce 100644 --- a/tools/internal_ci/linux/pull_request/grpc_basictests_c_dbg.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_c_dbg.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_c_opt.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_c_opt.cfg index 9e0b724b2e..665d7f3d2c 100644 --- a/tools/internal_ci/linux/pull_request/grpc_basictests_c_opt.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_c_opt.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_dbg.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_dbg.cfg index 0fda74cf44..163274d4dc 100644 --- a/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_dbg.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_dbg.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_opt.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_opt.cfg index 199a8905d9..b65cd3e05c 100644 --- a/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_opt.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_opt.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_multilang.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_multilang.cfg index f7e8d8ad33..59f38f0d1b 100644 --- a/tools/internal_ci/linux/pull_request/grpc_basictests_multilang.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_multilang.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_interop_alts.cfg b/tools/internal_ci/linux/pull_request/grpc_interop_alts.cfg index c1253b30f7..e91a612878 100644 --- a/tools/internal_ci/linux/pull_request/grpc_interop_alts.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_interop_alts.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_interop_tests.sh" timeout_mins: 60 action { define_artifacts { - regex: "**/sponge_log.xml" + regex: "**/sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_interop_tocloud.cfg b/tools/internal_ci/linux/pull_request/grpc_interop_tocloud.cfg index cb18e8e868..b1eb575605 100644 --- a/tools/internal_ci/linux/pull_request/grpc_interop_tocloud.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_interop_tocloud.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_interop_tests.sh" timeout_mins: 60 action { define_artifacts { - regex: "**/sponge_log.xml" + regex: "**/sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_interop_toprod.cfg b/tools/internal_ci/linux/pull_request/grpc_interop_toprod.cfg index d14c79a1f6..7321effc12 100644 --- a/tools/internal_ci/linux/pull_request/grpc_interop_toprod.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_interop_toprod.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_interop_tests.sh" timeout_mins: 60 action { define_artifacts { - regex: "**/sponge_log.xml" + regex: "**/sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_microbenchmark_diff.cfg b/tools/internal_ci/linux/pull_request/grpc_microbenchmark_diff.cfg index 9269c345f0..47301d6141 100644 --- a/tools/internal_ci/linux/pull_request/grpc_microbenchmark_diff.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_microbenchmark_diff.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_microbenchmark_diff.sh" timeout_mins: 120 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_sanity.cfg b/tools/internal_ci/linux/pull_request/grpc_sanity.cfg index 0f83299aab..276c34f0cf 100644 --- a/tools/internal_ci/linux/pull_request/grpc_sanity.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_sanity.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 40 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_trickle_diff.cfg b/tools/internal_ci/linux/pull_request/grpc_trickle_diff.cfg index e86b3ab475..78358eac28 100644 --- a/tools/internal_ci/linux/pull_request/grpc_trickle_diff.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_trickle_diff.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_trickle_diff.sh" timeout_mins: 120 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_tsan_on_foundry.sh b/tools/internal_ci/linux/pull_request/grpc_tsan_on_foundry.sh new file mode 100644 index 0000000000..3dee115300 --- /dev/null +++ b/tools/internal_ci/linux/pull_request/grpc_tsan_on_foundry.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=thread --linkopt=-fsanitize=thread --test_timeout=3600 --action_env=TSAN_OPTIONS=suppressions=test/core/util/tsan_suppressions.txt:halt_on_error=1:second_deadlock_stack=1" +EXCLUDE_TESTS="--test_tag_filters=-qps_json_driver,-json_run_localhost" +github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" "${EXCLUDE_TESTS}" diff --git a/tools/internal_ci/linux/pull_request/grpc_ubsan_on_foundry.sh b/tools/internal_ci/linux/pull_request/grpc_ubsan_on_foundry.sh new file mode 100644 index 0000000000..8547fa4d93 --- /dev/null +++ b/tools/internal_ci/linux/pull_request/grpc_ubsan_on_foundry.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# A temporary solution to give Kokoro credentials. +# The file name 4321_grpc-testing-service needs to match auth_credential in +# the build config. +# TODO: Use keystore. +mkdir -p ${KOKORO_KEYSTORE_DIR} +cp ${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json ${KOKORO_KEYSTORE_DIR}/4321_grpc-testing-service + +temp_dir=$(mktemp -d) +ln -f "${KOKORO_GFILE_DIR}/bazel-latest-release" ${temp_dir}/bazel +chmod 755 "${KOKORO_GFILE_DIR}/bazel-latest-release" +export PATH="${temp_dir}:${PATH}" +# This should show ${temp_dir}/bazel +which bazel +chmod +x "${KOKORO_GFILE_DIR}/bazel_wrapper.py" + +# change to grpc repo root +cd $(dirname $0)/../../../.. + +source tools/internal_ci/helper_scripts/prepare_build_linux_rc + +export KOKORO_FOUNDRY_PROJECT_ID="projects/grpc-testing/instances/default_instance" + +"${KOKORO_GFILE_DIR}/bazel_wrapper.py" \ + --host_jvm_args=-Dbazel.DigestFunction=SHA256 \ + test --jobs="200" \ + --test_timeout="3600,3600,3600,3600" \ + --test_output=errors \ + --verbose_failures=true \ + --keep_going \ + --remote_accept_cached=true \ + --spawn_strategy=remote \ + --remote_local_fallback=false \ + --remote_timeout=3600 \ + --strategy=Javac=remote \ + --strategy=Closure=remote \ + --genrule_strategy=remote \ + --experimental_strict_action_env=true \ + --define GRPC_PORT_ISOLATED_RUNTIME=1 \ + --copt=-gmlt \ + --strip=never \ + --copt=-fsanitize=undefined \ + --linkopt=-fsanitize=undefined \ + --crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/experimental/ubuntu16_04_clang/1.0/bazel_0.15.0/ubsan:toolchain \ + --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 \ + --extra_toolchains=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.0/bazel_0.16.1/cpp:cc-toolchain-clang-x86_64-default \ + --extra_execution_platforms=//third_party/toolchains:rbe_ubuntu1604 \ + --host_platform=//third_party/toolchains:rbe_ubuntu1604 \ + --platforms=//third_party/toolchains:rbe_ubuntu1604 \ + --test_env=GRPC_VERBOSITY=debug \ + --remote_instance_name=projects/grpc-testing/instances/default_instance \ + -- //test/... || FAILED="true" + +if [ "$FAILED" != "" ] +then + exit 1 +fi diff --git a/tools/internal_ci/linux/run_if_c_cpp_modified.sh b/tools/internal_ci/linux/run_if_c_cpp_modified.sh new file mode 100755 index 0000000000..736d759423 --- /dev/null +++ b/tools/internal_ci/linux/run_if_c_cpp_modified.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This script is invoked by a pull request job and executes all +# args passed to this script if the pull request affect C/C++ code +set -ex + +# Enter the gRPC repo root +cd $(dirname $0)/../../.. + +# TODO(jtattermusch): the "ghprbTargetBranch" is Jenkins specific and probably +# does not work on kokoro? +AFFECTS_C_CPP=`python -c 'import os; \ + import sys; \ + sys.path.insert(0, "tools/run_tests/python_utils"); \ + import filter_pull_request_tests as filter; \ + github_target_branch = os.environ.get("ghprbTargetBranch"); \ + print(filter.affects_c_cpp("origin/%s" % github_target_branch))'` + +if [ $AFFECTS_C_CPP == "False" ] ; then + echo "This pull request does not affect C/C++. Tests do not need to be run." +else + $@ +fi diff --git a/tools/internal_ci/linux/run_performance_profile_daily.sh b/tools/internal_ci/linux/run_performance_profile_daily.sh new file mode 100755 index 0000000000..45c7a99805 --- /dev/null +++ b/tools/internal_ci/linux/run_performance_profile_daily.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Copyright 2016 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +cd $(dirname $0)/../../.. + +# try to use pypy for generating reports +# each trace dumps 7-8gig of text to disk, and processing this into a report is +# heavyweight - so any speed boost is worthwhile +# TODO(ctiller): consider rewriting report generation in C++ for performance +if which pypy >/dev/null; then + PYTHON=pypy +else + PYTHON=python2.7 +fi + +BENCHMARKS_TO_RUN="bm_fullstack_unary_ping_pong bm_fullstack_streaming_ping_pong bm_fullstack_streaming_pump bm_closure bm_cq bm_call_create bm_error bm_chttp2_hpack bm_chttp2_transport bm_pollset bm_metadata" + +./tools/run_tests/start_port_server.py || true + +$PYTHON tools/run_tests/run_microbenchmark.py --collect summary perf latency -b $BENCHMARKS_TO_RUN diff --git a/tools/internal_ci/linux/run_performance_profile_hourly.sh b/tools/internal_ci/linux/run_performance_profile_hourly.sh new file mode 100755 index 0000000000..edf85c2e2c --- /dev/null +++ b/tools/internal_ci/linux/run_performance_profile_hourly.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2016 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +cd $(dirname $0)/../../.. + +./tools/run_tests/start_port_server.py || true + +CPUS=`python -c 'import multiprocessing; print multiprocessing.cpu_count()'` + +make CONFIG=opt memory_profile_test memory_profile_client memory_profile_server -j $CPUS +bins/opt/memory_profile_test +bq load microbenchmarks.memory memory_usage.csv + +tools/run_tests/run_microbenchmark.py --collect summary --bigquery_upload diff --git a/tools/internal_ci/linux/sanitizer/grpc_c_asan.cfg b/tools/internal_ci/linux/sanitizer/grpc_c_asan.cfg index 06a4372ce2..d6d70677dd 100644 --- a/tools/internal_ci/linux/sanitizer/grpc_c_asan.cfg +++ b/tools/internal_ci/linux/sanitizer/grpc_c_asan.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 1440 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/sanitizer/grpc_c_msan.cfg b/tools/internal_ci/linux/sanitizer/grpc_c_msan.cfg index f875327c1b..7b22c6afef 100644 --- a/tools/internal_ci/linux/sanitizer/grpc_c_msan.cfg +++ b/tools/internal_ci/linux/sanitizer/grpc_c_msan.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 1440 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/sanitizer/grpc_c_tsan.cfg b/tools/internal_ci/linux/sanitizer/grpc_c_tsan.cfg index 6658a804d8..6c9dd6ef8d 100644 --- a/tools/internal_ci/linux/sanitizer/grpc_c_tsan.cfg +++ b/tools/internal_ci/linux/sanitizer/grpc_c_tsan.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 1440 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/sanitizer/grpc_c_ubsan.cfg b/tools/internal_ci/linux/sanitizer/grpc_c_ubsan.cfg index 957a91ef2b..8700c74c8d 100644 --- a/tools/internal_ci/linux/sanitizer/grpc_c_ubsan.cfg +++ b/tools/internal_ci/linux/sanitizer/grpc_c_ubsan.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 1440 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/sanitizer/grpc_cpp_asan.cfg b/tools/internal_ci/linux/sanitizer/grpc_cpp_asan.cfg index dbbfce90cb..02162d860b 100644 --- a/tools/internal_ci/linux/sanitizer/grpc_cpp_asan.cfg +++ b/tools/internal_ci/linux/sanitizer/grpc_cpp_asan.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 1440 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/sanitizer/grpc_cpp_tsan.cfg b/tools/internal_ci/linux/sanitizer/grpc_cpp_tsan.cfg index fb0cefa160..95582184ed 100644 --- a/tools/internal_ci/linux/sanitizer/grpc_cpp_tsan.cfg +++ b/tools/internal_ci/linux/sanitizer/grpc_cpp_tsan.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 1440 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_asan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_asan.cfg index 1daf7a514e..d444596a81 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_asan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_asan.cfg @@ -20,7 +20,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_msan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_msan.cfg index a8503b7bcb..3891cc37e8 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_msan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_msan.cfg @@ -20,7 +20,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_tsan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_tsan.cfg index 12af4581eb..91ce1627af 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_tsan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_tsan.cfg @@ -20,7 +20,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_ubsan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_ubsan.cfg index 0d3803bf23..fcdc2de37b 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_ubsan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_ubsan.cfg @@ -20,7 +20,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_asan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_asan.cfg index 557561810b..de2a74051b 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_asan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_asan.cfg @@ -20,7 +20,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_tsan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_tsan.cfg index cb15ca34fd..51c291f7df 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_tsan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_tsan.cfg @@ -20,7 +20,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 1440 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/macos/grpc_basictests_dbg.cfg b/tools/internal_ci/macos/grpc_basictests_dbg.cfg index 53bda1ff0a..e6f9c7ec87 100644 --- a/tools/internal_ci/macos/grpc_basictests_dbg.cfg +++ b/tools/internal_ci/macos/grpc_basictests_dbg.cfg @@ -20,7 +20,7 @@ gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0e timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/macos/grpc_basictests_opt.cfg b/tools/internal_ci/macos/grpc_basictests_opt.cfg index d359eb601a..f2a83fe95a 100644 --- a/tools/internal_ci/macos/grpc_basictests_opt.cfg +++ b/tools/internal_ci/macos/grpc_basictests_opt.cfg @@ -20,7 +20,7 @@ gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0e timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/macos/grpc_build_artifacts.cfg b/tools/internal_ci/macos/grpc_build_artifacts.cfg index 4da61faed3..c73cf4359f 100644 --- a/tools/internal_ci/macos/grpc_build_artifacts.cfg +++ b/tools/internal_ci/macos/grpc_build_artifacts.cfg @@ -20,7 +20,7 @@ gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0e timeout_mins: 120 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" regex: "github/grpc/artifacts/**" } diff --git a/tools/internal_ci/macos/grpc_build_artifacts.sh b/tools/internal_ci/macos/grpc_build_artifacts.sh index eb4568c32b..54b171d209 100755 --- a/tools/internal_ci/macos/grpc_build_artifacts.sh +++ b/tools/internal_ci/macos/grpc_build_artifacts.sh @@ -25,6 +25,7 @@ python2.7 -m pip install cython setuptools wheel python3.4 -m pip install cython setuptools wheel python3.5 -m pip install cython setuptools wheel python3.6 -m pip install cython setuptools wheel +python3.7 -m pip install cython setuptools wheel # needed to build ruby artifacts time bash tools/distrib/build_ruby_environment_macos.sh @@ -32,4 +33,11 @@ time bash tools/distrib/build_ruby_environment_macos.sh gem install rubygems-update update_rubygems -tools/run_tests/task_runner.py -f artifact macos +tools/run_tests/task_runner.py -f artifact macos || FAILED="true" + +tools/internal_ci/helper_scripts/delete_nonartifacts.sh || true + +if [ "$FAILED" != "" ] +then + exit 1 +fi diff --git a/tools/internal_ci/macos/grpc_distribtests.cfg b/tools/internal_ci/macos/grpc_distribtests.cfg index ae88f39b90..156ec6fe33 100644 --- a/tools/internal_ci/macos/grpc_distribtests.cfg +++ b/tools/internal_ci/macos/grpc_distribtests.cfg @@ -20,7 +20,7 @@ gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0e timeout_mins: 120 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" regex: "github/grpc/artifacts/**" } diff --git a/tools/internal_ci/macos/grpc_distribtests.sh b/tools/internal_ci/macos/grpc_distribtests.sh index 59ea833eec..eb256d92ee 100644 --- a/tools/internal_ci/macos/grpc_distribtests.sh +++ b/tools/internal_ci/macos/grpc_distribtests.sh @@ -24,4 +24,11 @@ source tools/internal_ci/helper_scripts/prepare_build_macos_rc mv ${KOKORO_GFILE_DIR}/github/grpc/artifacts input_artifacts || true ls -R input_artifacts || true -tools/run_tests/task_runner.py -f distribtest macos +tools/run_tests/task_runner.py -f distribtest macos || FAILED="true" + +tools/internal_ci/helper_scripts/delete_nonartifacts.sh || true + +if [ "$FAILED" != "" ] +then + exit 1 +fi diff --git a/tools/internal_ci/macos/grpc_interop.cfg b/tools/internal_ci/macos/grpc_interop.cfg index b4b1b15cb4..434ecd19c4 100644 --- a/tools/internal_ci/macos/grpc_interop.cfg +++ b/tools/internal_ci/macos/grpc_interop.cfg @@ -20,7 +20,7 @@ gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0e timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/macos/grpc_interop.sh b/tools/internal_ci/macos/grpc_interop.sh index b03401bd96..e290ed60c4 100755 --- a/tools/internal_ci/macos/grpc_interop.sh +++ b/tools/internal_ci/macos/grpc_interop.sh @@ -21,4 +21,11 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc source tools/internal_ci/helper_scripts/prepare_build_macos_rc -tools/run_tests/run_interop_tests.py -l objc -s all --use_docker -t -j 1 +tools/run_tests/run_interop_tests.py -l objc -s all --use_docker -t -j 1 || FAILED="true" + +tools/internal_ci/helper_scripts/delete_nonartifacts.sh || true + +if [ "$FAILED" != "" ] +then + exit 1 +fi diff --git a/tools/internal_ci/macos/grpc_interop_toprod.cfg b/tools/internal_ci/macos/grpc_interop_toprod.cfg index c92c397daa..2cfc8a2d6d 100644 --- a/tools/internal_ci/macos/grpc_interop_toprod.cfg +++ b/tools/internal_ci/macos/grpc_interop_toprod.cfg @@ -21,7 +21,7 @@ gfile_resources: "/bigstore/grpc-testing-secrets/interop/service_account/GrpcTes timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/macos/grpc_interop_toprod.sh b/tools/internal_ci/macos/grpc_interop_toprod.sh index 819a4721fd..e748a62e76 100755 --- a/tools/internal_ci/macos/grpc_interop_toprod.sh +++ b/tools/internal_ci/macos/grpc_interop_toprod.sh @@ -18,8 +18,8 @@ set -ex # change to grpc repo root cd $(dirname $0)/../../.. -source tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc source tools/internal_ci/helper_scripts/prepare_build_macos_rc +source tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc # using run_interop_tests.py without --use_docker, so we need to build first tools/run_tests/run_tests.py -l c++ -c opt --build_only @@ -32,4 +32,11 @@ export GRPC_DEFAULT_SSL_ROOTS_FILE_PATH="$(pwd)/etc/roots.pem" tools/run_tests/run_interop_tests.py -l c++ \ --cloud_to_prod --cloud_to_prod_auth --prod_servers default gateway_v4 \ --service_account_key_file="${KOKORO_GFILE_DIR}/GrpcTesting-726eb1347f15.json" \ - --skip_compute_engine_creds --internal_ci -t -j 4 + --skip_compute_engine_creds --internal_ci -t -j 4 || FAILED="true" + +tools/internal_ci/helper_scripts/delete_nonartifacts.sh || true + +if [ "$FAILED" != "" ] +then + exit 1 +fi diff --git a/tools/internal_ci/macos/grpc_ios_binary_size.sh b/tools/internal_ci/macos/grpc_ios_binary_size.sh new file mode 100755 index 0000000000..ea39b0d6e9 --- /dev/null +++ b/tools/internal_ci/macos/grpc_ios_binary_size.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Copyright 2018 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This script is invoked by Jenkins and runs a diff on the microbenchmarks +set -ex + +# List of benchmarks that provide good signal for analyzing performance changes in pull requests + +# Enter the gRPC repo root +cd $(dirname $0)/../../.. + +source tools/internal_ci/helper_scripts/prepare_build_macos_rc + +tools/profiling/ios_bin/binary_size.py \ + -d origin/$ghprbTargetBranch diff --git a/tools/internal_ci/macos/grpc_run_tests_matrix.sh b/tools/internal_ci/macos/grpc_run_tests_matrix.sh index 6e0c2bb487..7636a31f41 100755 --- a/tools/internal_ci/macos/grpc_run_tests_matrix.sh +++ b/tools/internal_ci/macos/grpc_run_tests_matrix.sh @@ -25,13 +25,7 @@ tools/run_tests/run_tests_matrix.py $RUN_TESTS_FLAGS || FAILED="true" # kill port_server.py to prevent the build from hanging ps aux | grep port_server\\.py | awk '{print $2}' | xargs kill -9 -# Reveal leftover processes that might be left behind by the build -ps aux | grep -i kbuilder - -# TODO(jtattermusch): better debugging of clock skew, remove once not needed -date - -echo 'Exiting gRPC main test script.' +tools/internal_ci/helper_scripts/delete_nonartifacts.sh || true if [ "$FAILED" != "" ] then diff --git a/tools/internal_ci/macos/pull_request/grpc_basictests_dbg.cfg b/tools/internal_ci/macos/pull_request/grpc_basictests_dbg.cfg index 30c01d3e2f..c759397b78 100644 --- a/tools/internal_ci/macos/pull_request/grpc_basictests_dbg.cfg +++ b/tools/internal_ci/macos/pull_request/grpc_basictests_dbg.cfg @@ -20,7 +20,7 @@ gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0e timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/macos/pull_request/grpc_basictests_opt.cfg b/tools/internal_ci/macos/pull_request/grpc_basictests_opt.cfg index b63ee713bc..4d68341405 100644 --- a/tools/internal_ci/macos/pull_request/grpc_basictests_opt.cfg +++ b/tools/internal_ci/macos/pull_request/grpc_basictests_opt.cfg @@ -20,7 +20,7 @@ gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0e timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/macos/pull_request/grpc_interop.cfg b/tools/internal_ci/macos/pull_request/grpc_interop.cfg index b4b1b15cb4..434ecd19c4 100644 --- a/tools/internal_ci/macos/pull_request/grpc_interop.cfg +++ b/tools/internal_ci/macos/pull_request/grpc_interop.cfg @@ -20,7 +20,7 @@ gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0e timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/macos/pull_request/grpc_ios_binary_size.cfg b/tools/internal_ci/macos/pull_request/grpc_ios_binary_size.cfg new file mode 100644 index 0000000000..fb215bdf99 --- /dev/null +++ b/tools/internal_ci/macos/pull_request/grpc_ios_binary_size.cfg @@ -0,0 +1,26 @@ +# Copyright 2018 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/macos/grpc_ios_binary_size.sh" +timeout_mins: 60 +gfile_resources: "/bigstore/grpc-testing-secrets/github_credentials/oauth_token.txt" +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + } +} diff --git a/tools/internal_ci/windows/grpc_basictests.cfg b/tools/internal_ci/windows/grpc_basictests.cfg index 8e644e4c5e..fcf5237bf3 100644 --- a/tools/internal_ci/windows/grpc_basictests.cfg +++ b/tools/internal_ci/windows/grpc_basictests.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/windows/grpc_basictests_dbg.cfg b/tools/internal_ci/windows/grpc_basictests_dbg.cfg index 28d53cdc7b..4e5e7b6545 100644 --- a/tools/internal_ci/windows/grpc_basictests_dbg.cfg +++ b/tools/internal_ci/windows/grpc_basictests_dbg.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/windows/grpc_basictests_opt.cfg b/tools/internal_ci/windows/grpc_basictests_opt.cfg index 4b7a965977..f5db6a9897 100644 --- a/tools/internal_ci/windows/grpc_basictests_opt.cfg +++ b/tools/internal_ci/windows/grpc_basictests_opt.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/windows/grpc_build_artifacts.bat b/tools/internal_ci/windows/grpc_build_artifacts.bat index c6e523697c..0e02bdb6ec 100644 --- a/tools/internal_ci/windows/grpc_build_artifacts.bat +++ b/tools/internal_ci/windows/grpc_build_artifacts.bat @@ -18,14 +18,16 @@ rename C:\Python27_32bit Python27_32bits rename C:\Python34_32bit Python34_32bits rename C:\Python35_32bit Python35_32bits rename C:\Python36_32bit Python36_32bits +rename C:\Python37_32bit Python37_32bits @rem enter repo root cd /d %~dp0\..\..\.. call tools/internal_ci/helper_scripts/prepare_build_windows.bat -python tools/run_tests/task_runner.py -f artifact windows -j 4 || goto :error -goto :EOF +python tools/run_tests/task_runner.py -f artifact windows -j 4 +set RUNTESTS_EXITCODE=%errorlevel% -:error -exit /b %errorlevel% +bash tools/internal_ci/helper_scripts/delete_nonartifacts.sh + +exit /b %RUNTESTS_EXITCODE% diff --git a/tools/internal_ci/windows/grpc_build_artifacts.cfg b/tools/internal_ci/windows/grpc_build_artifacts.cfg index 38b0abd519..f45cfda121 100644 --- a/tools/internal_ci/windows/grpc_build_artifacts.cfg +++ b/tools/internal_ci/windows/grpc_build_artifacts.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_build_artifacts.bat" timeout_mins: 120 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" regex: "github/grpc/artifacts/**" } diff --git a/tools/internal_ci/windows/grpc_build_packages.bat b/tools/internal_ci/windows/grpc_build_packages.bat index 23ba365e40..61a636f607 100644 --- a/tools/internal_ci/windows/grpc_build_packages.bat +++ b/tools/internal_ci/windows/grpc_build_packages.bat @@ -28,8 +28,9 @@ call tools/internal_ci/helper_scripts/prepare_build_windows.bat powershell -Command "mv %KOKORO_GFILE_DIR%\github\grpc\artifacts input_artifacts" dir input_artifacts -python tools/run_tests/task_runner.py -f package windows -j 4 || goto :error -goto :EOF +python tools/run_tests/task_runner.py -f package windows -j 4 +set RUNTESTS_EXITCODE=%errorlevel% -:error -exit /b %errorlevel% +bash tools/internal_ci/helper_scripts/delete_nonartifacts.sh + +exit /b %RUNTESTS_EXITCODE% diff --git a/tools/internal_ci/windows/grpc_build_packages.cfg b/tools/internal_ci/windows/grpc_build_packages.cfg index 65a8b1eef3..b351bbb11b 100644 --- a/tools/internal_ci/windows/grpc_build_packages.cfg +++ b/tools/internal_ci/windows/grpc_build_packages.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_build_packages.bat" timeout_mins: 120 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" regex: "github/grpc/artifacts/**" } diff --git a/tools/internal_ci/windows/grpc_distribtests.bat b/tools/internal_ci/windows/grpc_distribtests.bat index d6d50b684f..65fd01ef72 100644 --- a/tools/internal_ci/windows/grpc_distribtests.bat +++ b/tools/internal_ci/windows/grpc_distribtests.bat @@ -28,8 +28,9 @@ call tools/internal_ci/helper_scripts/prepare_build_windows.bat powershell -Command "mv %KOKORO_GFILE_DIR%\github\grpc\artifacts input_artifacts" dir input_artifacts -python tools/run_tests/task_runner.py -f distribtest windows -j 4 || goto :error -goto :EOF +python tools/run_tests/task_runner.py -f distribtest windows -j 4 +set RUNTESTS_EXITCODE=%errorlevel% -:error -exit /b %errorlevel% +bash tools/internal_ci/helper_scripts/delete_nonartifacts.sh + +exit /b %RUNTESTS_EXITCODE% diff --git a/tools/internal_ci/windows/grpc_distribtests.cfg b/tools/internal_ci/windows/grpc_distribtests.cfg index 1766e601e5..e12d4bf98d 100644 --- a/tools/internal_ci/windows/grpc_distribtests.cfg +++ b/tools/internal_ci/windows/grpc_distribtests.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_distribtests.bat" timeout_mins: 120 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" regex: "github/grpc/artifacts/**" } diff --git a/tools/internal_ci/windows/grpc_distribtests_standalone.bat b/tools/internal_ci/windows/grpc_distribtests_standalone.bat index 3eb33b1548..ea4c0d7a97 100644 --- a/tools/internal_ci/windows/grpc_distribtests_standalone.bat +++ b/tools/internal_ci/windows/grpc_distribtests_standalone.bat @@ -24,8 +24,9 @@ cd /d %~dp0\..\..\.. call tools/internal_ci/helper_scripts/prepare_build_windows.bat -python tools/run_tests/task_runner.py -f distribtest windows cpp -j 4 || goto :error -goto :EOF +python tools/run_tests/task_runner.py -f distribtest windows cpp -j 4 +set RUNTESTS_EXITCODE=%errorlevel% -:error -exit /b %errorlevel% +bash tools/internal_ci/helper_scripts/delete_nonartifacts.sh + +exit /b %RUNTESTS_EXITCODE% diff --git a/tools/internal_ci/windows/grpc_distribtests_standalone.cfg b/tools/internal_ci/windows/grpc_distribtests_standalone.cfg index 33a50fdc45..7f32ba4c70 100644 --- a/tools/internal_ci/windows/grpc_distribtests_standalone.cfg +++ b/tools/internal_ci/windows/grpc_distribtests_standalone.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_distribtests_standalone.bat" timeout_mins: 120 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" regex: "github/grpc/artifacts/**" } diff --git a/tools/internal_ci/windows/grpc_portability.cfg b/tools/internal_ci/windows/grpc_portability.cfg index 94e71753ef..aa86c06632 100644 --- a/tools/internal_ci/windows/grpc_portability.cfg +++ b/tools/internal_ci/windows/grpc_portability.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/windows/grpc_portability_build_only.cfg b/tools/internal_ci/windows/grpc_portability_build_only.cfg index 3bc27f1f24..3dace627eb 100644 --- a/tools/internal_ci/windows/grpc_portability_build_only.cfg +++ b/tools/internal_ci/windows/grpc_portability_build_only.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/windows/grpc_run_tests_matrix.bat b/tools/internal_ci/windows/grpc_run_tests_matrix.bat index 10627add25..c6277d0929 100644 --- a/tools/internal_ci/windows/grpc_run_tests_matrix.bat +++ b/tools/internal_ci/windows/grpc_run_tests_matrix.bat @@ -20,7 +20,6 @@ call tools/internal_ci/helper_scripts/prepare_build_windows.bat python tools/run_tests/run_tests_matrix.py %RUN_TESTS_FLAGS% set RUNTESTS_EXITCODE=%errorlevel% -@rem Reveal leftover processes that might be left behind by the build -tasklist /V +bash tools/internal_ci/helper_scripts/delete_nonartifacts.sh exit /b %RUNTESTS_EXITCODE% diff --git a/tools/internal_ci/windows/pull_request/grpc_basictests.cfg b/tools/internal_ci/windows/pull_request/grpc_basictests.cfg index 91777cd7cb..a8e6f0675b 100644 --- a/tools/internal_ci/windows/pull_request/grpc_basictests.cfg +++ b/tools/internal_ci/windows/pull_request/grpc_basictests.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/windows/pull_request/grpc_basictests_dbg.cfg b/tools/internal_ci/windows/pull_request/grpc_basictests_dbg.cfg index 81a1e9d3c8..ce957232fd 100644 --- a/tools/internal_ci/windows/pull_request/grpc_basictests_dbg.cfg +++ b/tools/internal_ci/windows/pull_request/grpc_basictests_dbg.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/windows/pull_request/grpc_basictests_opt.cfg b/tools/internal_ci/windows/pull_request/grpc_basictests_opt.cfg index 3bb6510ca7..7d2abbfb19 100644 --- a/tools/internal_ci/windows/pull_request/grpc_basictests_opt.cfg +++ b/tools/internal_ci/windows/pull_request/grpc_basictests_opt.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/windows/pull_request/grpc_portability.cfg b/tools/internal_ci/windows/pull_request/grpc_portability.cfg index 2bda487629..6f332416fe 100644 --- a/tools/internal_ci/windows/pull_request/grpc_portability.cfg +++ b/tools/internal_ci/windows/pull_request/grpc_portability.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } |