diff options
Diffstat (limited to 'tools/internal_ci/helper_scripts')
6 files changed, 65 insertions, 11 deletions
diff --git a/tools/internal_ci/helper_scripts/gen_report_index.sh b/tools/internal_ci/helper_scripts/gen_report_index.sh index 0af89c331c..576ff67d54 100755 --- a/tools/internal_ci/helper_scripts/gen_report_index.sh +++ b/tools/internal_ci/helper_scripts/gen_report_index.sh @@ -26,7 +26,7 @@ mkdir -p reports echo '<html><head></head><body>' > reports/kokoro_index.html echo '<h1>'${KOKORO_JOB_NAME}', build '#${KOKORO_BUILD_NUMBER}'</h1>' >> reports/kokoro_index.html -echo '<h2><a href="https://kokoro.corp.google.com/job/'${KOKORO_JOB_PATH}'/'${KOKORO_BUILD_NUMBER}'/">Kokoro build dashboard (internal only)</a></h2>' >> reports/kokoro_index.html +echo '<h2><a href="https://kokoro2.corp.google.com/job/'${KOKORO_JOB_PATH}'/'${KOKORO_BUILD_NUMBER}'/">Kokoro build dashboard (internal only)</a></h2>' >> reports/kokoro_index.html echo '<h2><a href="https://sponge.corp.google.com/invocation?id='${KOKORO_BUILD_ID}'&searchFor=">Test result dashboard (internal only)</a></h2>' >> reports/kokoro_index.html echo '<h2><a href="test_report.html">HTML test report (Not available yet)</a></h2>' >> reports/kokoro_index.html echo '<h2><a href="test_log.txt">Test log (Not available yet)</a></h2>' >> reports/kokoro_index.html diff --git a/tools/internal_ci/helper_scripts/prepare_build_interop_rc b/tools/internal_ci/helper_scripts/prepare_build_interop_rc index 859ce621f9..db978c8b3f 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_interop_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_interop_rc @@ -26,6 +26,7 @@ git submodule update --init # Set up gRPC-Go and gRPC-Java to test git clone --recursive https://github.com/grpc/grpc-go ./../grpc-go git clone --recursive https://github.com/grpc/grpc-java ./../grpc-java +git clone --recursive https://github.com/grpc/grpc-node ./../grpc-node # Download json file. mkdir ~/service_account diff --git a/tools/internal_ci/helper_scripts/prepare_build_linux_perf_multilang_rc b/tools/internal_ci/helper_scripts/prepare_build_linux_perf_multilang_rc new file mode 100644 index 0000000000..f1031aedf3 --- /dev/null +++ b/tools/internal_ci/helper_scripts/prepare_build_linux_perf_multilang_rc @@ -0,0 +1,40 @@ +#!/bin/bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Source this rc script to prepare the environment for linux perf builds + +# Need to increase open files limit and size for perf test +ulimit -n 32768 +ulimit -c unlimited + +# Download non-core gRPC repos +git clone --recursive https://github.com/grpc/grpc-go ./../grpc-go +git clone --recursive https://github.com/grpc/grpc-java ./../grpc-java +git clone --recursive https://github.com/grpc/grpc-node ./../grpc-node + +sudo pip install tabulate + +# Set up Ruby +export PATH="$HOME/.rbenv/bin:$PATH" +eval "$(rbenv init -)" +gem list bundler +gem install bundler --no-ri --no-rdoc + +# Allow SSH to Kokoro performance workers without explicit key verification +gsutil cp gs://grpc-testing-secrets/grpc_kokoro_performance_ssh_keys/id_rsa ~/.ssh +echo -e 'Host grpc-kokoro-performance*\n\tStrictHostKeyChecking no' >> ~/.ssh/config +chmod 600 ~/.ssh/id_rsa ~/.ssh/config + +git submodule update --init diff --git a/tools/internal_ci/helper_scripts/prepare_build_linux_rc b/tools/internal_ci/helper_scripts/prepare_build_linux_rc index 2ade8dac51..8715d6c8e0 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_linux_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_linux_rc @@ -22,6 +22,9 @@ ulimit -n 32768 # Move docker's storage location to scratch disk so we don't run out of space. echo 'DOCKER_OPTS="${DOCKER_OPTS} --graph=/tmpfs/docker"' | sudo tee --append /etc/default/docker +# Use container registry mirror for pulling docker images (should make downloads faster) +# See https://cloud.google.com/container-registry/docs/using-dockerhub-mirroring +echo 'DOCKER_OPTS="${DOCKER_OPTS} --registry-mirror=https://mirror.gcr.io"' | sudo tee --append /etc/default/docker sudo service docker restart # Populate xdg-cache-home to workaround https://github.com/grpc/grpc/issues/11968 @@ -32,11 +35,4 @@ PYTHONWARNINGS=ignore XDG_CACHE_HOME=/tmp/xdg-cache-home sudo -E pip install cov # Download Docker images from DockerHub export DOCKERHUB_ORGANIZATION=grpctesting -# If this is a PR using RUN_TESTS_FLAGS var, then add flags to filter tests -if [ -n "$KOKORO_GITHUB_PULL_REQUEST_NUMBER" ] && [ -n "$RUN_TESTS_FLAGS" ]; then - sudo apt-get install -y jq - ghprbTargetBranch=$(curl -s https://api.github.com/repos/grpc/grpc/pulls/$KOKORO_GITHUB_PULL_REQUEST_NUMBER | jq -r .base.ref) - export RUN_TESTS_FLAGS="$RUN_TESTS_FLAGS --filter_pr_tests --base_branch origin/$ghprbTargetBranch" -fi - git submodule update --init diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc index f467ac006a..6ecf51dd3f 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc @@ -25,11 +25,13 @@ cp -R ./ /Users/kbuilder/workspace/grpc cd /Users/kbuilder/workspace/grpc # Needed for identifying Docker image sha1 +brew update brew install md5sha1sum # Set up gRPC-Go and gRPC-Java to test git clone --recursive https://github.com/grpc/grpc-go ./../grpc-go git clone --recursive https://github.com/grpc/grpc-java ./../grpc-java +git clone --recursive https://github.com/grpc/grpc-node ./../grpc-node # Set up Docker for Mac docker-machine create -d virtualbox --virtualbox-share-folder "/Users/kbuilder/workspace:" default diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc index bec529f85e..3a09701a30 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc @@ -27,12 +27,21 @@ ulimit -n 10000 # show current limits ulimit -a +# synchronize the clock +date +sudo systemsetup -setusingnetworktime off +sudo systemsetup -setnetworktimeserver "$( ipconfig getoption en0 server_identifier )" +sudo systemsetup -settimezone America/Los_Angeles +sudo systemsetup -setusingnetworktime on +date + # Add GCP credentials for BQ access pip install google-api-python-client --user python export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json # If this is a PR using RUN_TESTS_FLAGS var, then add flags to filter tests if [ -n "$KOKORO_GITHUB_PULL_REQUEST_NUMBER" ] && [ -n "$RUN_TESTS_FLAGS" ]; then + brew update brew install jq ghprbTargetBranch=$(curl -s https://api.github.com/repos/grpc/grpc/pulls/$KOKORO_GITHUB_PULL_REQUEST_NUMBER | jq -r .base.ref) export RUN_TESTS_FLAGS="$RUN_TESTS_FLAGS --filter_pr_tests --base_branch origin/$ghprbTargetBranch" @@ -41,7 +50,11 @@ fi set +ex # rvm script is very verbose and exits with errorcode source $HOME/.rvm/scripts/rvm set -e # rvm commands are very verbose -rvm use ruby-2.4 +time rvm install 2.5.0 +rvm use 2.5.0 --default +gem install bundler --no-ri --no-doc +gem install cocoapods --version 1.3.1 --no-ri --no-doc +gem install rake-compiler --no-ri --no-doc rvm osx-ssl-certs status all rvm osx-ssl-certs update all set -ex @@ -51,12 +64,14 @@ export LANG=en_US.UTF-8 pod repo update # needed by python # python -brew install coreutils # we need grealpath pip install virtualenv --user python -pip install -U six tox setuptools twisted pyyaml --user python +pip install -U Mako six tox setuptools twisted pyyaml --user python export PYTHONPATH=/Library/Python/3.4/site-packages # set xcode version for Obj-C tests sudo xcode-select -switch /Applications/Xcode_8.2.1.app/Contents/Developer +# TODO(jtattermusch): better debugging of clock skew, remove once not needed +date + git submodule update --init |