diff options
Diffstat (limited to 'tools')
| -rwxr-xr-x | tools/gce/create_interop_worker.sh | 50 | ||||
| -rwxr-xr-x | tools/gce/create_linux_kokoro_performance_worker.sh | 9 | ||||
| -rwxr-xr-x | tools/gce/create_linux_performance_worker.sh | 53 | ||||
| -rwxr-xr-x | tools/gce/create_linux_worker.sh | 48 | ||||
| -rw-r--r-- | tools/gce/jenkins_master.pub | 1 | ||||
| -rw-r--r-- | tools/gce/kokoro_performance.pub | 3 | ||||
| -rwxr-xr-x | tools/gce/linux_kokoro_performance_worker_init.sh | 84 | ||||
| -rwxr-xr-x | tools/gce/linux_performance_worker_init.sh | 184 | ||||
| -rwxr-xr-x | tools/gce/linux_worker_init.sh | 78 | ||||
| -rwxr-xr-x | tools/run_tests/artifacts/build_package_python.sh | 14 | ||||
| -rwxr-xr-x | tools/run_tests/performance/build_performance.sh | 4 | ||||
| -rwxr-xr-x | tools/run_tests/run_interop_tests.py | 3 |
12 files changed, 78 insertions, 453 deletions
diff --git a/tools/gce/create_interop_worker.sh b/tools/gce/create_interop_worker.sh deleted file mode 100755 index 205c0bf8c5..0000000000 --- a/tools/gce/create_interop_worker.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash -# Copyright 2015 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Creates an interop worker on GCE. -# IMPORTANT: After this script finishes, there are still some manual -# steps needed there are hard to automatize. -# See go/grpc-jenkins-setup for followup instructions. - -set -ex - -cd "$(dirname "$0")" - -CLOUD_PROJECT=grpc-testing -ZONE=us-east1-a # canary gateway is reachable from this zone - -INSTANCE_NAME="${1:-grpc-canary-interop2}" - -gcloud compute instances create "$INSTANCE_NAME" \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - --machine-type n1-standard-16 \ - --image ubuntu-15-10 \ - --boot-disk-size 1000 \ - --scopes https://www.googleapis.com/auth/xapi.zoo \ - --tags=allow-ssh - -echo 'Created GCE instance, waiting 60 seconds for it to come online.' -sleep 60 - -gcloud compute copy-files \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - jenkins_master.pub linux_worker_init.sh "${INSTANCE_NAME}":~ - -gcloud compute ssh \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - "$INSTANCE_NAME" --command "./linux_worker_init.sh" diff --git a/tools/gce/create_linux_kokoro_performance_worker.sh b/tools/gce/create_linux_kokoro_performance_worker.sh index d08a1aa2c6..63d54a48bf 100755 --- a/tools/gce/create_linux_kokoro_performance_worker.sh +++ b/tools/gce/create_linux_kokoro_performance_worker.sh @@ -15,6 +15,13 @@ # Creates a performance worker on GCE to be used on Kokoro. +# IMPORTANT: Instructions for updating +# If the VM configuration / installed software is updated, +# - all existing performance worker VMs need to be updated to reflect the changes +# - a new GCE image named "grpc-performance-kokoro-v1" needs to be created, +# incrementing the version number. +# - kokoro jobs need to be reconfigured to use the new image version + set -ex cd "$(dirname "$0")" @@ -30,7 +37,7 @@ gcloud compute instances create "$INSTANCE_NAME" \ --zone "$ZONE" \ --machine-type $MACHINE_TYPE \ --image-project ubuntu-os-cloud \ - --image-family ubuntu-1710 \ + --image-family ubuntu-1804-lts \ --boot-disk-size 300 \ --scopes https://www.googleapis.com/auth/bigquery \ --tags=allow-ssh diff --git a/tools/gce/create_linux_performance_worker.sh b/tools/gce/create_linux_performance_worker.sh deleted file mode 100755 index e9033ec443..0000000000 --- a/tools/gce/create_linux_performance_worker.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash -# Copyright 2015 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Creates a performance worker on GCE. -# IMPORTANT: After creating the worker, one needs to manually add the pubkey -# of jenkins@the-machine-where-jenkins-starts-perf-tests -# to ~/.ssh/authorized_keys so that multi-machine scenarios can work. -# See tools/run_tests/run_performance_tests.py for details. - -set -ex - -cd "$(dirname "$0")" - -CLOUD_PROJECT=grpc-testing -ZONE=us-central1-b # this zone allows 32core machines - -INSTANCE_NAME="${1:-grpc-performance-server1}" -MACHINE_TYPE=n1-standard-32 - -gcloud compute instances create "$INSTANCE_NAME" \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - --machine-type $MACHINE_TYPE \ - --image-project ubuntu-os-cloud \ - --image-family ubuntu-1710 \ - --boot-disk-size 300 \ - --scopes https://www.googleapis.com/auth/bigquery \ - --tags=allow-ssh - -echo 'Created GCE instance, waiting 60 seconds for it to come online.' -sleep 60 - -gcloud compute copy-files \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - jenkins_master.pub linux_performance_worker_init.sh "jenkins@${INSTANCE_NAME}":~ - -gcloud compute ssh \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - "jenkins@${INSTANCE_NAME}" --command "./linux_performance_worker_init.sh" diff --git a/tools/gce/create_linux_worker.sh b/tools/gce/create_linux_worker.sh deleted file mode 100755 index a93d8c5e83..0000000000 --- a/tools/gce/create_linux_worker.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash -# Copyright 2015 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Creates a standard jenkins worker on GCE. - -set -ex - -cd "$(dirname "$0")" - -CLOUD_PROJECT=grpc-testing -ZONE=us-central1-a - -INSTANCE_NAME="${1:-grpc-jenkins-worker1}" - -gcloud compute instances create "$INSTANCE_NAME" \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - --machine-type n1-standard-16 \ - --image=ubuntu-1510 \ - --image-project=grpc-testing \ - --boot-disk-size 1000 \ - --scopes https://www.googleapis.com/auth/bigquery \ - --tags=allow-ssh - -echo 'Created GCE instance, waiting 60 seconds for it to come online.' -sleep 60 - -gcloud compute copy-files \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - jenkins_master.pub linux_worker_init.sh "${INSTANCE_NAME}":~ - -gcloud compute ssh \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - "$INSTANCE_NAME" --command "./linux_worker_init.sh" diff --git a/tools/gce/jenkins_master.pub b/tools/gce/jenkins_master.pub deleted file mode 100644 index e9853224e1..0000000000 --- a/tools/gce/jenkins_master.pub +++ /dev/null @@ -1 +0,0 @@ -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDzj9l7Tp4yKnMV8sSMNvm5Q9v/F2F187xF93niJFY8lz6ig4bhusqvNbAxPoeypds9NYjLDK6kONN9teemgv2+IcmmlAI4wkCkkWcL/kzdNNH0h5J7+YbPiUGFAu0hZNHg5jzwrZ3VFKwv6d/7dUdPOYmPaOG1JOEcxXcBvm1hMIe474jpUTTiG4/gMDJ1GhMg5T3cuCm2l0gCiv7ybRAgwaZ2EKEEWLy9qAL/pnr3umBjQvzAUGcOgXJyG0mbr977YdJo9kb+EELRTVN2q8mKZJEZ1BJAylkaI9783K2+cGaM8hPtKFcX4ImEYEkWgfOyGNolGDquWtvusGGzQXwF jenkins@grpc-jenkins-master diff --git a/tools/gce/kokoro_performance.pub b/tools/gce/kokoro_performance.pub index 1154debe78..4eca916b8a 100644 --- a/tools/gce/kokoro_performance.pub +++ b/tools/gce/kokoro_performance.pub @@ -1 +1,4 @@ +# Enable Kokoro CI to SSH to the VM (Added by linux_kokoro_performance_worker_init.sh) +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDg7L/ZaEauETWrPklUTky3kvxqQfe2Ax/2CsSqhNIGNMnK/8d79CHlmY9+dE1FFQ/RzKNCaltgy7XcN/fCYiCZr5jm2ZtnLuGNOTzupMNhaYiPL419qmL+5rZXt4/dWTrsHbFRACxT8j51PcRMO5wgbL0Bg2XXimbx8kDFaurL2gqduQYqlu4lxWCaJqOL71WogcimeL63Nq/yeH5PJPWpqE4P9VUQSwAzBWFK/hLeds/AiP3MgVS65qHBnhq0JsHy8JQsqjZbG7Iidt/Ll0+gqzEbi62gDIcczG4KC0iOVzDDP/1BxDtt1lKeA23ll769Fcm3rJyoBMYxjvdw1TDx sabujp@trigger.mtv.corp.google.com +# Enable kokoro multi-machine benchmark driver VM to SSH to the VM (Added by linux_kokoro_performance_worker_init.sh) ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDKQ5UEX4AFefec9BKICupFS7x9Hoq4ZyLKy+QX0J31I49ew9mG2AJlr3sp8ql15eX+A2Ml9MKJkmgZGHpJtw+SfvmI94SmomSyiCLAK92sQ85NMzaRdo4b9e30E9nhXnAvAaemvIEQbgCMYFvzk0C8AtXj6+htCrN4jFaLqTCPISJhX3ETc4TgX1qaHQHyl31tdaXHYlITvBDsfokcGcZQnhmCUDtD8wyaSC8GFk9gZbXshkfaYCuuLPPA0vwWGBw+YPbonHsFCsOog1IYSzYPCkIjq8dt6evsusK6Kaoyw/Z+l2kYty2FKTj+wU3l06QMoxwcfNT4WxdhcnVbY71r kbuilder@kokoro-performance-driver diff --git a/tools/gce/linux_kokoro_performance_worker_init.sh b/tools/gce/linux_kokoro_performance_worker_init.sh index 4a1e3e608b..b78695d802 100755 --- a/tools/gce/linux_kokoro_performance_worker_init.sh +++ b/tools/gce/linux_kokoro_performance_worker_init.sh @@ -47,7 +47,6 @@ sudo apt-get install -y \ libtool \ make \ strace \ - pypy \ python-dev \ python-pip \ python-setuptools \ @@ -68,30 +67,34 @@ sudo apt-get install -y google-perftools libgoogle-perftools-dev # netperf sudo apt-get install -y netperf +# required to run kokoro_log_reader.py +sudo apt-get install -y python-psutil python3-psutil + +# gcloud tools, including gsutil +sudo apt-get install -y google-cloud-sdk + # C++ dependencies sudo apt-get install -y libgflags-dev libgtest-dev libc++-dev clang # Python dependencies sudo pip install --upgrade pip==10.0.1 sudo pip install tabulate -sudo pip install google-api-python-client +sudo pip install google-api-python-client oauth2client sudo pip install virtualenv -# Building gRPC Python depends on python3.4 being installed, but python3.4 -# is not available on Ubuntu 16.10, so install from source -curl -O https://www.python.org/ftp/python/3.4.6/Python-3.4.6.tgz -tar xzvf Python-3.4.6.tgz -( -cd Python-3.4.6 || exit -./configure --enable-shared --prefix=/usr/local LDFLAGS="-Wl,--rpath=/usr/local/lib" -sudo make altinstall -) -rm Python-3.4.6.tgz - +# pypy is used instead of python for postprocessing benchmark outputs +# because some reports are huge and pypy is much faster. +# TODO(jtattermusch): get rid of pypy once possible, it's hard to +# keep track of all the installed variants of python. +sudo apt-get install -y pypy pypy-dev curl -O https://bootstrap.pypa.io/get-pip.py sudo pypy get-pip.py sudo pypy -m pip install tabulate -sudo pip install google-api-python-client +sudo pypy -m pip install google-api-python-client oauth2client +# TODO(jtattermusch): for some reason, we need psutil installed +# in pypy for kokoro_log_reader.py (strange, because the comand is +# "python kokoro_log_reader.py" and pypy is not the system default) +sudo pypy -m pip install psutil # Node dependencies (nvm has to be installed under user kbuilder) touch .profile @@ -104,31 +107,31 @@ nvm install 4 && npm config set cache /tmp/npm-cache nvm install 5 && npm config set cache /tmp/npm-cache nvm alias default 4 +# C# dependencies +sudo apt-get install -y cmake + # C# mono dependencies (http://www.mono-project.com/docs/getting-started/install/linux/#debian-ubuntu-and-derivatives) sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF -echo "deb http://download.mono-project.com/repo/debian wheezy main" | sudo tee /etc/apt/sources.list.d/mono-xamarin.list +echo "deb https://download.mono-project.com/repo/ubuntu stable-bionic main" | sudo tee /etc/apt/sources.list.d/mono-official-stable.list sudo apt-get update -sudo apt-get install -y mono-devel nuget +sudo apt-get install -y mono-devel -# C# .NET Core dependencies (https://www.microsoft.com/net/core#ubuntu) -sudo sh -c 'echo "deb [arch=amd64] https://apt-mo.trafficmanager.net/repos/dotnet-release/ yakkety main" > /etc/apt/sources.list.d/dotnetdev.list' -sudo apt-key adv --keyserver apt-mo.trafficmanager.net --recv-keys 417A0893 +# C# .NET Core dependencies (https://www.microsoft.com/net/download) +wget -q https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb +sudo dpkg -i packages-microsoft-prod.deb + +sudo apt-get install -y apt-transport-https sudo apt-get update -sudo apt-get install -y dotnet-dev-1.0.0-preview2.1-003155 -sudo apt-get install -y dotnet-dev-1.0.1 - -# C# 1.0.4 SDK -curl -O https://download.microsoft.com/download/2/4/A/24A06858-E8AC-469B-8AE6-D0CEC9BA982A/dotnet-ubuntu.16.04-x64.1.0.5.tar.gz -sudo mkdir -p /opt/dotnet -sudo tar zxf dotnet-ubuntu.16.04-x64.1.0.5.tar.gz -C /opt/dotnet -sudo ln -s /opt/dotnet/dotnet /usr/local/bin - -# C# .NET dependencies -wget http://security.ubuntu.com/ubuntu/pool/main/i/icu/libicu52_52.1-8ubuntu0.2_amd64.deb -sudo dpkg -i libicu52_52.1-8ubuntu0.2_amd64.deb -wget http://security.ubuntu.com/ubuntu/pool/main/i/icu/libicu55_55.1-7ubuntu0.3_amd64.deb -sudo dpkg -i libicu55_55.1-7ubuntu0.3_amd64.deb -sudo apt-get update && sudo apt-get install -y libicu55 +sudo apt-get install -y dotnet-sdk-2.1 + +# Install .NET Core 1.0.5 Runtime (required to run netcoreapp1.0) +wget -q https://download.microsoft.com/download/2/4/A/24A06858-E8AC-469B-8AE6-D0CEC9BA982A/dotnet-ubuntu.16.04-x64.1.0.5.tar.gz +mkdir -p dotnet105_download +tar zxf dotnet-ubuntu.16.04-x64.1.0.5.tar.gz -C dotnet105_download +sudo cp -r dotnet105_download/shared/Microsoft.NETCore.App/1.0.5/ /usr/share/dotnet/shared/Microsoft.NETCore.App/ +# To prevent "Failed to initialize CoreCLR, HRESULT: 0x80131500" with .NET Core 1.0.5 runtime +wget -q http://security.ubuntu.com/ubuntu/pool/main/i/icu/libicu55_55.1-7ubuntu0.4_amd64.deb +sudo dpkg -i libicu55_55.1-7ubuntu0.4_amd64.deb # Ruby dependencies gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 @@ -163,7 +166,7 @@ sudo mv composer.phar /usr/local/bin/composer # Significant performance improvements with grpc-go have been observed after # upgrading from go 1.5 to a later version, so a later go version is preferred. # Following go install instructions from https://golang.org/doc/install -GO_VERSION=1.8 +GO_VERSION=1.10 OS=linux ARCH=amd64 curl -O https://storage.googleapis.com/golang/go${GO_VERSION}.${OS}-${ARCH}.tar.gz @@ -190,11 +193,22 @@ git clone -v https://github.com/brendangregg/FlameGraph ~/FlameGraph # Install scipy and numpy for benchmarking scripts sudo apt-get install -y python-scipy python-numpy +# Install docker +curl -sSL https://get.docker.com/ | sh +# Enable kbuilder to use docker without sudo: +sudo usermod -aG docker kbuilder + # Add pubkey of Kokoro driver VM to allow SSH # silence false-positive shellcheck warning ("< redirect does not affect sudo") # shellcheck disable=SC2024 sudo tee --append ~kbuilder/.ssh/authorized_keys < kokoro_performance.pub +# Kokoro requires /tmpfs/READY file to exist the directory and file itself should +# be owned by kbuilder. +sudo mkdir /tmpfs +sudo chown kbuilder /tmpfs +touch /tmpfs/READY + # Restart for VM to pick up kernel update echo 'Successfully initialized the linux worker, going for reboot in 10 seconds' sleep 10 diff --git a/tools/gce/linux_performance_worker_init.sh b/tools/gce/linux_performance_worker_init.sh deleted file mode 100755 index 7222cef9a2..0000000000 --- a/tools/gce/linux_performance_worker_init.sh +++ /dev/null @@ -1,184 +0,0 @@ -#!/bin/bash -# Copyright 2015 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Initializes a fresh GCE VM to become a jenkins linux performance worker. -# You shouldn't run this script on your own, -# use create_linux_performance_worker.sh instead. - -set -ex - -sudo apt-get update - -# Install Java 8 JDK (to build gRPC Java) -sudo apt-get install -y openjdk-8-jdk -sudo apt-get install -y unzip lsof - -sudo apt-get install -y \ - autoconf \ - autotools-dev \ - build-essential \ - bzip2 \ - ccache \ - curl \ - gcc \ - gcc-multilib \ - git \ - gyp \ - lcov \ - libc6 \ - libc6-dbg \ - libc6-dev \ - libcurl4-openssl-dev \ - libgtest-dev \ - libreadline-dev \ - libssl-dev \ - libtool \ - make \ - strace \ - pypy \ - python-dev \ - python-pip \ - python-setuptools \ - python-yaml \ - python3-dev \ - python3-pip \ - python3-setuptools \ - python3-yaml \ - telnet \ - unzip \ - wget \ - zip \ - zlib1g-dev - -# perftools -sudo apt-get install -y google-perftools libgoogle-perftools-dev - -# netperf -sudo apt-get install -y netperf - -# C++ dependencies -sudo apt-get install -y libgflags-dev libgtest-dev libc++-dev clang - -# Python dependencies -sudo pip install --upgrade pip==10.0.1 -sudo pip install tabulate -sudo pip install google-api-python-client -sudo pip install virtualenv - -# Building gRPC Python depends on python3.4 being installed, but python3.4 -# is not available on Ubuntu 16.10, so install from source -curl -O https://www.python.org/ftp/python/3.4.6/Python-3.4.6.tgz -tar xzvf Python-3.4.6.tgz -( -cd Python-3.4.6 || exit -./configure --enable-shared --prefix=/usr/local LDFLAGS="-Wl,--rpath=/usr/local/lib" -sudo make altinstall -) -rm Python-3.4.6.tgz - -curl -O https://bootstrap.pypa.io/get-pip.py -sudo pypy get-pip.py -sudo pypy -m pip install tabulate -sudo pip install google-api-python-client - -# Node dependencies (nvm has to be installed under user jenkins) -touch .profile -curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.25.4/install.sh | bash -# silence shellcheck warning as it cannot follow the `source` path statically: -# shellcheck disable=SC1090 -source ~/.nvm/nvm.sh -nvm install 0.12 && npm config set cache /tmp/npm-cache -nvm install 4 && npm config set cache /tmp/npm-cache -nvm install 5 && npm config set cache /tmp/npm-cache -nvm alias default 4 - -# C# mono dependencies (http://www.mono-project.com/docs/getting-started/install/linux/#debian-ubuntu-and-derivatives) -sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF -echo "deb http://download.mono-project.com/repo/debian wheezy main" | sudo tee /etc/apt/sources.list.d/mono-xamarin.list -sudo apt-get update -sudo apt-get install -y mono-devel nuget - -# C# .NET Core dependencies (https://www.microsoft.com/net/core#ubuntu) -sudo sh -c 'echo "deb [arch=amd64] https://apt-mo.trafficmanager.net/repos/dotnet-release/ yakkety main" > /etc/apt/sources.list.d/dotnetdev.list' -sudo apt-key adv --keyserver apt-mo.trafficmanager.net --recv-keys 417A0893 -sudo apt-get update -sudo apt-get install -y dotnet-dev-1.0.0-preview2.1-003155 -sudo apt-get install -y dotnet-dev-1.0.1 - -# Ruby dependencies -git clone https://github.com/rbenv/rbenv.git ~/.rbenv -export PATH="$HOME/.rbenv/bin:$PATH" -eval "$(rbenv init -)" - -git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build -export PATH="$HOME/.rbenv/plugins/ruby-build/bin:$PATH" - -rbenv install 2.4.0 -rbenv global 2.4.0 -ruby -v - -# Install bundler (prerequisite for gRPC Ruby) -gem install bundler - -# PHP dependencies -sudo apt-get install -y php php-dev phpunit php-pear unzip zlib1g-dev -curl -sS https://getcomposer.org/installer | php -sudo mv composer.phar /usr/local/bin/composer - -# Java dependencies - nothing as we already have Java JDK 8 - -# Go dependencies -# Currently, the golang package available via apt-get doesn't have the latest go. -# Significant performance improvements with grpc-go have been observed after -# upgrading from go 1.5 to a later version, so a later go version is preferred. -# Following go install instructions from https://golang.org/doc/install -GO_VERSION=1.8 -OS=linux -ARCH=amd64 -curl -O https://storage.googleapis.com/golang/go${GO_VERSION}.${OS}-${ARCH}.tar.gz -sudo tar -C /usr/local -xzf go$GO_VERSION.$OS-$ARCH.tar.gz -# Put go on the PATH, keep the usual installation dir -sudo ln -s /usr/local/go/bin/go /usr/bin/go -rm go$GO_VERSION.$OS-$ARCH.tar.gz - -# Install perf, to profile benchmarks. (need to get the right linux-tools-<> for kernel version) -sudo apt-get install -y linux-tools-common linux-tools-generic "linux-tools-$(uname -r)" -# see http://unix.stackexchange.com/questions/14227/do-i-need-root-admin-permissions-to-run-userspace-perf-tool-perf-events-ar -echo 0 | sudo tee /proc/sys/kernel/perf_event_paranoid -# see http://stackoverflow.com/questions/21284906/perf-couldnt-record-kernel-reference-relocation-symbol -echo 0 | sudo tee /proc/sys/kernel/kptr_restrict - -# qps workers under perf appear to need a lot of mmap pages under certain scenarios and perf args in -# order to not lose perf events or time out -echo 4096 | sudo tee /proc/sys/kernel/perf_event_mlock_kb - -# Fetch scripts to generate flame graphs from perf data collected -# on benchmarks -git clone -v https://github.com/brendangregg/FlameGraph ~/FlameGraph - -# Install scipy and numpy for benchmarking scripts -sudo apt-get install -y python-scipy python-numpy - -# Add pubkey of jenkins@grpc-jenkins-master to authorized keys of jenkins@ -# This needs to happen as the last step to prevent Jenkins master from connecting -# to a machine that hasn't been properly setup yet. -# silence false-positive shellcheck warning ("< redirect does not affect sudo") -# shellcheck disable=SC2024 -sudo tee --append ~jenkins/.ssh/authorized_keys < jenkins_master.pub - -# Restart for VM to pick up kernel update -echo 'Successfully initialized the linux worker, going for reboot in 10 seconds' -sleep 10 -sudo reboot diff --git a/tools/gce/linux_worker_init.sh b/tools/gce/linux_worker_init.sh deleted file mode 100755 index 05855354ff..0000000000 --- a/tools/gce/linux_worker_init.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash -# Copyright 2015 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Initializes a fresh GCE VM to become a jenkins linux worker. -# You shouldn't run this script on your own, use create_linux_worker.sh -# instead. - -set -ex - -# Create some swap space -sudo dd if=/dev/zero of=/swap bs=1024 count=10485760 -sudo chmod 600 /swap -sudo mkswap /swap -sudo sed -i '$ a\/swap none swap sw 0 0' /etc/fstab -sudo swapon -a - -# Typical apt-get maintenance -sudo apt-get update - -# Install JRE -sudo apt-get install -y openjdk-8-jre -sudo apt-get install -y unzip lsof - -# Install Docker -curl -sSL https://get.docker.com/ | sh - -# Setup jenkins user (or the user will already exist bcuz magic) -sudo adduser jenkins --disabled-password || true - -# Enable jenkins to use docker without sudo: -sudo usermod -aG docker jenkins - -# Use "overlay" storage driver for docker -# see https://github.com/grpc/grpc/issues/4988 -printf "{\n\t\"storage-driver\": \"overlay\"\n}" | sudo tee /etc/docker/daemon.json - -# Install pip and Google API library to enable using GCP services -sudo apt-get install -y python-pip -sudo pip install google-api-python-client - -# Install RVM -# TODO(jtattermusch): why is RVM needed? -gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 -curl -sSL https://get.rvm.io | bash -s stable --ruby - -# Upgrade Linux kernel to 4.9 -wget \ - kernel.ubuntu.com/~kernel-ppa/mainline/v4.9.20/linux-headers-4.9.20-040920_4.9.20-040920.201703310531_all.deb \ - kernel.ubuntu.com/~kernel-ppa/mainline/v4.9.20/linux-headers-4.9.20-040920-generic_4.9.20-040920.201703310531_amd64.deb \ - kernel.ubuntu.com/~kernel-ppa/mainline/v4.9.20/linux-image-4.9.20-040920-generic_4.9.20-040920.201703310531_amd64.deb -sudo dpkg -i linux-headers-4.9*.deb linux-image-4.9*.deb -rm linux-* - -# Add pubkey of jenkins@grpc-jenkins-master to authorized keys of jenkins@ -# This needs to happen as the last step to prevent Jenkins master from connecting -# to a machine that hasn't been properly setup yet. - -# disable superfluous warning by shellcheck: -# shellcheck disable=SC2024 -sudo tee --append ~jenkins/.ssh/authorized_keys < jenkins_master.pub - -# Restart for docker to pick up the config changes. -echo 'Successfully initialized the linux worker, going for reboot in 10 seconds' -sleep 10 - -sudo reboot diff --git a/tools/run_tests/artifacts/build_package_python.sh b/tools/run_tests/artifacts/build_package_python.sh index 29801a5b86..d93e8979fc 100755 --- a/tools/run_tests/artifacts/build_package_python.sh +++ b/tools/run_tests/artifacts/build_package_python.sh @@ -19,10 +19,20 @@ cd "$(dirname "$0")/../../.." mkdir -p artifacts/ -# All the python packages have been built in the artifact phase already -# and we only collect them here to deliver them to the distribtest phase. cp -r "${EXTERNAL_GIT_ROOT}"/input_artifacts/python_*/* artifacts/ || true +strip_binary_wheel() { + TEMP_WHEEL_DIR=$(mktemp -d) + unzip "$1" -d "$TEMP_WHEEL_DIR" + find "$TEMP_WHEEL_DIR" -name "_protoc_compiler*.so" -exec strip --strip-debug {} ";" + find "$TEMP_WHEEL_DIR" -name "cygrpc*.so" -exec strip --strip-debug {} ";" + (cd "$TEMP_WHEEL_DIR" && zip -r - .) > "$1" +} + +for wheel in artifacts/*.whl; do + strip_binary_wheel "$wheel" +done + # TODO: all the artifact builder configurations generate a grpcio-VERSION.tar.gz # source distribution package, and only one of them will end up # in the artifacts/ directory. They should be all equivalent though. diff --git a/tools/run_tests/performance/build_performance.sh b/tools/run_tests/performance/build_performance.sh index f235582579..9e6e72d97b 100755 --- a/tools/run_tests/performance/build_performance.sh +++ b/tools/run_tests/performance/build_performance.sh @@ -61,6 +61,10 @@ do "node"|"node_purejs") tools/run_tests/performance/build_performance_node.sh ;; + "python") + # python workers are only run with python2.7 and building with multiple python versions is costly + python tools/run_tests/run_tests.py -l "$language" -c "$CONFIG" --compiler python2.7 --build_only -j 8 + ;; *) python tools/run_tests/run_tests.py -l "$language" -c "$CONFIG" --build_only -j 8 ;; diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index e8d6b59687..9752e741fb 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -1376,7 +1376,8 @@ try: transport_security='tls') jobs.append(tls_test_job) if str(language) in [ - 'go' + 'c++', + 'go', ]: # Add more languages to the list to turn on tests. google_default_creds_test_job = cloud_to_prod_jobspec( language, |