From b95aa40ede91f023c79f1d7933fc6c8b6be1f620 Mon Sep 17 00:00:00 2001
From: Matt Kwong <mattkwong@google.com>
Date: Fri, 13 Oct 2017 13:25:23 -0700
Subject: Add setup scripts for performance worker in Kokoro

---
 tools/gce/linux_kokoro_performance_worker_init.sh | 181 ++++++++++++++++++++++
 1 file changed, 181 insertions(+)
 create mode 100755 tools/gce/linux_kokoro_performance_worker_init.sh

(limited to 'tools/gce/linux_kokoro_performance_worker_init.sh')

diff --git a/tools/gce/linux_kokoro_performance_worker_init.sh b/tools/gce/linux_kokoro_performance_worker_init.sh
new file mode 100755
index 0000000000..ac3d39383b
--- /dev/null
+++ b/tools/gce/linux_kokoro_performance_worker_init.sh
@@ -0,0 +1,181 @@
+#!/bin/bash
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Initializes a fresh GCE VM to become a Kokoro Linux performance worker.
+# You shouldn't run this script on your own,
+# use create_linux_kokoro_performance_worker.sh instead.
+
+set -ex
+
+sudo apt-get update
+
+# Install Java 8 JDK (to build gRPC Java)
+sudo apt-get install -y openjdk-8-jdk
+sudo apt-get install -y unzip lsof
+
+sudo apt-get install -y \
+  autoconf \
+  autotools-dev \
+  build-essential \
+  bzip2 \
+  ccache \
+  curl \
+  gcc \
+  gcc-multilib \
+  git \
+  gyp \
+  lcov \
+  libc6 \
+  libc6-dbg \
+  libc6-dev \
+  libcurl4-openssl-dev \
+  libgtest-dev \
+  libreadline-dev \
+  libssl-dev \
+  libtool \
+  make \
+  strace \
+  pypy \
+  python-dev \
+  python-pip \
+  python-setuptools \
+  python-yaml \
+  python3-dev \
+  python3-pip \
+  python3-setuptools \
+  python3-yaml \
+  telnet \
+  unzip \
+  wget \
+  zip \
+  zlib1g-dev
+
+# perftools
+sudo apt-get install -y google-perftools libgoogle-perftools-dev
+
+# netperf
+sudo apt-get install -y netperf
+
+# C++ dependencies
+sudo apt-get install -y libgflags-dev libgtest-dev libc++-dev clang
+
+# Python dependencies
+sudo pip install --upgrade pip==9.0.1
+sudo pip install tabulate
+sudo pip install google-api-python-client
+sudo pip install virtualenv
+
+# Building gRPC Python depends on python3.4 being installed, but python3.4
+# is not available on Ubuntu 16.10, so install from source
+curl -O https://www.python.org/ftp/python/3.4.6/Python-3.4.6.tgz
+tar xzvf Python-3.4.6.tgz
+cd Python-3.4.6
+./configure --enable-shared --prefix=/usr/local LDFLAGS="-Wl,--rpath=/usr/local/lib"
+sudo make altinstall
+cd ..
+rm Python-3.4.6.tgz
+
+curl -O https://bootstrap.pypa.io/get-pip.py
+sudo pypy get-pip.py
+sudo pypy -m pip install tabulate
+sudo pip install google-api-python-client
+
+# Node dependencies (nvm has to be installed under user kbuilder)
+touch .profile
+curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.25.4/install.sh | bash
+source ~/.nvm/nvm.sh
+nvm install 0.12 && npm config set cache /tmp/npm-cache
+nvm install 4 && npm config set cache /tmp/npm-cache
+nvm install 5 && npm config set cache /tmp/npm-cache
+nvm alias default 4
+
+# C# mono dependencies (http://www.mono-project.com/docs/getting-started/install/linux/#debian-ubuntu-and-derivatives)
+sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
+echo "deb http://download.mono-project.com/repo/debian wheezy main" | sudo tee /etc/apt/sources.list.d/mono-xamarin.list
+sudo apt-get update
+sudo apt-get install -y mono-devel nuget
+
+# C# .NET Core dependencies (https://www.microsoft.com/net/core#ubuntu)
+sudo sh -c 'echo "deb [arch=amd64] https://apt-mo.trafficmanager.net/repos/dotnet-release/ yakkety main" > /etc/apt/sources.list.d/dotnetdev.list'
+sudo apt-key adv --keyserver apt-mo.trafficmanager.net --recv-keys 417A0893
+sudo apt-get update
+sudo apt-get install -y dotnet-dev-1.0.0-preview2.1-003155
+sudo apt-get install -y dotnet-dev-1.0.1
+
+# Ruby dependencies
+gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3
+curl -sSL https://get.rvm.io | bash -s stable --ruby
+source ~/.rvm/scripts/rvm
+
+git clone https://github.com/rbenv/rbenv.git ~/.rbenv
+export PATH="$HOME/.rbenv/bin:$PATH"
+eval "$(rbenv init -)"
+
+git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build
+export PATH="$HOME/.rbenv/plugins/ruby-build/bin:$PATH"
+
+rbenv install 2.4.0
+rbenv global 2.4.0
+ruby -v
+
+# Install bundler (prerequisite for gRPC Ruby)
+gem install bundler
+
+# PHP dependencies
+sudo apt-get install -y php php-dev phpunit php-pear unzip zlib1g-dev
+curl -sS https://getcomposer.org/installer | php
+sudo mv composer.phar /usr/local/bin/composer
+
+# Java dependencies - nothing as we already have Java JDK 8
+
+# Go dependencies
+# Currently, the golang package available via apt-get doesn't have the latest go.
+# Significant performance improvements with grpc-go have been observed after
+# upgrading from go 1.5 to a later version, so a later go version is preferred.
+# Following go install instructions from https://golang.org/doc/install
+GO_VERSION=1.8
+OS=linux
+ARCH=amd64
+curl -O https://storage.googleapis.com/golang/go${GO_VERSION}.${OS}-${ARCH}.tar.gz
+sudo tar -C /usr/local -xzf go$GO_VERSION.$OS-$ARCH.tar.gz
+# Put go on the PATH, keep the usual installation dir
+sudo ln -s /usr/local/go/bin/go /usr/bin/go
+rm go$GO_VERSION.$OS-$ARCH.tar.gz
+
+# Install perf, to profile benchmarks. (need to get the right linux-tools-<> for kernel version)
+sudo apt-get install -y linux-tools-common linux-tools-generic linux-tools-`uname -r`
+# see http://unix.stackexchange.com/questions/14227/do-i-need-root-admin-permissions-to-run-userspace-perf-tool-perf-events-ar
+echo 0 | sudo tee /proc/sys/kernel/perf_event_paranoid
+# see http://stackoverflow.com/questions/21284906/perf-couldnt-record-kernel-reference-relocation-symbol
+echo 0 | sudo tee /proc/sys/kernel/kptr_restrict
+
+# qps workers under perf appear to need a lot of mmap pages under certain scenarios and perf args in
+# order to not lose perf events or time out
+echo 4096 | sudo tee /proc/sys/kernel/perf_event_mlock_kb
+
+# Fetch scripts to generate flame graphs from perf data collected
+# on benchmarks
+git clone -v https://github.com/brendangregg/FlameGraph ~/FlameGraph
+
+# Install scipy and numpy for benchmarking scripts
+sudo apt-get install -y python-scipy python-numpy
+
+# Add pubkey of Kokoro driver VM to allow SSH
+cat kokoro_performance.pub | sudo tee --append ~kbuilder/.ssh/authorized_keys
+
+# Restart for VM to pick up kernel update
+echo 'Successfully initialized the linux worker, going for reboot in 10 seconds'
+sleep 10
+sudo reboot
-- 
cgit v1.2.3