15 files changed, 270 insertions, 40 deletions
diff --git a/tools/internal_ci/helper_scripts/prepare_build_interop_rc b/tools/internal_ci/helper_scripts/prepare_build_interop_rc
index cc956b8075..859ce621f9 100644
--- a/tools/internal_ci/helper_scripts/prepare_build_interop_rc
+++ b/tools/internal_ci/helper_scripts/prepare_build_interop_rc
@@ -26,3 +26,8 @@ git submodule update --init
 # Set up gRPC-Go and gRPC-Java to test
 git clone --recursive https://github.com/grpc/grpc-go ./../grpc-go
 git clone --recursive https://github.com/grpc/grpc-java ./../grpc-java
+
+# Download json file.
+mkdir ~/service_account
+gsutil cp gs://grpc-testing-secrets/interop/service_account/GrpcTesting-726eb1347f15.json ~/service_account
+export GOOGLE_APPLICATION_CREDENTIALS=~/service_account/GrpcTesting-726eb1347f15.json
diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc
index 3851e565a4..57463e6f6d 100644
--- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc
+++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc
@@ -15,53 +15,46 @@
 
 # Source this rc script to prepare the environment for macos builds
 
-# TODO(jtattermusch): remove all deps once installed on MacOS workers
+sudo launchctl limit maxfiles unlimited unlimited
 
-# brew and C++ deps
-yes | ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
-brew install autoconf automake libtool ccache cmake gflags gpg wget
+# show current maxfiles
+launchctl limit maxfiles
 
-# TODO(jtattermusch): hkp://keys.gnupg.net fails with "No route to host"
-gpg --keyserver hkp://193.164.133.100 --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3
-curl -sSL https://get.rvm.io | sudo bash -s stable
+ulimit -n 10000
 
-# add ourselves to rvm group to prevent later "access denied" errors.
-sudo dseditgroup -o edit -a `whoami` -t user rvm
+# show current limits
+ulimit -a
 
-set +ex
-source /etc/profile.d/rvm.sh
-rvm install ruby-2.3
-gem install bundler
 
+# required to build protobuf
+brew install gflags
+
+set +ex  # rvm script is very verbose and exits with errorcode
+source $HOME/.rvm/scripts/rvm
+set -e  # rvm commands are very verbose
+rvm install ruby-2.4
 rvm osx-ssl-certs status all
 rvm osx-ssl-certs update all
 set -ex
 
+gem install bundler
+
 # cocoapods
-gem install cocoapods --version 1.0.0
+export LANG=en_US.UTF-8
+gem install cocoapods
+gem install xcpretty
+pod repo update  # needed by python
 
 # python
-wget -q https://bootstrap.pypa.io/get-pip.py
-sudo python get-pip.py
+brew install coreutils  # we need grealpath
 sudo pip install virtualenv
+sudo pip install -U six tox setuptools
 
-# TODO(jtattermusch): install python3
-
-# mono
-wget -q https://download.mono-project.com/archive/5.0.1/macos-10-universal/MonoFramework-MDK-5.0.1.1.macos10.xamarin.universal.pkg
-sudo installer -pkg MonoFramework-MDK-5.0.1.1.macos10.xamarin.universal.pkg -target /
-ln -s /Library/Frameworks/Mono.framework/Versions/Current/bin/mono /usr/local/bin/mono
- 
-# dotnet SDK
-brew install openssl
-wget -q https://go.microsoft.com/fwlink/?linkid=843444 -O dotnet-dev-osx-x64.1.0.1.pkg
-sudo installer -pkg dotnet-dev-osx-x64.1.0.1.pkg -target /
-ln -s /usr/local/share/dotnet/dotnet /usr/local/bin/dotnet
-dotnet --version  # bootstrap dotnet SDK
-
-# nvm
-wget -qO- https://raw.githubusercontent.com/creationix/nvm/v0.30.2/install.sh | bash
+# python 3.4
+wget -q https://www.python.org/ftp/python/3.4.4/python-3.4.4-macosx10.6.pkg
+sudo installer -pkg python-3.4.4-macosx10.6.pkg -target /
 
-# TODO(jtattermusch): install node if needed
+# set xcode version for Obj-C tests
+sudo xcode-select -switch /Applications/Xcode_8.2.1.app/Contents/Developer
 
 git submodule update --init
diff --git a/tools/internal_ci/linux/grpc_basictests_c_cpp_dbg.cfg b/tools/internal_ci/linux/grpc_basictests_c_cpp_dbg.cfg
new file mode 100644
index 0000000000..ca547a0a4f
--- /dev/null
+++ b/tools/internal_ci/linux/grpc_basictests_c_cpp_dbg.cfg
@@ -0,0 +1,29 @@
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh"
+timeout_mins: 240
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.xml"
+  }
+}
+
+env_vars {
+  key: "RUN_TESTS_FLAGS"
+  value: "-f basictests linux corelang dbg --inner_jobs 16 -j 1 --internal_ci --bq_result_table aggregate_results"
+}
diff --git a/tools/internal_ci/linux/grpc_basictests_c_cpp_opt.cfg b/tools/internal_ci/linux/grpc_basictests_c_cpp_opt.cfg
new file mode 100644
index 0000000000..62f05cef2e
--- /dev/null
+++ b/tools/internal_ci/linux/grpc_basictests_c_cpp_opt.cfg
@@ -0,0 +1,29 @@
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh"
+timeout_mins: 240
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.xml"
+  }
+}
+
+env_vars {
+  key: "RUN_TESTS_FLAGS"
+  value: "-f basictests linux corelang opt --inner_jobs 16 -j 1 --internal_ci --bq_result_table aggregate_results"
+}
diff --git a/tools/internal_ci/linux/grpc_basictests_multilang.cfg b/tools/internal_ci/linux/grpc_basictests_multilang.cfg
new file mode 100644
index 0000000000..be099d78c7
--- /dev/null
+++ b/tools/internal_ci/linux/grpc_basictests_multilang.cfg
@@ -0,0 +1,29 @@
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh"
+timeout_mins: 240
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.xml"
+  }
+}
+
+env_vars {
+  key: "RUN_TESTS_FLAGS"
+  value: "-f basictests linux multilang --inner_jobs 16 -j 2 --internal_ci --bq_result_table aggregate_results"
+}
diff --git a/tools/internal_ci/linux/grpc_build_artifacts.sh b/tools/internal_ci/linux/grpc_build_artifacts.sh
index 1f438a6730..3997a13087 100755
--- a/tools/internal_ci/linux/grpc_build_artifacts.sh
+++ b/tools/internal_ci/linux/grpc_build_artifacts.sh
@@ -20,8 +20,10 @@ cd $(dirname $0)/../../..
 
 source tools/internal_ci/helper_scripts/prepare_build_linux_rc
 
-# TODO(jtattermusch): install ruby on the internal_ci worker
-gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3
-curl -sSL https://get.rvm.io | bash -s stable --ruby
+set +ex
+[[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh
+set -e  # rvm commands are very verbose
+rvm --default use ruby-2.4.1
+set -ex
 
 tools/run_tests/task_runner.py -f artifact linux
diff --git a/tools/internal_ci/linux/grpc_interop_matrix.cfg b/tools/internal_ci/linux/grpc_interop_matrix.cfg
new file mode 100644
index 0000000000..2fa3e8f897
--- /dev/null
+++ b/tools/internal_ci/linux/grpc_interop_matrix.cfg
@@ -0,0 +1,25 @@
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/linux/grpc_interop_matrix.sh"
+# grpc_interop tests can take 1 hours to complete.
+timeout_mins: 60
+action {
+  define_artifacts {
+    regex: "**/sponge_log.xml"
+  }
+}
diff --git a/tools/internal_ci/linux/grpc_interop_matrix.sh b/tools/internal_ci/linux/grpc_interop_matrix.sh
new file mode 100755
index 0000000000..e199d48931
--- /dev/null
+++ b/tools/internal_ci/linux/grpc_interop_matrix.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+export LANG=en_US.UTF-8
+
+# Enter the gRPC repo root
+cd $(dirname $0)/../../..
+
+source tools/internal_ci/helper_scripts/prepare_build_linux_rc
+
+tools/interop_matrix/run_interop_matrix_tests.py --language=all --release=all --report_file=sponge_log.xml $@
diff --git a/tools/internal_ci/linux/grpc_interop_tocloud.cfg b/tools/internal_ci/linux/grpc_interop_tocloud.cfg
index 1f6421c83d..0c31b497af 100644
--- a/tools/internal_ci/linux/grpc_interop_tocloud.cfg
+++ b/tools/internal_ci/linux/grpc_interop_tocloud.cfg
@@ -20,6 +20,6 @@ build_file: "grpc/tools/internal_ci/linux/grpc_interop_tocloud.sh"
 timeout_mins: 480
 action {
   define_artifacts {
-    regex: "**/report.xml"
+    regex: "**/sponge_log.xml"
   }
 }
diff --git a/tools/internal_ci/linux/grpc_interop_tocloud.sh b/tools/internal_ci/linux/grpc_interop_tocloud.sh
index fe5c9a5130..e3ba25af5d 100755
--- a/tools/internal_ci/linux/grpc_interop_tocloud.sh
+++ b/tools/internal_ci/linux/grpc_interop_tocloud.sh
@@ -23,4 +23,4 @@ cd $(dirname $0)/../../..
 source tools/internal_ci/helper_scripts/prepare_build_linux_rc
 source tools/internal_ci/helper_scripts/prepare_build_interop_rc
 
-tools/run_tests/run_interop_tests.py -l all -s all --use_docker --http2_interop -t -j 12 $@
+tools/run_tests/run_interop_tests.py -l all -s all --use_docker --http2_interop --internal_ci -t -j 12 $@
diff --git a/tools/internal_ci/linux/grpc_interop_toprod.cfg b/tools/internal_ci/linux/grpc_interop_toprod.cfg
new file mode 100644
index 0000000000..18978b87e5
--- /dev/null
+++ b/tools/internal_ci/linux/grpc_interop_toprod.cfg
@@ -0,0 +1,26 @@
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/linux/grpc_interop_toprod.sh"
+# grpc_interop tests can take 6+ hours to complete.
+timeout_mins: 60
+action {
+  define_artifacts {
+    regex: "**/sponge_log.xml"
+  }
+}
+
diff --git a/tools/internal_ci/linux/grpc_interop_toprod.sh b/tools/internal_ci/linux/grpc_interop_toprod.sh
new file mode 100755
index 0000000000..3d06185406
--- /dev/null
+++ b/tools/internal_ci/linux/grpc_interop_toprod.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+export LANG=en_US.UTF-8
+
+# Enter the gRPC repo root
+cd $(dirname $0)/../../..
+
+source tools/internal_ci/helper_scripts/prepare_build_linux_rc
+source tools/internal_ci/helper_scripts/prepare_build_interop_rc
+
+tools/run_tests/run_interop_tests.py \
+    -l all \
+    --cloud_to_prod \
+    --cloud_to_prod_auth \
+    --prod_servers default gateway_v4 \
+    --use_docker --internal_ci -t -j 12 $@
+
diff --git a/tools/internal_ci/macos/grpc_build_artifacts.sh b/tools/internal_ci/macos/grpc_build_artifacts.sh
index aad99b068d..603c15f210 100755
--- a/tools/internal_ci/macos/grpc_build_artifacts.sh
+++ b/tools/internal_ci/macos/grpc_build_artifacts.sh
@@ -18,6 +18,31 @@ set -ex
 # change to grpc repo root
 cd $(dirname $0)/../../..
 
-git submodule update --init
+source tools/internal_ci/helper_scripts/prepare_build_macos_rc
+
+# python 3.5
+wget -q https://www.python.org/ftp/python/3.5.2/python-3.5.2-macosx10.6.pkg
+sudo installer -pkg python-3.5.2-macosx10.6.pkg -target /
+
+# install cython for all python versions
+python2.7 -m pip install cython setuptools wheel
+python3.4 -m pip install cython setuptools wheel
+python3.5 -m pip install cython setuptools wheel
+python3.6 -m pip install cython setuptools wheel
+
+# node-gyp (needed for node artifacts)
+npm install -g node-gyp
+
+# php dependencies: pecl
+curl -O http://pear.php.net/go-pear.phar
+sudo php -d detect_unicode=0 go-pear.phar
+
+# needed to build ruby artifacts
+gem install rake-compiler
+wget https://raw.githubusercontent.com/grpc/grpc/master/tools/distrib/build_ruby_environment_macos.sh
+bash build_ruby_environment_macos.sh
+
+gem install rubygems-update
+update_rubygems
 
 tools/run_tests/task_runner.py -f artifact macos
diff --git a/tools/internal_ci/macos/grpc_master.sh b/tools/internal_ci/macos/grpc_master.sh
index 786859be3f..c64666b2de 100755
--- a/tools/internal_ci/macos/grpc_master.sh
+++ b/tools/internal_ci/macos/grpc_master.sh
@@ -20,7 +20,7 @@ cd $(dirname $0)/../../..
 
 source tools/internal_ci/helper_scripts/prepare_build_macos_rc
 
-tools/run_tests/run_tests_matrix.py -f basictests macos --internal_ci || FAILED="true"
+tools/run_tests/run_tests_matrix.py -f basictests macos --internal_ci -j 2 --inner_jobs 4 || FAILED="true"
 
 # kill port_server.py to prevent the build from hanging
 ps aux | grep port_server\\.py | awk '{print $2}' | xargs kill -9
diff --git a/tools/internal_ci/windows/grpc_build_artifacts.bat b/tools/internal_ci/windows/grpc_build_artifacts.bat
index 6dca2f72c1..8cfa2fb0be 100644
--- a/tools/internal_ci/windows/grpc_build_artifacts.bat
+++ b/tools/internal_ci/windows/grpc_build_artifacts.bat
@@ -12,10 +12,20 @@
 @rem See the License for the specific language governing permissions and
 @rem limitations under the License.
 
+@rem Move python installation from _32bit to _32bits where they are expected python artifact builder
+@rem TODO(jtattermusch): get rid of this hack
+rename C:\Python27_32bit Python27_32bits
+rename C:\Python34_32bit Python34_32bits
+rename C:\Python35_32bit Python35_32bits
+rename C:\Python36_32bit Python36_32bits
+
+pacman -S --noconfirm mingw64/mingw-w64-x86_64-gcc mingw32/mingw-w64-i686-gcc
+
 @rem make sure msys binaries are preferred over cygwin binaries
 @rem set path to python 2.7
 set PATH=C:\tools\msys64\usr\bin;C:\Python27;%PATH%
 
+
 @rem enter repo root
 cd /d %~dp0\..\..\..