Merge branch 'master' of https://github.com/grpc/grpc into channel-tracing

15 files changed, 303 insertions, 3 deletions

diff --git a/tools/internal_ci/helper_scripts/prepare_build_interop_rc b/tools/internal_ci/helper_scripts/prepare_build_interop_rc
index db978c8b3f..fb0f4b8054 100644
--- a/tools/internal_ci/helper_scripts/prepare_build_interop_rc
+++ b/tools/internal_ci/helper_scripts/prepare_build_interop_rc
@@ -27,6 +27,7 @@ git submodule update --init
 git clone --recursive https://github.com/grpc/grpc-go ./../grpc-go
 git clone --recursive https://github.com/grpc/grpc-java ./../grpc-java
 git clone --recursive https://github.com/grpc/grpc-node ./../grpc-node
+git clone --recursive https://github.com/grpc/grpc-dart ./../grpc-dart
 
 # Download json file.
 mkdir ~/service_account
diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc
index 6ecf51dd3f..786cd4559e 100644
--- a/tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc
+++ b/tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc
@@ -32,6 +32,7 @@ brew install md5sha1sum
 git clone --recursive https://github.com/grpc/grpc-go ./../grpc-go
 git clone --recursive https://github.com/grpc/grpc-java ./../grpc-java
 git clone --recursive https://github.com/grpc/grpc-node ./../grpc-node
+git clone --recursive https://github.com/grpc/grpc-dart ./../grpc-dart
 
 # Set up Docker for Mac
 docker-machine create -d virtualbox --virtualbox-share-folder "/Users/kbuilder/workspace:" default
diff --git a/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh b/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh
index 098d588e3b..a202aafb50 100755
--- a/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh
+++ b/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh
@@ -37,7 +37,7 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_rc
 # TODO(adelez): implement size for test targets and change test_timeout back
 "${KOKORO_GFILE_DIR}/bazel_wrapper.py" \
   --host_jvm_args=-Dbazel.DigestFunction=SHA256 \
-  test --jobs="50" \
+  test --jobs="100" \
   --test_timeout="3600,3600,3600,3600" \
   --test_output=errors  \
   --verbose_failures=true  \
diff --git a/tools/internal_ci/linux/grpc_build_packages.cfg b/tools/internal_ci/linux/grpc_build_packages.cfg
new file mode 100644
index 0000000000..6a4a163dfc
--- /dev/null
+++ b/tools/internal_ci/linux/grpc_build_packages.cfg
@@ -0,0 +1,26 @@
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/linux/grpc_build_packages.sh"
+timeout_mins: 120
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.xml"
+    regex: "github/grpc/reports/**"
+    regex: "github/grpc/artifacts/**"
+  }
+}
diff --git a/tools/internal_ci/linux/grpc_build_packages.sh b/tools/internal_ci/linux/grpc_build_packages.sh
new file mode 100644
index 0000000000..9f65d8ed5f
--- /dev/null
+++ b/tools/internal_ci/linux/grpc_build_packages.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# change to grpc repo root
+cd $(dirname $0)/../../..
+
+source tools/internal_ci/helper_scripts/prepare_build_linux_rc
+
+set +ex
+[[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh
+set -e  # rvm commands are very verbose
+rvm --default use ruby-2.4.1
+set -ex
+
+# Move artifacts generated by the previous step in the build chain to a place
+# where they can be accessed from within a docker container that builds
+# the packages
+mv ${KOKORO_GFILE_DIR}/github/grpc/artifacts input_artifacts || true
+ls -R input_artifacts || true
+
+tools/run_tests/task_runner.py -f package linux -j 6
diff --git a/tools/internal_ci/linux/grpc_distribtests.cfg b/tools/internal_ci/linux/grpc_distribtests.cfg
new file mode 100644
index 0000000000..0f1d79355a
--- /dev/null
+++ b/tools/internal_ci/linux/grpc_distribtests.cfg
@@ -0,0 +1,26 @@
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/linux/grpc_distribtests.sh"
+timeout_mins: 120
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.xml"
+    regex: "github/grpc/reports/**"
+    regex: "github/grpc/artifacts/**"
+  }
+}
diff --git a/tools/internal_ci/linux/grpc_distribtests.sh b/tools/internal_ci/linux/grpc_distribtests.sh
new file mode 100644
index 0000000000..47e4bf877a
--- /dev/null
+++ b/tools/internal_ci/linux/grpc_distribtests.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# change to grpc repo root
+cd $(dirname $0)/../../..
+
+source tools/internal_ci/helper_scripts/prepare_build_linux_rc
+
+set +ex
+[[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh
+set -e  # rvm commands are very verbose
+rvm --default use ruby-2.4.1
+set -ex
+
+# Move packages generated by the previous step in the build chain to a place
+# where they can be accessed from within a docker container that run the
+# distribtests
+mv ${KOKORO_GFILE_DIR}/github/grpc/artifacts input_artifacts || true
+ls -R input_artifacts || true
+
+tools/run_tests/task_runner.py -f distribtest linux -j 6
diff --git a/tools/internal_ci/linux/grpc_msan_on_foundry.sh b/tools/internal_ci/linux/grpc_msan_on_foundry.sh
index 16586a2d0d..60b30ff4f6 100644
--- a/tools/internal_ci/linux/grpc_msan_on_foundry.sh
+++ b/tools/internal_ci/linux/grpc_msan_on_foundry.sh
@@ -37,7 +37,7 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_rc
 
 "${KOKORO_GFILE_DIR}/bazel_wrapper.py" \
   --host_jvm_args=-Dbazel.DigestFunction=SHA256 \
-  test --jobs="50" \
+  test --jobs="100" \
   --test_timeout="3600,3600,3600,3600" \
   --test_output=errors  \
   --verbose_failures=true  \
diff --git a/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh b/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh
index cd1a340431..a87547d1b3 100644
--- a/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh
+++ b/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh
@@ -37,7 +37,7 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_rc
 
 "${KOKORO_GFILE_DIR}/bazel_wrapper.py" \
   --host_jvm_args=-Dbazel.DigestFunction=SHA256 \
-  test --jobs="50" \
+  test --jobs="100" \
   --test_timeout="3600,3600,3600,3600" \
   --test_output=errors  \
   --verbose_failures=true  \
diff --git a/tools/internal_ci/macos/grpc_distribtests.cfg b/tools/internal_ci/macos/grpc_distribtests.cfg
new file mode 100644
index 0000000000..ae88f39b90
--- /dev/null
+++ b/tools/internal_ci/macos/grpc_distribtests.cfg
@@ -0,0 +1,27 @@
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/macos/grpc_distribtests.sh"
+gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json"
+timeout_mins: 120
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.xml"
+    regex: "github/grpc/reports/**"
+    regex: "github/grpc/artifacts/**"
+  }
+}
diff --git a/tools/internal_ci/macos/grpc_distribtests.sh b/tools/internal_ci/macos/grpc_distribtests.sh
new file mode 100644
index 0000000000..59ea833eec
--- /dev/null
+++ b/tools/internal_ci/macos/grpc_distribtests.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# change to grpc repo root
+cd $(dirname $0)/../../..
+
+source tools/internal_ci/helper_scripts/prepare_build_macos_rc
+
+# Move packages generated by the previous step in the build chain.
+mv ${KOKORO_GFILE_DIR}/github/grpc/artifacts input_artifacts || true
+ls -R input_artifacts || true
+
+tools/run_tests/task_runner.py -f distribtest macos
diff --git a/tools/internal_ci/windows/grpc_build_packages.bat b/tools/internal_ci/windows/grpc_build_packages.bat
new file mode 100644
index 0000000000..23ba365e40
--- /dev/null
+++ b/tools/internal_ci/windows/grpc_build_packages.bat
@@ -0,0 +1,35 @@
+@rem Copyright 2017 gRPC authors.
+@rem
+@rem Licensed under the Apache License, Version 2.0 (the "License");
+@rem you may not use this file except in compliance with the License.
+@rem You may obtain a copy of the License at
+@rem
+@rem     http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+
+@rem Move python installation from _32bit to _32bits where they are expected by python artifact builder
+@rem TODO(jtattermusch): get rid of this hack
+rename C:\Python27_32bit Python27_32bits
+rename C:\Python34_32bit Python34_32bits
+rename C:\Python35_32bit Python35_32bits
+rename C:\Python36_32bit Python36_32bits
+
+@rem enter repo root
+cd /d %~dp0\..\..\..
+
+call tools/internal_ci/helper_scripts/prepare_build_windows.bat
+
+@rem Move artifacts generated by the previous step in the build chain.
+powershell -Command "mv %KOKORO_GFILE_DIR%\github\grpc\artifacts input_artifacts"
+dir input_artifacts
+
+python tools/run_tests/task_runner.py -f package windows -j 4 || goto :error
+goto :EOF
+
+:error
+exit /b %errorlevel%
diff --git a/tools/internal_ci/windows/grpc_build_packages.cfg b/tools/internal_ci/windows/grpc_build_packages.cfg
new file mode 100644
index 0000000000..65a8b1eef3
--- /dev/null
+++ b/tools/internal_ci/windows/grpc_build_packages.cfg
@@ -0,0 +1,26 @@
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/windows/grpc_build_packages.bat"
+timeout_mins: 120
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.xml"
+    regex: "github/grpc/reports/**"
+    regex: "github/grpc/artifacts/**"
+  }
+}
diff --git a/tools/internal_ci/windows/grpc_distribtests.bat b/tools/internal_ci/windows/grpc_distribtests.bat
new file mode 100644
index 0000000000..d6d50b684f
--- /dev/null
+++ b/tools/internal_ci/windows/grpc_distribtests.bat
@@ -0,0 +1,35 @@
+@rem Copyright 2017 gRPC authors.
+@rem
+@rem Licensed under the Apache License, Version 2.0 (the "License");
+@rem you may not use this file except in compliance with the License.
+@rem You may obtain a copy of the License at
+@rem
+@rem     http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+
+@rem Move python installation from _32bit to _32bits where they are expected by python artifact builder
+@rem TODO(jtattermusch): get rid of this hack
+rename C:\Python27_32bit Python27_32bits
+rename C:\Python34_32bit Python34_32bits
+rename C:\Python35_32bit Python35_32bits
+rename C:\Python36_32bit Python36_32bits
+
+@rem enter repo root
+cd /d %~dp0\..\..\..
+
+call tools/internal_ci/helper_scripts/prepare_build_windows.bat
+
+@rem Move packages generated by the previous step in the build chain.
+powershell -Command "mv %KOKORO_GFILE_DIR%\github\grpc\artifacts input_artifacts"
+dir input_artifacts
+
+python tools/run_tests/task_runner.py -f distribtest windows -j 4 || goto :error
+goto :EOF
+
+:error
+exit /b %errorlevel%
diff --git a/tools/internal_ci/windows/grpc_distribtests.cfg b/tools/internal_ci/windows/grpc_distribtests.cfg
new file mode 100644
index 0000000000..1766e601e5
--- /dev/null
+++ b/tools/internal_ci/windows/grpc_distribtests.cfg
@@ -0,0 +1,26 @@
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/windows/grpc_distribtests.bat"
+timeout_mins: 120
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.xml"
+    regex: "github/grpc/reports/**"
+    regex: "github/grpc/artifacts/**"
+  }
+}