diff options
18 files changed, 346 insertions, 8 deletions
diff --git a/src/csharp/build_packages_dotnetcli.bat b/src/csharp/build_packages_dotnetcli.bat index 4dd4947f00..76391738ff 100755 --- a/src/csharp/build_packages_dotnetcli.bat +++ b/src/csharp/build_packages_dotnetcli.bat @@ -19,17 +19,21 @@ set VERSION=1.11.0-dev set NUGET=C:\nuget\nuget.exe set DOTNET=dotnet -set -ex - mkdir ..\..\artifacts @rem Collect the artifacts built by the previous build step if running on Jenkins mkdir nativelibs +@rem Jenkins flow (deprecated) powershell -Command "cp -r ..\..\platform=*\artifacts\csharp_ext_* nativelibs" +@rem Kokoro flow +powershell -Command "cp -r ..\..\input_artifacts\csharp_ext_* nativelibs" @rem Collect protoc artifacts built by the previous build step mkdir protoc_plugins +@rem Jenkins flow (deprecated) powershell -Command "cp -r ..\..\platform=*\artifacts\protoc_* protoc_plugins" +@rem Kokoro flow +powershell -Command "cp -r ..\..\input_artifacts\protoc_* protoc_plugins" %DOTNET% restore Grpc.sln || goto :error diff --git a/src/csharp/build_packages_dotnetcli.sh b/src/csharp/build_packages_dotnetcli.sh index e3f8463ee8..1b73614b91 100755 --- a/src/csharp/build_packages_dotnetcli.sh +++ b/src/csharp/build_packages_dotnetcli.sh @@ -21,11 +21,17 @@ mkdir -p ../../artifacts/ # Collect the artifacts built by the previous build step mkdir -p nativelibs +# Jenkins flow (deprecated) cp -r $EXTERNAL_GIT_ROOT/platform={windows,linux,macos}/artifacts/csharp_ext_* nativelibs || true +# Kokoro flow +cp -r $EXTERNAL_GIT_ROOT/input_artifacts/csharp_ext_* nativelibs || true # Collect protoc artifacts built by the previous build step mkdir -p protoc_plugins +# Jenkins flow (deprecated) cp -r $EXTERNAL_GIT_ROOT/platform={windows,linux,macos}/artifacts/protoc_* protoc_plugins || true +# Kokoro flow +cp -r $EXTERNAL_GIT_ROOT/input_artifacts/protoc_* protoc_plugins || true dotnet restore Grpc.sln diff --git a/templates/src/csharp/build_packages_dotnetcli.bat.template b/templates/src/csharp/build_packages_dotnetcli.bat.template index 6671991ad2..1bf78c4d23 100755 --- a/templates/src/csharp/build_packages_dotnetcli.bat.template +++ b/templates/src/csharp/build_packages_dotnetcli.bat.template @@ -21,17 +21,21 @@ set NUGET=C:\nuget\nuget.exe set DOTNET=dotnet - set -ex - mkdir ..\..\artifacts @rem Collect the artifacts built by the previous build step if running on Jenkins mkdir nativelibs + @rem Jenkins flow (deprecated) powershell -Command "cp -r ..\..\platform=*\artifacts\csharp_ext_* nativelibs" + @rem Kokoro flow + powershell -Command "cp -r ..\..\input_artifacts\csharp_ext_* nativelibs" @rem Collect protoc artifacts built by the previous build step mkdir protoc_plugins + @rem Jenkins flow (deprecated) powershell -Command "cp -r ..\..\platform=*\artifacts\protoc_* protoc_plugins" + @rem Kokoro flow + powershell -Command "cp -r ..\..\input_artifacts\protoc_* protoc_plugins" %%DOTNET% restore Grpc.sln || goto :error diff --git a/templates/src/csharp/build_packages_dotnetcli.sh.template b/templates/src/csharp/build_packages_dotnetcli.sh.template index be52b4631c..ddfea74744 100755 --- a/templates/src/csharp/build_packages_dotnetcli.sh.template +++ b/templates/src/csharp/build_packages_dotnetcli.sh.template @@ -23,11 +23,17 @@ # Collect the artifacts built by the previous build step mkdir -p nativelibs + # Jenkins flow (deprecated) cp -r $EXTERNAL_GIT_ROOT/platform={windows,linux,macos}/artifacts/csharp_ext_* nativelibs || true + # Kokoro flow + cp -r $EXTERNAL_GIT_ROOT/input_artifacts/csharp_ext_* nativelibs || true # Collect protoc artifacts built by the previous build step mkdir -p protoc_plugins + # Jenkins flow (deprecated) cp -r $EXTERNAL_GIT_ROOT/platform={windows,linux,macos}/artifacts/protoc_* protoc_plugins || true + # Kokoro flow + cp -r $EXTERNAL_GIT_ROOT/input_artifacts/protoc_* protoc_plugins || true dotnet restore Grpc.sln diff --git a/tools/internal_ci/linux/grpc_build_packages.cfg b/tools/internal_ci/linux/grpc_build_packages.cfg new file mode 100644 index 0000000000..6a4a163dfc --- /dev/null +++ b/tools/internal_ci/linux/grpc_build_packages.cfg @@ -0,0 +1,26 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_build_packages.sh" +timeout_mins: 120 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} diff --git a/tools/internal_ci/linux/grpc_build_packages.sh b/tools/internal_ci/linux/grpc_build_packages.sh new file mode 100644 index 0000000000..9f65d8ed5f --- /dev/null +++ b/tools/internal_ci/linux/grpc_build_packages.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# change to grpc repo root +cd $(dirname $0)/../../.. + +source tools/internal_ci/helper_scripts/prepare_build_linux_rc + +set +ex +[[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh +set -e # rvm commands are very verbose +rvm --default use ruby-2.4.1 +set -ex + +# Move artifacts generated by the previous step in the build chain to a place +# where they can be accessed from within a docker container that builds +# the packages +mv ${KOKORO_GFILE_DIR}/github/grpc/artifacts input_artifacts || true +ls -R input_artifacts || true + +tools/run_tests/task_runner.py -f package linux -j 6 diff --git a/tools/internal_ci/linux/grpc_distribtests.cfg b/tools/internal_ci/linux/grpc_distribtests.cfg new file mode 100644 index 0000000000..0f1d79355a --- /dev/null +++ b/tools/internal_ci/linux/grpc_distribtests.cfg @@ -0,0 +1,26 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_distribtests.sh" +timeout_mins: 120 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} diff --git a/tools/internal_ci/linux/grpc_distribtests.sh b/tools/internal_ci/linux/grpc_distribtests.sh new file mode 100644 index 0000000000..47e4bf877a --- /dev/null +++ b/tools/internal_ci/linux/grpc_distribtests.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# change to grpc repo root +cd $(dirname $0)/../../.. + +source tools/internal_ci/helper_scripts/prepare_build_linux_rc + +set +ex +[[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh +set -e # rvm commands are very verbose +rvm --default use ruby-2.4.1 +set -ex + +# Move packages generated by the previous step in the build chain to a place +# where they can be accessed from within a docker container that run the +# distribtests +mv ${KOKORO_GFILE_DIR}/github/grpc/artifacts input_artifacts || true +ls -R input_artifacts || true + +tools/run_tests/task_runner.py -f distribtest linux -j 6 diff --git a/tools/internal_ci/macos/grpc_distribtests.cfg b/tools/internal_ci/macos/grpc_distribtests.cfg new file mode 100644 index 0000000000..ae88f39b90 --- /dev/null +++ b/tools/internal_ci/macos/grpc_distribtests.cfg @@ -0,0 +1,27 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/macos/grpc_distribtests.sh" +gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json" +timeout_mins: 120 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} diff --git a/tools/internal_ci/macos/grpc_distribtests.sh b/tools/internal_ci/macos/grpc_distribtests.sh new file mode 100644 index 0000000000..59ea833eec --- /dev/null +++ b/tools/internal_ci/macos/grpc_distribtests.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# change to grpc repo root +cd $(dirname $0)/../../.. + +source tools/internal_ci/helper_scripts/prepare_build_macos_rc + +# Move packages generated by the previous step in the build chain. +mv ${KOKORO_GFILE_DIR}/github/grpc/artifacts input_artifacts || true +ls -R input_artifacts || true + +tools/run_tests/task_runner.py -f distribtest macos diff --git a/tools/internal_ci/windows/grpc_build_packages.bat b/tools/internal_ci/windows/grpc_build_packages.bat new file mode 100644 index 0000000000..23ba365e40 --- /dev/null +++ b/tools/internal_ci/windows/grpc_build_packages.bat @@ -0,0 +1,35 @@ +@rem Copyright 2017 gRPC authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. + +@rem Move python installation from _32bit to _32bits where they are expected by python artifact builder +@rem TODO(jtattermusch): get rid of this hack +rename C:\Python27_32bit Python27_32bits +rename C:\Python34_32bit Python34_32bits +rename C:\Python35_32bit Python35_32bits +rename C:\Python36_32bit Python36_32bits + +@rem enter repo root +cd /d %~dp0\..\..\.. + +call tools/internal_ci/helper_scripts/prepare_build_windows.bat + +@rem Move artifacts generated by the previous step in the build chain. +powershell -Command "mv %KOKORO_GFILE_DIR%\github\grpc\artifacts input_artifacts" +dir input_artifacts + +python tools/run_tests/task_runner.py -f package windows -j 4 || goto :error +goto :EOF + +:error +exit /b %errorlevel% diff --git a/tools/internal_ci/windows/grpc_build_packages.cfg b/tools/internal_ci/windows/grpc_build_packages.cfg new file mode 100644 index 0000000000..65a8b1eef3 --- /dev/null +++ b/tools/internal_ci/windows/grpc_build_packages.cfg @@ -0,0 +1,26 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/windows/grpc_build_packages.bat" +timeout_mins: 120 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} diff --git a/tools/internal_ci/windows/grpc_distribtests.bat b/tools/internal_ci/windows/grpc_distribtests.bat new file mode 100644 index 0000000000..d6d50b684f --- /dev/null +++ b/tools/internal_ci/windows/grpc_distribtests.bat @@ -0,0 +1,35 @@ +@rem Copyright 2017 gRPC authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. + +@rem Move python installation from _32bit to _32bits where they are expected by python artifact builder +@rem TODO(jtattermusch): get rid of this hack +rename C:\Python27_32bit Python27_32bits +rename C:\Python34_32bit Python34_32bits +rename C:\Python35_32bit Python35_32bits +rename C:\Python36_32bit Python36_32bits + +@rem enter repo root +cd /d %~dp0\..\..\.. + +call tools/internal_ci/helper_scripts/prepare_build_windows.bat + +@rem Move packages generated by the previous step in the build chain. +powershell -Command "mv %KOKORO_GFILE_DIR%\github\grpc\artifacts input_artifacts" +dir input_artifacts + +python tools/run_tests/task_runner.py -f distribtest windows -j 4 || goto :error +goto :EOF + +:error +exit /b %errorlevel% diff --git a/tools/internal_ci/windows/grpc_distribtests.cfg b/tools/internal_ci/windows/grpc_distribtests.cfg new file mode 100644 index 0000000000..1766e601e5 --- /dev/null +++ b/tools/internal_ci/windows/grpc_distribtests.cfg @@ -0,0 +1,26 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/windows/grpc_distribtests.bat" +timeout_mins: 120 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} diff --git a/tools/run_tests/artifacts/build_package_php.sh b/tools/run_tests/artifacts/build_package_php.sh index 85e4dda40a..9a8f25a6f8 100755 --- a/tools/run_tests/artifacts/build_package_php.sh +++ b/tools/run_tests/artifacts/build_package_php.sh @@ -17,5 +17,10 @@ set -ex cd "$(dirname "$0")/../../.." +# All the PHP packages have been built in the artifact phase already +# and we only collect them here to deliver them to the distribtest phase. mkdir -p artifacts/ -cp -r "$EXTERNAL_GIT_ROOT"/platform={windows,linux,macos}/artifacts/php_*/* artifacts/ || true +# Jenkins flow (deprecated) +cp -r "${EXTERNAL_GIT_ROOT}"/platform={windows,linux,macos}/artifacts/php_*/* artifacts/ || true +# Kokoro flow +cp -r "${EXTERNAL_GIT_ROOT}"/input_artifacts/php_*/* artifacts/ || true diff --git a/tools/run_tests/artifacts/build_package_python.sh b/tools/run_tests/artifacts/build_package_python.sh index d596e35000..15627881e8 100755 --- a/tools/run_tests/artifacts/build_package_python.sh +++ b/tools/run_tests/artifacts/build_package_python.sh @@ -21,7 +21,10 @@ mkdir -p artifacts/ # All the python packages have been built in the artifact phase already # and we only collect them here to deliver them to the distribtest phase. -cp -r "$EXTERNAL_GIT_ROOT"/platform={windows,linux,macos}/artifacts/python_*/* artifacts/ || true +# Jenkins flow (deprecated) +cp -r "${EXTERNAL_GIT_ROOT}"/platform={windows,linux,macos}/artifacts/python_*/* artifacts/ || true +# Kokoro flow +cp -r "${EXTERNAL_GIT_ROOT}"/input_artifacts/python_*/* artifacts/ || true # TODO: all the artifact builder configurations generate a grpcio-VERSION.tar.gz # source distribution package, and only one of them will end up diff --git a/tools/run_tests/artifacts/build_package_ruby.sh b/tools/run_tests/artifacts/build_package_ruby.sh index 0283c43843..05eacd4d1e 100755 --- a/tools/run_tests/artifacts/build_package_ruby.sh +++ b/tools/run_tests/artifacts/build_package_ruby.sh @@ -23,7 +23,10 @@ mkdir -p artifacts/ # All the ruby packages have been built in the artifact phase already # and we only collect them here to deliver them to the distribtest phase. -cp -r "$EXTERNAL_GIT_ROOT"/platform={windows,linux,macos}/artifacts/ruby_native_gem_*/* artifacts/ || true +# Jenkins flow (deprecated) +cp -r "${EXTERNAL_GIT_ROOT}"/platform={windows,linux,macos}/artifacts/ruby_native_gem_*/* artifacts/ || true +# Kokoro flow +cp -r "${EXTERNAL_GIT_ROOT}"/input_artifacts/ruby_native_gem_*/* artifacts/ || true well_known_protos=( any api compiler/plugin descriptor duration empty field_mask source_context struct timestamp type wrappers ) @@ -41,7 +44,12 @@ for arch in {x86,x64}; do ;; esac for plat in {windows,linux,macos}; do - input_dir="$EXTERNAL_GIT_ROOT/platform=${plat}/artifacts/protoc_${plat}_${arch}" + if [ "${KOKORO_JOB_NAME}" != "" ] + then + input_dir="${EXTERNAL_GIT_ROOT}/input_artifacts/protoc_${plat}_${arch}" + else + input_dir="${EXTERNAL_GIT_ROOT}/platform=${plat}/artifacts/protoc_${plat}_${arch}" + fi output_dir="$base/src/ruby/tools/bin/${ruby_arch}-${plat}" mkdir -p "$output_dir"/google/protobuf mkdir -p "$output_dir"/google/protobuf/compiler # needed for plugin.proto diff --git a/tools/run_tests/dockerize/build_and_run_docker.sh b/tools/run_tests/dockerize/build_and_run_docker.sh index b8f0a55025..3ea3ca3c0b 100755 --- a/tools/run_tests/dockerize/build_and_run_docker.sh +++ b/tools/run_tests/dockerize/build_and_run_docker.sh @@ -58,6 +58,10 @@ docker run \ "$@" \ -e EXTERNAL_GIT_ROOT="/var/local/jenkins/grpc" \ -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \ + -e "KOKORO_BUILD_ID=$KOKORO_BUILD_ID" \ + -e "KOKORO_BUILD_NUMBER=$KOKORO_BUILD_NUMBER" \ + -e "KOKORO_BUILD_URL=$KOKORO_BUILD_URL" \ + -e "KOKORO_JOB_NAME=$KOKORO_JOB_NAME" \ -v "$git_root:/var/local/jenkins/grpc:ro" \ -w /var/local/git/grpc \ --name="$CONTAINER_NAME" \ |