From f9b6c88f010373b52518b3c90d77fe339b5a3994 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 5 Oct 2018 16:53:20 +0200 Subject: align configuration of kokoro and manual foundry runs --- tools/remote_build/README.md | 26 +++++++++++++++++++++ tools/remote_build/kokoro.bazelrc | 23 ++++++++++++++++++ tools/remote_build/manual.bazelrc | 42 +++++++++++++++++++++++++++++++++ tools/remote_build/rbe_common.bazelrc | 44 +++++++++++++++++++++++++++++++++++ 4 files changed, 135 insertions(+) create mode 100644 tools/remote_build/README.md create mode 100644 tools/remote_build/kokoro.bazelrc create mode 100644 tools/remote_build/manual.bazelrc create mode 100644 tools/remote_build/rbe_common.bazelrc (limited to 'tools/remote_build') diff --git a/tools/remote_build/README.md b/tools/remote_build/README.md new file mode 100644 index 0000000000..019033e52e --- /dev/null +++ b/tools/remote_build/README.md @@ -0,0 +1,26 @@ +# Running Remote Builds with bazel + +This allows you to spawn gRPC C/C++ remote build and tests from your workstation with +configuration that's very similar to what's used by our CI Kokoro. + +Note that this will only work for gRPC team members (it requires access to the +remote build and execution cluster), others will need to rely on local test runs +and tests run by Kokoro CI. + + +## Prerequisites + +- See [Installing Bazel](https://docs.bazel.build/versions/master/install.html) for instructions how to install bazel on your system. + +- Setup application default credentials for running remote builds by following [RBE Credentials Setup](https://cloud.google.com/remote-build-execution/docs/getting-started#set_credentials) + + +## Running remote build manually from dev workstation + +Run from repository root: +``` +# manual run of bazel tests remotely on Foundry +bazel --bazelrc=tools/remote_build/manual.bazelrc test -c opt //test/... +``` + +TODO: add instructions for running sanitizer builds diff --git a/tools/remote_build/kokoro.bazelrc b/tools/remote_build/kokoro.bazelrc new file mode 100644 index 0000000000..0aacddf663 --- /dev/null +++ b/tools/remote_build/kokoro.bazelrc @@ -0,0 +1,23 @@ +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# bazelrc file for running gRPC tests on Kokoro using Foundry + +import %workspace%/tools/remote_build/rbe_common.bazelrc + +build --jobs=200 +build --test_output=errors +build --keep_going +build --remote_accept_cached=true +build --remote_local_fallback=true diff --git a/tools/remote_build/manual.bazelrc b/tools/remote_build/manual.bazelrc new file mode 100644 index 0000000000..040e6bd620 --- /dev/null +++ b/tools/remote_build/manual.bazelrc @@ -0,0 +1,42 @@ +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# bazelrc file for running gRPC tests with Foundry (remote build execution) +# manually from developer's workstation + +import %workspace%/tools/remote_build/rbe_common.bazelrc + +build --remote_cache=remotebuildexecution.googleapis.com +build --remote_executor=remotebuildexecution.googleapis.com +build --tls_enabled=true + +# Enable authentication. This will pick up application default credentials by +# default. You can use --auth_credentials=some_file.json to use a service +# account credential instead. +# How to setup credentials: +# See https://cloud.google.com/remote-build-execution/docs/getting-started#set_credentials +build --auth_enabled=true + +# Set flags for uploading to BES in order to view results in the Bazel Build +# Results UI. +#build --bes_backend="buildeventservice.googleapis.com" +#build --bes_timeout=60s +#build --bes_results_url="https://source.cloud.google.com/results/invocations/" +#build --project_id=grpc-testing + +build --jobs=100 + +# TODO(jtattermusch): this should be part of the common config +# but currently sanitizers use different test_timeout values +build --test_timeout=300,450,1200,3600 diff --git a/tools/remote_build/rbe_common.bazelrc b/tools/remote_build/rbe_common.bazelrc new file mode 100644 index 0000000000..ae382aeabe --- /dev/null +++ b/tools/remote_build/rbe_common.bazelrc @@ -0,0 +1,44 @@ +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# bazelrc with Foundry setting common to both manual run and runs started by Kokoro +# see https://github.com/bazelbuild/bazel-toolchains/tree/master/bazelrc +# for examples and more documentation + +startup --host_jvm_args=-Dbazel.DigestFunction=SHA256 + +build --crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.0/bazel_0.16.1/default:toolchain +build --extra_toolchains=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.0/bazel_0.16.1/cpp:cc-toolchain-clang-x86_64-default +# Use custom execution platforms defined in third_party/toolchains +build --extra_execution_platforms=//third_party/toolchains:rbe_ubuntu1604 +build --host_platform=//third_party/toolchains:rbe_ubuntu1604 +build --platforms=//third_party/toolchains:rbe_ubuntu1604 + +build --spawn_strategy=remote +build --strategy=Javac=remote +build --strategy=Closure=remote +build --genrule_strategy=remote +build --remote_timeout=3600 + +build --remote_instance_name=projects/grpc-testing/instances/default_instance + +build --verbose_failures=true + +build --experimental_strict_action_env=true +build --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 + +# don't use port server +build --define GRPC_PORT_ISOLATED_RUNTIME=1 +# without verbose gRPC logs the test outputs are not very useful +test --test_env=GRPC_VERBOSITY=debug -- cgit v1.2.3