Merge branch master into package-grpc-tools-doc

4 files changed, 199 insertions, 0 deletions

diff --git a/tools/remote_build/README.md b/tools/remote_build/README.md
new file mode 100644
index 0000000000..c4d03547a2
--- /dev/null
+++ b/tools/remote_build/README.md
@@ -0,0 +1,33 @@
+# Running Remote Builds with bazel
+
+This allows you to spawn gRPC C/C++ remote build and tests from your workstation with
+configuration that's very similar to what's used by our CI Kokoro.
+
+Note that this will only work for gRPC team members (it requires access to the
+remote build and execution cluster), others will need to rely on local test runs
+and tests run by Kokoro CI.
+
+
+## Prerequisites
+
+- See [Installing Bazel](https://docs.bazel.build/versions/master/install.html) for instructions how to install bazel on your system.
+
+- Setup application default credentials for running remote builds by following [RBE Credentials Setup](https://cloud.google.com/remote-build-execution/docs/getting-started#set_credentials)
+
+
+## Running remote build manually from dev workstation
+
+Run from repository root:
+```
+# manual run of bazel tests remotely on Foundry
+bazel --bazelrc=tools/remote_build/manual.bazelrc test -c opt //test/...
+```
+
+Sanitizer runs (asan, msan, tsan, ubsan):
+```
+# manual run of bazel tests remotely on Foundry with given sanitizer
+bazel --bazelrc=tools/remote_build/manual.bazelrc test --config=asan //test/...
+```
+
+Available command line options can be found in
+[Bazel command line reference](https://docs.bazel.build/versions/master/command-line-reference.html)
diff --git a/tools/remote_build/kokoro.bazelrc b/tools/remote_build/kokoro.bazelrc
new file mode 100644
index 0000000000..11462bd301
--- /dev/null
+++ b/tools/remote_build/kokoro.bazelrc
@@ -0,0 +1,38 @@
+# Copyright 2018 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# bazelrc file for running gRPC tests on Kokoro using Foundry
+
+import %workspace%/tools/remote_build/rbe_common.bazelrc
+
+build --remote_cache=remotebuildexecution.googleapis.com
+build --remote_executor=remotebuildexecution.googleapis.com
+build --tls_enabled=true
+
+build --auth_enabled=true
+# magic location where kokoro script puts the credentials
+build --auth_credentials=/tmpfs/src/keystore/4321_grpc-testing-service
+build --auth_scope=https://www.googleapis.com/auth/cloud-source-tools
+
+build --bes_backend=buildeventservice.googleapis.com
+build --bes_best_effort=false
+build --bes_timeout=600s
+build --project_id=grpc-testing
+
+# required by kokoro for some reason
+build --test_env=USER=anon
+
+build --jobs=200
+build --test_output=errors
+build --keep_going=true
diff --git a/tools/remote_build/manual.bazelrc b/tools/remote_build/manual.bazelrc
new file mode 100644
index 0000000000..b4fdc70637
--- /dev/null
+++ b/tools/remote_build/manual.bazelrc
@@ -0,0 +1,45 @@
+# Copyright 2018 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# bazelrc file for running gRPC tests with Foundry (remote build execution)
+# manually from developer's workstation
+
+import %workspace%/tools/remote_build/rbe_common.bazelrc
+
+build --remote_cache=remotebuildexecution.googleapis.com
+build --remote_executor=remotebuildexecution.googleapis.com
+build --tls_enabled=true
+
+# Enable authentication. This will pick up application default credentials by
+# default. You can use --auth_credentials=some_file.json to use a service
+# account credential instead.
+# How to setup credentials:
+# See https://cloud.google.com/remote-build-execution/docs/getting-started#set_credentials
+build --auth_enabled=true
+
+# Set flags for uploading to BES in order to view results in the Bazel Build
+# Results UI.
+build --bes_backend="buildeventservice.googleapis.com"
+build --bes_timeout=60s
+build --bes_results_url="https://source.cloud.google.com/results/invocations/"
+build --project_id=grpc-testing
+
+build --jobs=100
+
+# TODO(jtattermusch): this should be part of the common config
+# but currently sanitizers use different test_timeout values
+build --test_timeout=300,450,1200,3600
+
+# print output for tests that fail (default is "summary")
+build --test_output=errors
diff --git a/tools/remote_build/rbe_common.bazelrc b/tools/remote_build/rbe_common.bazelrc
new file mode 100644
index 0000000000..e8c7f0b9cb
--- /dev/null
+++ b/tools/remote_build/rbe_common.bazelrc
@@ -0,0 +1,83 @@
+# Copyright 2018 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# bazelrc with Foundry setting common to both manual run and runs started by Kokoro
+# see https://github.com/bazelbuild/bazel-toolchains/tree/master/bazelrc
+# for examples and more documentation
+
+startup --host_jvm_args=-Dbazel.DigestFunction=SHA256
+
+build --crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.0/bazel_0.16.1/default:toolchain
+build --extra_toolchains=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.0/bazel_0.16.1/cpp:cc-toolchain-clang-x86_64-default
+# Use custom execution platforms defined in third_party/toolchains
+build --extra_execution_platforms=//third_party/toolchains:rbe_ubuntu1604
+build --host_platform=//third_party/toolchains:rbe_ubuntu1604
+build --platforms=//third_party/toolchains:rbe_ubuntu1604
+
+build --spawn_strategy=remote
+build --strategy=Javac=remote
+build --strategy=Closure=remote
+build --genrule_strategy=remote
+build --remote_timeout=3600
+
+build --remote_instance_name=projects/grpc-testing/instances/default_instance
+
+build --verbose_failures=true
+
+build --experimental_strict_action_env=true
+build --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
+
+# don't use port server
+build --define GRPC_PORT_ISOLATED_RUNTIME=1
+# without verbose gRPC logs the test outputs are not very useful
+test --test_env=GRPC_VERBOSITY=debug
+
+# address sanitizer: most settings are already in %workspace%/.bazelrc
+# we only need a few additional ones that are Foundry specific
+build:asan --copt=-gmlt
+# TODO(jtattermusch): use more reasonable test timeout
+build:asan --test_timeout=3600
+build:asan --test_tag_filters=-qps_json_driver,-json_run_localhost
+
+# memory sanitizer: most settings are already in %workspace%/.bazelrc
+# we only need a few additional ones that are Foundry specific
+build:msan --copt=-gmlt
+# TODO(jtattermusch): use more reasonable test timeout
+build:msan --test_timeout=3600
+build:msan --cxxopt=--stdlib=libc++
+# setting LD_LIBRARY_PATH is necessary
+# to avoid "libc++.so.1: cannot open shared object file"
+build:msan --action_env=LD_LIBRARY_PATH=/usr/local/lib
+build:msan --host_crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.0/bazel_0.16.1/default:toolchain
+# override the config-agnostic crosstool_top
+build:msan --crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.0/bazel_0.16.1/msan:toolchain
+
+# thread sanitizer: most settings are already in %workspace%/.bazelrc
+# we only need a few additional ones that are Foundry specific
+build:tsan --copt=-gmlt
+# TODO(jtattermusch): use more reasonable test timeout
+build:tsan --test_timeout=3600
+build:tsan --test_tag_filters=-qps_json_driver,-json_run_localhost
+
+# undefined behavior sanitizer: most settings are already in %workspace%/.bazelrc
+# we only need a few additional ones that are Foundry specific
+build:ubsan --copt=-gmlt
+# TODO(jtattermusch): use more reasonable test timeout
+build:ubsan --test_timeout=3600
+# override the config-agnostic crosstool_top
+--crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/experimental/ubuntu16_04_clang/1.0/bazel_0.16.1/ubsan:toolchain
+# TODO(jtattermusch): remove this once Foundry adds the env to the docker image.
+# ubsan needs symbolizer to work properly, otherwise the suppression file doesn't work
+# and we get test failures.
+build:ubsan --action_env=UBSAN_SYMBOLIZER_PATH=/usr/local/bin/llvm-symbolizer