grpc: Consolidate gRPC code from BES and Remote Execution. Fixes #3460, #3486

BES and Remote Execution have separate implementations of gRPC channel
creation, authentication and TLS. We should merge them, to avoid
duplication and bugs. One such bug is #3640, where the BES code had a
different implementation for Google Application Default Credentials.

RELNOTES: The Build Event Service (BES) client now properly supports
Google Applicaton Default Credentials.
PiperOrigin-RevId: 164253879

11 files changed, 126 insertions, 239 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/BUILD b/src/main/java/com/google/devtools/build/lib/BUILD
index f17579cdfa..66a5f5fa9a 100644
--- a/src/main/java/com/google/devtools/build/lib/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/BUILD
@@ -391,7 +391,7 @@ java_library(
 
 java_library(
     name = "auth_and_tls_options",
-    srcs = ["authandtls/AuthAndTLSOptions.java"],
+    srcs = glob(["authandtls/*.java"]),
     visibility = [
         "//src/main/java/com/google/devtools/build/lib/remote:__pkg__",
         "//src/test/java/com/google/devtools/build/lib:__pkg__",
@@ -401,6 +401,13 @@ java_library(
     ],
     deps = [
         "//src/main/java/com/google/devtools/common/options",
+        "//third_party:apache_httpclient",
+        "//third_party:apache_httpcore",
+        "//third_party:auth",
+        "//third_party:guava",
+        "//third_party:jsr305",
+        "//third_party:netty",
+        "//third_party/grpc:grpc-jar",
     ],
 )
 
diff --git a/src/main/java/com/google/devtools/build/lib/authandtls/AuthAndTLSOptions.java b/src/main/java/com/google/devtools/build/lib/authandtls/AuthAndTLSOptions.java
index a1dbc01f4f..a38adb0519 100644
--- a/src/main/java/com/google/devtools/build/lib/authandtls/AuthAndTLSOptions.java
+++ b/src/main/java/com/google/devtools/build/lib/authandtls/AuthAndTLSOptions.java
@@ -39,7 +39,7 @@ public class AuthAndTLSOptions extends OptionsBase {
 
   @Option(
     name = "auth_scope",
-    defaultValue = "null",
+    defaultValue = "https://www.googleapis.com/auth/cloud-build-service",
     category = "remote",
     documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
     effectTags = {OptionEffectTag.UNKNOWN},
diff --git a/src/main/java/com/google/devtools/build/lib/remote/ChannelOptions.java b/src/main/java/com/google/devtools/build/lib/authandtls/GrpcUtils.java
index 9a3f736455..5f1ff70c2d 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/ChannelOptions.java
+++ b/src/main/java/com/google/devtools/build/lib/authandtls/GrpcUtils.java
@@ -12,16 +12,19 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package com.google.devtools.build.lib.remote;
+package com.google.devtools.build.lib.authandtls;
 
 import com.google.auth.oauth2.GoogleCredentials;
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
-import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions;
-import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
 import io.grpc.CallCredentials;
+import io.grpc.ManagedChannel;
 import io.grpc.auth.MoreCallCredentials;
 import io.grpc.netty.GrpcSslContexts;
+import io.grpc.netty.NegotiationType;
+import io.grpc.netty.NettyChannelBuilder;
+import io.grpc.util.RoundRobinLoadBalancerFactory;
 import io.netty.handler.ssl.SslContext;
 import java.io.File;
 import java.io.FileInputStream;
@@ -30,70 +33,91 @@ import java.io.IOException;
 import java.io.InputStream;
 import javax.annotation.Nullable;
 
-/** Instantiate all authentication helpers from build options. */
-@ThreadSafe
-public final class ChannelOptions {
-  private final boolean tlsEnabled;
-  private final SslContext sslContext;
-  private final String tlsAuthorityOverride;
-  private final CallCredentials credentials;
+/**
+ * Utility methods for using {@link AuthAndTLSOptions} with gRPC.
+ */
+public final class GrpcUtils {
 
-  private ChannelOptions(
-      boolean tlsEnabled,
-      SslContext sslContext,
-      String tlsAuthorityOverride,
-      CallCredentials credentials) {
-    this.tlsEnabled = tlsEnabled;
-    this.sslContext = sslContext;
-    this.tlsAuthorityOverride = tlsAuthorityOverride;
-    this.credentials = credentials;
-  }
+  /**
+   * Create a new gRPC {@link ManagedChannel}.
+   *
+   * @throws IOException  in case the channel can't be constructed.
+   */
+  public static ManagedChannel newChannel(String target, AuthAndTLSOptions options)
+      throws IOException {
+    Preconditions.checkNotNull(target);
+    Preconditions.checkNotNull(options);
 
-  public boolean tlsEnabled() {
-    return tlsEnabled;
-  }
+    final SslContext sslContext =
+        options.tlsEnabled ? createSSlContext(options.tlsCertificate) : null;
 
-  public CallCredentials getCallCredentials() {
-    return credentials;
+    try {
+      NettyChannelBuilder builder =
+          NettyChannelBuilder.forTarget(target)
+              .negotiationType(
+                  options.tlsEnabled ? NegotiationType.TLS : NegotiationType.PLAINTEXT)
+              .loadBalancerFactory(RoundRobinLoadBalancerFactory.getInstance());
+      if (sslContext != null) {
+        builder.sslContext(sslContext);
+        if (options.tlsAuthorityOverride != null) {
+          builder.overrideAuthority(options.tlsAuthorityOverride);
+        }
+      }
+      return builder.build();
+    } catch (RuntimeException e) {
+      // gRPC might throw all kinds of RuntimeExceptions: StatusRuntimeException,
+      // IllegalStateException, NullPointerException, ...
+      String message = "Failed to connect to '%s': %s";
+      throw new IOException(String.format(message, target, e.getMessage()));
+    }
   }
 
-  public String getTlsAuthorityOverride() {
-    return tlsAuthorityOverride;
+  private static SslContext createSSlContext(@Nullable String rootCert) throws IOException {
+    if (rootCert == null) {
+      try {
+        return GrpcSslContexts.forClient().build();
+      } catch (Exception e) {
+        String message = "Failed to init TLS infrastructure: "
+            + e.getMessage();
+        throw new IOException(message, e);
+      }
+    } else {
+      try {
+        return GrpcSslContexts.forClient().trustManager(new File(rootCert)).build();
+      } catch (Exception e) {
+        String message = "Failed to init TLS infrastructure using '%s' as root certificate: %s";
+        message = String.format(message, rootCert, e.getMessage());
+        throw new IOException(message, e);
+      }
+    }
   }
 
-  public SslContext getSslContext() {
-    return sslContext;
-  }
+  /**
+   * Create a new {@link CallCredentials} object.
+   *
+   * @throws IOException  in case the call credentials can't be constructed.
+   */
+  public static CallCredentials newCallCredentials(AuthAndTLSOptions options) throws IOException {
+    if (!options.authEnabled) {
+      return null;
+    }
 
-  public static ChannelOptions create(AuthAndTLSOptions options) throws IOException {
     if (options.authCredentials != null) {
+      // Credentials from file
       try (InputStream authFile = new FileInputStream(options.authCredentials)) {
-        return create(options, authFile);
+        return newCallCredentials(authFile, options.authScope);
       } catch (FileNotFoundException e) {
         String message = String.format("Could not open auth credentials file '%s': %s",
             options.authCredentials, e.getMessage());
         throw new IOException(message, e);
       }
-    } else {
-      return create(options, null);
     }
+    // Google Application Default Credentials
+    return newCallCredentials(null, options.authScope);
   }
 
   @VisibleForTesting
-  static ChannelOptions create(
-      AuthAndTLSOptions options,
-      @Nullable InputStream credentialsFile) throws IOException {
-    final SslContext sslContext =
-        options.tlsEnabled ? createSSlContext(options.tlsCertificate) : null;
-
-    final CallCredentials callCredentials =
-        options.authEnabled ? createCallCredentials(credentialsFile, options.authScope) : null;
-
-    return new ChannelOptions(
-        sslContext != null, sslContext, options.tlsAuthorityOverride, callCredentials);
-  }
-
-  private static CallCredentials createCallCredentials(@Nullable InputStream credentialsFile,
+  public static CallCredentials newCallCredentials(@Nullable InputStream credentialsFile,
       @Nullable String authScope) throws IOException {
     try {
       GoogleCredentials creds =
@@ -105,30 +129,9 @@ public final class ChannelOptions {
       }
       return MoreCallCredentials.from(creds);
     } catch (IOException e) {
-      String message = "Failed to init auth credentials for remote caching/execution: "
+      String message = "Failed to init auth credentials: "
           + e.getMessage();
       throw new IOException(message, e);
     }
   }
-
-  private static SslContext createSSlContext(@Nullable String rootCert) throws IOException {
-    if (rootCert == null) {
-      try {
-        return GrpcSslContexts.forClient().build();
-      } catch (Exception e) {
-        String message = "Failed to init TLS infrastructure for remote caching/execution: "
-            + e.getMessage();
-        throw new IOException(message, e);
-      }
-    } else {
-      try {
-        return GrpcSslContexts.forClient().trustManager(new File(rootCert)).build();
-      } catch (Exception e) {
-        String message = "Failed to init TLS infrastructure for remote caching/execution using "
-            + "'%s' as root certificate: %s";
-        message = String.format(message, rootCert, e.getMessage());
-        throw new IOException(message, e);
-      }
-    }
-  }
 }
diff --git a/src/main/java/com/google/devtools/build/lib/buildeventservice/BazelBuildEventServiceModule.java b/src/main/java/com/google/devtools/build/lib/buildeventservice/BazelBuildEventServiceModule.java
index cdbc58afbe..4f205e1189 100644
--- a/src/main/java/com/google/devtools/build/lib/buildeventservice/BazelBuildEventServiceModule.java
+++ b/src/main/java/com/google/devtools/build/lib/buildeventservice/BazelBuildEventServiceModule.java
@@ -16,8 +16,10 @@ package com.google.devtools.build.lib.buildeventservice;
 
 import com.google.common.collect.ImmutableSet;
 import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions;
+import com.google.devtools.build.lib.authandtls.GrpcUtils;
 import com.google.devtools.build.lib.buildeventservice.client.BuildEventServiceClient;
 import com.google.devtools.build.lib.buildeventservice.client.BuildEventServiceGrpcClient;
+import java.io.IOException;
 import java.util.Set;
 
 /**
@@ -33,11 +35,10 @@ public class BazelBuildEventServiceModule
 
   @Override
   protected BuildEventServiceClient createBesClient(BuildEventServiceOptions besOptions,
-      AuthAndTLSOptions authAndTLSOptions) {
+      AuthAndTLSOptions authAndTLSOptions) throws IOException {
     return new BuildEventServiceGrpcClient(
-        besOptions.besBackend, authAndTLSOptions.tlsEnabled, authAndTLSOptions.tlsCertificate,
-        authAndTLSOptions.tlsAuthorityOverride, authAndTLSOptions.authCredentials,
-        authAndTLSOptions.authScope);
+        GrpcUtils.newChannel(besOptions.besBackend, authAndTLSOptions),
+        GrpcUtils.newCallCredentials(authAndTLSOptions));
   }
 
   @Override
diff --git a/src/main/java/com/google/devtools/build/lib/buildeventservice/BuildEventServiceModule.java b/src/main/java/com/google/devtools/build/lib/buildeventservice/BuildEventServiceModule.java
index ead187d005..19b3e966c8 100644
--- a/src/main/java/com/google/devtools/build/lib/buildeventservice/BuildEventServiceModule.java
+++ b/src/main/java/com/google/devtools/build/lib/buildeventservice/BuildEventServiceModule.java
@@ -42,6 +42,7 @@ import com.google.devtools.build.lib.util.ExitCode;
 import com.google.devtools.build.lib.util.io.OutErr;
 import com.google.devtools.common.options.OptionsBase;
 import com.google.devtools.common.options.OptionsProvider;
+import java.io.IOException;
 import java.util.Set;
 import java.util.UUID;
 import java.util.logging.Logger;
@@ -184,7 +185,7 @@ public abstract class BuildEventServiceModule<T extends BuildEventServiceOptions
   @Nullable
   private BuildEventTransport tryCreateBesTransport(T besOptions, AuthAndTLSOptions authTlsOptions,
       String buildRequestId, String invocationId, ModuleEnvironment moduleEnvironment, Clock clock,
-      PathConverter pathConverter, EventHandler commandLineReporter) {
+      PathConverter pathConverter, EventHandler commandLineReporter) throws IOException {
     if (isNullOrEmpty(besOptions.besBackend)) {
       logger.fine("BuildEventServiceTransport is disabled.");
       return null;
@@ -222,7 +223,7 @@ public abstract class BuildEventServiceModule<T extends BuildEventServiceOptions
   protected abstract Class<T> optionsClass();
 
   protected abstract BuildEventServiceClient createBesClient(T besOptions,
-      AuthAndTLSOptions authAndTLSOptions);
+      AuthAndTLSOptions authAndTLSOptions) throws IOException;
 
   protected abstract Set<String> whitelistedCommands();
 }
diff --git a/src/main/java/com/google/devtools/build/lib/buildeventservice/client/BuildEventServiceGrpcClient.java b/src/main/java/com/google/devtools/build/lib/buildeventservice/client/BuildEventServiceGrpcClient.java
index 583837992d..6a32ddef90 100644
--- a/src/main/java/com/google/devtools/build/lib/buildeventservice/client/BuildEventServiceGrpcClient.java
+++ b/src/main/java/com/google/devtools/build/lib/buildeventservice/client/BuildEventServiceGrpcClient.java
@@ -14,17 +14,12 @@
 
 package com.google.devtools.build.lib.buildeventservice.client;
 
-import static com.google.common.base.Strings.isNullOrEmpty;
 import static com.google.devtools.build.lib.util.Preconditions.checkNotNull;
 import static com.google.devtools.build.lib.util.Preconditions.checkState;
-import static java.lang.System.getenv;
-import static java.nio.file.Files.newInputStream;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 
-import com.google.auth.oauth2.GoogleCredentials;
 import com.google.common.base.Function;
 import com.google.common.base.Throwables;
-import com.google.common.collect.ImmutableList;
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.SettableFuture;
 import com.google.devtools.build.v1.PublishBuildEventGrpc;
@@ -37,49 +32,23 @@ import io.grpc.CallCredentials;
 import io.grpc.ManagedChannel;
 import io.grpc.Status;
 import io.grpc.StatusRuntimeException;
-import io.grpc.auth.MoreCallCredentials;
-import io.grpc.netty.GrpcSslContexts;
-import io.grpc.netty.NegotiationType;
-import io.grpc.netty.NettyChannelBuilder;
 import io.grpc.stub.AbstractStub;
 import io.grpc.stub.StreamObserver;
-import io.netty.handler.ssl.SslContext;
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Paths;
 import java.util.concurrent.atomic.AtomicReference;
-import java.util.logging.Level;
-import java.util.logging.Logger;
 import javax.annotation.Nullable;
-import javax.net.ssl.SSLException;
 import org.joda.time.Duration;
 
 /** Implementation of BuildEventServiceClient that uploads data using gRPC. */
 public class BuildEventServiceGrpcClient implements BuildEventServiceClient {
 
-  private static final Logger logger =
-      Logger.getLogger(BuildEventServiceGrpcClient.class.getName());
-
   /** Max wait time for a single non-streaming RPC to finish */
   private static final Duration RPC_TIMEOUT = Duration.standardSeconds(15);
-  /** See https://developers.google.com/identity/protocols/application-default-credentials * */
-  private static final String DEFAULT_APP_CREDENTIALS_ENV_VAR = "GOOGLE_APPLICATION_CREDENTIALS";
-  /** TODO(eduardocolaco): Scope documentation.* */
-  private static final String CREDENTIALS_SCOPE =
-      "https://www.googleapis.com/auth/cloud-build-service";
 
   private final PublishBuildEventStub besAsync;
   private final PublishBuildEventBlockingStub besBlocking;
   private final ManagedChannel channel;
   private final AtomicReference<StreamObserver<PublishBuildToolEventStreamRequest>> streamReference;
 
-  public BuildEventServiceGrpcClient(String serverSpec, boolean tlsEnabled,
-      @Nullable String tlsCertificateFile, @Nullable String tlsAuthorityOverride,
-      @Nullable String credentialsFile, @Nullable String credentialsScope) {
-    this(getChannel(serverSpec, tlsEnabled, tlsCertificateFile, tlsAuthorityOverride),
-        getCallCredentials(credentialsFile, credentialsScope));
-  }
-
   public BuildEventServiceGrpcClient(
       ManagedChannel channel,
       @Nullable CallCredentials callCredentials) {
@@ -183,52 +152,4 @@ public class BuildEventServiceGrpcClient implements BuildEventServiceClient {
       return t.getMessage();
     }
   }
-
-  /**
-   * Returns call credentials read from the specified file (if non-empty) or from
-   * env(GOOGLE_APPLICATION_CREDENTIALS) otherwise.
-   */
-  @Nullable
-  private static CallCredentials getCallCredentials(@Nullable String credentialsFile,
-      @Nullable String credentialsScope) {
-    String effectiveScope = credentialsScope != null ? credentialsScope : CREDENTIALS_SCOPE;
-    try {
-      if (!isNullOrEmpty(credentialsFile)) {
-        return MoreCallCredentials.from(
-            GoogleCredentials.fromStream(newInputStream(Paths.get(credentialsFile)))
-                .createScoped(ImmutableList.of(effectiveScope)));
-
-      } else if (!isNullOrEmpty(getenv(DEFAULT_APP_CREDENTIALS_ENV_VAR))) {
-        return MoreCallCredentials.from(
-            GoogleCredentials.getApplicationDefault()
-                .createScoped(ImmutableList.of(effectiveScope)));
-      }
-    } catch (IOException e) {
-      logger.log(Level.WARNING, "Failed to read credentials", e);
-    }
-    return null;
-  }
-
-  /**
-   * Returns a ManagedChannel to the specified server.
-   */
-  private static ManagedChannel getChannel(String serverSpec, boolean tlsEnabled,
-      @Nullable String tlsCertificateFile, @Nullable String tlsAuthorityOverride) {
-    //TODO(buchgr): Use ManagedChannelBuilder once bazel uses a newer gRPC version.
-    NettyChannelBuilder builder = NettyChannelBuilder.forTarget(serverSpec);
-    builder.negotiationType(tlsEnabled ? NegotiationType.TLS : NegotiationType.PLAINTEXT);
-    if (tlsCertificateFile != null) {
-      try {
-        SslContext sslContext =
-            GrpcSslContexts.forClient().trustManager(new File(tlsCertificateFile)).build();
-        builder.sslContext(sslContext);
-      } catch (SSLException e) {
-        throw new RuntimeException(e);
-      }
-    }
-    if (tlsAuthorityOverride != null) {
-      builder.overrideAuthority(tlsAuthorityOverride);
-    }
-    return builder.build();
-  }
 }
diff --git a/src/main/java/com/google/devtools/build/lib/remote/GrpcRemoteCache.java b/src/main/java/com/google/devtools/build/lib/remote/GrpcRemoteCache.java
index fa759414c3..09bc29f91d 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/GrpcRemoteCache.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/GrpcRemoteCache.java
@@ -49,6 +49,7 @@ import com.google.devtools.remoteexecution.v1test.GetActionResultRequest;
 import com.google.devtools.remoteexecution.v1test.OutputFile;
 import com.google.devtools.remoteexecution.v1test.UpdateActionResultRequest;
 import com.google.protobuf.ByteString;
+import io.grpc.CallCredentials;
 import io.grpc.Channel;
 import io.grpc.Status;
 import io.grpc.StatusRuntimeException;
@@ -69,7 +70,7 @@ import java.util.concurrent.TimeUnit;
 @ThreadSafe
 public class GrpcRemoteCache implements RemoteActionCache {
   private final RemoteOptions options;
-  private final ChannelOptions channelOptions;
+  private final CallCredentials credentials;
   private final Channel channel;
   private final Retrier retrier;
 
@@ -79,32 +80,32 @@ public class GrpcRemoteCache implements RemoteActionCache {
       MoreExecutors.listeningDecorator(Executors.newScheduledThreadPool(1));
 
   @VisibleForTesting
-  public GrpcRemoteCache(Channel channel, ChannelOptions channelOptions, RemoteOptions options,
+  public GrpcRemoteCache(Channel channel, CallCredentials credentials, RemoteOptions options,
       Retrier retrier) {
     this.options = options;
-    this.channelOptions = channelOptions;
+    this.credentials = credentials;
     this.channel = channel;
     this.retrier = retrier;
 
-    uploader = new ByteStreamUploader(options.remoteInstanceName, channel,
-        channelOptions.getCallCredentials(), options.remoteTimeout, retrier, retryScheduler);
+    uploader = new ByteStreamUploader(options.remoteInstanceName, channel, credentials,
+        options.remoteTimeout, retrier, retryScheduler);
   }
 
   private ContentAddressableStorageBlockingStub casBlockingStub() {
     return ContentAddressableStorageGrpc.newBlockingStub(channel)
-        .withCallCredentials(channelOptions.getCallCredentials())
+        .withCallCredentials(credentials)
         .withDeadlineAfter(options.remoteTimeout, TimeUnit.SECONDS);
   }
 
   private ByteStreamBlockingStub bsBlockingStub() {
     return ByteStreamGrpc.newBlockingStub(channel)
-        .withCallCredentials(channelOptions.getCallCredentials())
+        .withCallCredentials(credentials)
         .withDeadlineAfter(options.remoteTimeout, TimeUnit.SECONDS);
   }
 
   private ActionCacheBlockingStub acBlockingStub() {
     return ActionCacheGrpc.newBlockingStub(channel)
-        .withCallCredentials(channelOptions.getCallCredentials())
+        .withCallCredentials(credentials)
         .withDeadlineAfter(options.remoteTimeout, TimeUnit.SECONDS);
   }
 
diff --git a/src/main/java/com/google/devtools/build/lib/remote/GrpcUtils.java b/src/main/java/com/google/devtools/build/lib/remote/GrpcUtils.java
deleted file mode 100644
index e559885a02..0000000000
--- a/src/main/java/com/google/devtools/build/lib/remote/GrpcUtils.java
+++ /dev/null
@@ -1,50 +0,0 @@
-// Copyright 2016 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.remote;
-
-import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
-import io.grpc.ManagedChannel;
-import io.grpc.netty.NegotiationType;
-import io.grpc.netty.NettyChannelBuilder;
-import io.grpc.util.RoundRobinLoadBalancerFactory;
-
-import java.io.IOException;
-
-/** Helper methods for gRPC calls */
-@ThreadSafe
-final class GrpcUtils {
-  public static ManagedChannel createChannel(String target, ChannelOptions channelOptions)
-      throws IOException {
-    try {
-      NettyChannelBuilder builder =
-          NettyChannelBuilder.forTarget(target)
-              .negotiationType(
-                  channelOptions.tlsEnabled() ? NegotiationType.TLS : NegotiationType.PLAINTEXT)
-            .loadBalancerFactory(RoundRobinLoadBalancerFactory.getInstance());
-      if (channelOptions.getSslContext() != null) {
-        builder.sslContext(channelOptions.getSslContext());
-        if (channelOptions.getTlsAuthorityOverride() != null) {
-          builder.overrideAuthority(channelOptions.getTlsAuthorityOverride());
-        }
-      }
-      return builder.build();
-    } catch (RuntimeException e) {
-      // gRPC might throw all kinds of RuntimeExceptions: StatusRuntimeException,
-      // IllegalStateException, NullPointerException, ...
-      String message = "Failed to connect to the remote cache/executor '%s': %s";
-      throw new IOException(String.format(message, target, e.getMessage()));
-    }
-  }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
index 1a4b874db9..6b8eb87b98 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
@@ -17,6 +17,7 @@ package com.google.devtools.build.lib.remote;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
 import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions;
+import com.google.devtools.build.lib.authandtls.GrpcUtils;
 import com.google.devtools.build.lib.buildeventstream.PathConverter;
 import com.google.devtools.build.lib.buildtool.BuildRequest;
 import com.google.devtools.build.lib.events.Event;
@@ -31,6 +32,8 @@ import com.google.devtools.build.lib.vfs.Path;
 import com.google.devtools.common.options.OptionsBase;
 import com.google.devtools.common.options.OptionsProvider;
 import com.google.devtools.remoteexecution.v1test.Digest;
+import io.grpc.CallCredentials;
+import io.grpc.Channel;
 import java.io.IOException;
 
 /** RemoteModule provides distributed cache and remote execution for Bazel. */
@@ -97,23 +100,19 @@ public final class RemoteModule extends BlazeModule {
     }
 
     try {
-      ChannelOptions channelOpts = ChannelOptions.create(authAndTlsOptions);
-
       boolean restCache = SimpleBlobStoreFactory.isRemoteCacheOptions(remoteOptions);
       boolean grpcCache = GrpcRemoteCache.isRemoteCacheOptions(remoteOptions);
 
       Retrier retrier = new Retrier(remoteOptions);
+      CallCredentials creds = GrpcUtils.newCallCredentials(authAndTlsOptions);
       final RemoteActionCache cache;
       if (restCache) {
         cache = new SimpleBlobStoreActionCache(SimpleBlobStoreFactory.create(remoteOptions));
-      } else if (grpcCache) {
-        cache = new GrpcRemoteCache(GrpcUtils.createChannel(remoteOptions.remoteCache, channelOpts),
-            channelOpts, remoteOptions, retrier);
-      } else if (remoteOptions.remoteExecutor != null) {
+      } else if (grpcCache || remoteOptions.remoteExecutor != null) {
         // If a remote executor but no remote cache is specified, assume both at the same target.
-        cache =
-            new GrpcRemoteCache(GrpcUtils.createChannel(remoteOptions.remoteExecutor, channelOpts),
-                channelOpts, remoteOptions, retrier);
+        String target = grpcCache ? remoteOptions.remoteCache : remoteOptions.remoteExecutor;
+        Channel ch = GrpcUtils.newChannel(target, authAndTlsOptions);
+        cache = new GrpcRemoteCache(ch, creds, remoteOptions, retrier);
       } else {
         cache = null;
       }
@@ -121,8 +120,8 @@ public final class RemoteModule extends BlazeModule {
       final GrpcRemoteExecutor executor;
       if (remoteOptions.remoteExecutor != null) {
         executor = new GrpcRemoteExecutor(
-            GrpcUtils.createChannel(remoteOptions.remoteExecutor, channelOpts),
-            channelOpts.getCallCredentials(),
+            GrpcUtils.newChannel(remoteOptions.remoteExecutor, authAndTlsOptions),
+            creds,
             remoteOptions.remoteTimeout,
             retrier);
       } else {
diff --git a/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteCacheTest.java b/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteCacheTest.java
index c43a09f690..1df06cdf42 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteCacheTest.java
+++ b/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteCacheTest.java
@@ -28,6 +28,7 @@ import com.google.bytestream.ByteStreamProto.WriteResponse;
 import com.google.common.collect.ImmutableList;
 import com.google.devtools.build.lib.actions.ActionInputHelper;
 import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions;
+import com.google.devtools.build.lib.authandtls.GrpcUtils;
 import com.google.devtools.build.lib.remote.Digests.ActionKey;
 import com.google.devtools.build.lib.testutil.Scratch;
 import com.google.devtools.build.lib.util.io.FileOutErr;
@@ -46,6 +47,7 @@ import com.google.devtools.remoteexecution.v1test.FindMissingBlobsResponse;
 import com.google.devtools.remoteexecution.v1test.GetActionResultRequest;
 import com.google.devtools.remoteexecution.v1test.UpdateActionResultRequest;
 import com.google.protobuf.ByteString;
+import io.grpc.CallCredentials;
 import io.grpc.CallOptions;
 import io.grpc.Channel;
 import io.grpc.ClientCall;
@@ -108,17 +110,17 @@ public class GrpcRemoteCacheTest {
     fakeServer.shutdownNow();
   }
 
-  private static class ChannelOptionsInterceptor implements ClientInterceptor {
-    private final ChannelOptions channelOptions;
+  private static class CallCredentialsInterceptor implements ClientInterceptor {
+    private final CallCredentials credentials;
 
-    public ChannelOptionsInterceptor(ChannelOptions channelOptions) {
-      this.channelOptions = channelOptions;
+    public CallCredentialsInterceptor(CallCredentials credentials) {
+      this.credentials = credentials;
     }
 
     @Override
     public <RequestT, ResponseT> ClientCall<RequestT, ResponseT> interceptCall(
         MethodDescriptor<RequestT, ResponseT> method, CallOptions callOptions, Channel next) {
-      assertThat(callOptions.getCredentials()).isEqualTo(channelOptions.getCallCredentials());
+      assertThat(callOptions.getCredentials()).isEqualTo(credentials);
       // Remove the call credentials to allow testing with dummy ones.
       return next.newCall(method, callOptions.withCallCredentials(null));
     }
@@ -138,16 +140,16 @@ public class GrpcRemoteCacheTest {
     Scratch scratch = new Scratch();
     scratch.file(authTlsOptions.authCredentials, new JacksonFactory().toString(json));
 
-    ChannelOptions channelOptions =
-        ChannelOptions.create(
-            authTlsOptions, scratch.resolve(authTlsOptions.authCredentials).getInputStream());
+    CallCredentials creds = GrpcUtils.newCallCredentials(
+        scratch.resolve(authTlsOptions.authCredentials).getInputStream(),
+        authTlsOptions.authScope);
     RemoteOptions remoteOptions = Options.getDefaults(RemoteOptions.class);
     Retrier retrier = new Retrier(remoteOptions);
     return new GrpcRemoteCache(
         ClientInterceptors.intercept(
             InProcessChannelBuilder.forName(fakeServerName).directExecutor().build(),
-            ImmutableList.of(new ChannelOptionsInterceptor(channelOptions))),
-        channelOptions,
+            ImmutableList.of(new CallCredentialsInterceptor(creds))),
+        creds,
         remoteOptions,
         retrier);
   }
diff --git a/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteExecutionClientTest.java b/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteExecutionClientTest.java
index 96e95487dc..2fcfa7ee91 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteExecutionClientTest.java
+++ b/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteExecutionClientTest.java
@@ -35,6 +35,7 @@ import com.google.devtools.build.lib.actions.EnvironmentalExecException;
 import com.google.devtools.build.lib.actions.ResourceSet;
 import com.google.devtools.build.lib.actions.SimpleSpawn;
 import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions;
+import com.google.devtools.build.lib.authandtls.GrpcUtils;
 import com.google.devtools.build.lib.exec.SpawnInputExpander;
 import com.google.devtools.build.lib.exec.SpawnResult;
 import com.google.devtools.build.lib.exec.SpawnRunner.ProgressStatus;
@@ -67,6 +68,7 @@ import com.google.watcher.v1.Change;
 import com.google.watcher.v1.ChangeBatch;
 import com.google.watcher.v1.Request;
 import com.google.watcher.v1.WatcherGrpc.WatcherImplBase;
+import io.grpc.CallCredentials;
 import io.grpc.Channel;
 import io.grpc.Server;
 import io.grpc.Status;
@@ -216,10 +218,10 @@ public class GrpcRemoteExecutionClientTest {
     Channel channel = InProcessChannelBuilder.forName(fakeServerName).directExecutor().build();
     GrpcRemoteExecutor executor =
         new GrpcRemoteExecutor(channel, null, options.remoteTimeout, retrier);
-    ChannelOptions defaultOpts =
-        ChannelOptions.create(Options.getDefaults(AuthAndTLSOptions.class));
+    CallCredentials creds =
+        GrpcUtils.newCallCredentials(Options.getDefaults(AuthAndTLSOptions.class));
     GrpcRemoteCache remoteCache =
-        new GrpcRemoteCache(channel, defaultOpts, options, retrier);
+        new GrpcRemoteCache(channel, creds, options, retrier);
     client = new RemoteSpawnRunner(execRoot, options, null, true, remoteCache, executor);
     inputDigest = fakeFileCache.createScratchInput(simpleSpawn.getInputFiles().get(0), "xyz");
   }