Implement retry logic for the gRPC calls in remote execution and caching. The

retry strategy may need tuning.
Other behavior changes: swallowing gRPC CANCELLED errors when the thread is interrupted, as these are expected and just make debugging difficult. Also, distinguishing between the gRPC DEADLINE_EXCEEDED caused by the actual command timing out on the server vs. other causes (the former should not be retriable, while the latter should retry).

TESTED=unit tests, remote worker on Bazel
PiperOrigin-RevId: 160605830

1 files changed, 48 insertions, 0 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteOptions.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteOptions.java
index 07aaff11f7..d472adae09 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteOptions.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteOptions.java
@@ -154,4 +154,52 @@ public final class RemoteOptions extends OptionsBase {
     help = "Value to pass as instance_name in the remote execution API."
   )
   public String remoteInstanceName;
+
+  @Option(
+    name = "experimental_remote_retry",
+    defaultValue = "true",
+    category = "remote",
+    help = "Whether to retry transient remote execution/cache errors."
+  )
+  public boolean experimentalRemoteRetry;
+
+  @Option(
+    name = "experimental_remote_retry_start_delay_millis",
+    defaultValue = "100",
+    category = "remote",
+    help = "The initial delay before retrying a transient error."
+  )
+  public long experimentalRemoteRetryStartDelayMillis;
+
+  @Option(
+    name = "experimental_remote_retry_max_delay_millis",
+    defaultValue = "5000",
+    category = "remote",
+    help = "The maximum delay before retrying a transient error."
+  )
+  public long experimentalRemoteRetryMaxDelayMillis;
+
+  @Option(
+    name = "experimental_remote_retry_max_attempts",
+    defaultValue = "5",
+    category = "remote",
+    help = "The maximum number of attempts to retry a transient error."
+  )
+  public int experimentalRemoteRetryMaxAttempts;
+
+  @Option(
+    name = "experimental_remote_retry_multiplier",
+    defaultValue = "2",
+    category = "remote",
+    help = "The multiplier by which to increase the retry delay on transient errors."
+  )
+  public double experimentalRemoteRetryMultiplier;
+
+  @Option(
+    name = "experimental_remote_retry_jitter",
+    defaultValue = "0.1",
+    category = "remote",
+    help = "The random factor to apply to retry delays on transient errors."
+  )
+  public double experimentalRemoteRetryJitter;
 }