Record correct exit code for uncaught exceptions in async threads.

Async threads are divorced from the server's error-reporting mechanism. In the event
of a server shutdown originating in an async-thread, write the exit code to a file that
can be read by the client.

--
PiperOrigin-RevId: 141920284
MOS_MIGRATED_REVID=141920284

4 files changed, 63 insertions, 1 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java
index 999d286c84..74c49f4189 100644
--- a/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java
+++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java
@@ -58,6 +58,7 @@ import com.google.devtools.build.lib.shell.SubprocessBuilder;
 import com.google.devtools.build.lib.util.AbruptExitException;
 import com.google.devtools.build.lib.util.BlazeClock;
 import com.google.devtools.build.lib.util.Clock;
+import com.google.devtools.build.lib.util.CustomExitCodePublisher;
 import com.google.devtools.build.lib.util.ExitCode;
 import com.google.devtools.build.lib.util.LoggingUtil;
 import com.google.devtools.build.lib.util.OS;
@@ -983,7 +984,9 @@ public final class BlazeRuntime {
           ExitCode.LOCAL_ENVIRONMENTAL_ERROR);
     }
     runtime.initWorkspace(directories, binTools);
-
+    if (startupOptions.useCustomExitCodeOnAbruptExit) {
+      CustomExitCodePublisher.setAbruptExitStatusFileDir(serverDirectories.getOutputBase());
+    }
     AutoProfiler.setClock(runtime.getClock());
     BugReport.setRuntime(runtime);
     return runtime;
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeServerStartupOptions.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeServerStartupOptions.java
index 64a9da783e..fe4fe8cfe1 100644
--- a/src/main/java/com/google/devtools/build/lib/runtime/BlazeServerStartupOptions.java
+++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeServerStartupOptions.java
@@ -286,4 +286,16 @@ public class BlazeServerStartupOptions extends OptionsBase {
       category = "server startup",
       help = "The amount of time the client waits for each attempt to connect to the server")
   public int connectTimeoutSecs;
+
+  @Option(
+    name = "use_custom_exit_code_on_abrupt_exit",
+    defaultValue = "true", // NOTE: purely decorative!
+    category = "undocumented",
+    help =
+        "If the Blaze server exits abruptly without access to traditional channels to "
+            + "communicate its exit code, attempt to communicate through a backup channel. "
+            + "This flag is temporary to allow easy disabling of a new behavior. "
+            + "Please do not use this flag."
+  )
+  public boolean useCustomExitCodeOnAbruptExit;
 }
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BugReport.java b/src/main/java/com/google/devtools/build/lib/runtime/BugReport.java
index 9a6e978878..d0470364d9 100644
--- a/src/main/java/com/google/devtools/build/lib/runtime/BugReport.java
+++ b/src/main/java/com/google/devtools/build/lib/runtime/BugReport.java
@@ -17,6 +17,7 @@ import com.google.common.base.Joiner;
 import com.google.common.base.Throwables;
 import com.google.common.collect.ImmutableList;
 import com.google.devtools.build.lib.analysis.BlazeVersionInfo;
+import com.google.devtools.build.lib.util.CustomExitCodePublisher;
 import com.google.devtools.build.lib.util.ExitCode;
 import com.google.devtools.build.lib.util.LoggingUtil;
 import com.google.devtools.build.lib.util.Preconditions;
@@ -94,6 +95,7 @@ public abstract class BugReport {
             // to do as a best-effort operation.
             runtime.shutdownOnCrash();
           }
+          CustomExitCodePublisher.maybeWriteExitStatusFile(exitCode);
         } finally {
           // Avoid shutdown deadlock issues: If an application shutdown hook crashes, it will
           // trigger our Blaze crash handler (this method). Calling System#exit() here, would
diff --git a/src/main/java/com/google/devtools/build/lib/util/CustomExitCodePublisher.java b/src/main/java/com/google/devtools/build/lib/util/CustomExitCodePublisher.java
new file mode 100644
index 0000000000..c281f5a1fa
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/CustomExitCodePublisher.java
@@ -0,0 +1,45 @@
+// Copyright 2016 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import java.io.IOException;
+import javax.annotation.Nullable;
+
+/**
+ * Provides an external way for the Bazel server to communicate its exit code to the client, when
+ * the main gRPC channel is unavailable because the exit is too abrupt or originated in an async
+ * thread.
+ */
+public class CustomExitCodePublisher {
+  private static final String EXIT_CODE_FILENAME = "exit_code_to_use_on_abrupt_exit";
+  @Nullable private static Path abruptExitCodeFilePath = null;
+
+  public static void setAbruptExitStatusFileDir(Path path) {
+    abruptExitCodeFilePath = path.getChild(EXIT_CODE_FILENAME);
+  }
+
+  public static void maybeWriteExitStatusFile(int exitCode) {
+    if (abruptExitCodeFilePath != null) {
+      try {
+        FileSystemUtils.writeContentAsLatin1(abruptExitCodeFilePath, String.valueOf(exitCode));
+      } catch (IOException ioe) {
+        System.err.printf(
+            "io error writing %d to abrupt exit status file %s: %s\n",
+            exitCode, abruptExitCodeFilePath, ioe.getMessage());
+      }
+    }
+  }
+}