Record correct exit code for uncaught exceptions in async threads.

Async threads are divorced from the server's error-reporting mechanism. In the event
of a server shutdown originating in an async-thread, write the exit code to a file that
can be read by the client.

--
PiperOrigin-RevId: 141920284
MOS_MIGRATED_REVID=141920284

6 files changed, 61 insertions, 78 deletions

diff --git a/src/main/cpp/BUILD b/src/main/cpp/BUILD
index 9b8289ecb1..82ed5510cf 100644
--- a/src/main/cpp/BUILD
+++ b/src/main/cpp/BUILD
@@ -60,19 +60,6 @@ cc_library(
     ],
 )
 
-cc_library(
-    name = "blaze_abrupt_exit",
-    srcs = [
-        "blaze_abrupt_exit.cc",
-    ],
-    hdrs = [
-        "blaze_abrupt_exit.h",
-    ],
-    deps = [
-        "//src/main/cpp/util:blaze_exit_code",
-    ],
-)
-
 cc_binary(
     name = "client",
     srcs = [
@@ -109,7 +96,6 @@ cc_binary(
     }),
     visibility = ["//src:__pkg__"],
     deps = [
-        ":blaze_abrupt_exit",
         ":blaze_util",
         "//src/main/cpp/util",
         "//src/main/cpp/util:logging",
diff --git a/src/main/cpp/blaze.cc b/src/main/cpp/blaze.cc
index d3b039e764..07057b9a62 100644
--- a/src/main/cpp/blaze.cc
+++ b/src/main/cpp/blaze.cc
@@ -54,7 +54,6 @@
 #include <vector>
 
 
-#include "src/main/cpp/blaze_abrupt_exit.h"
 #include "src/main/cpp/blaze_util.h"
 #include "src/main/cpp/blaze_util_platform.h"
 #include "src/main/cpp/global_variables.h"
@@ -484,6 +483,11 @@ static vector<string> GetArgumentArray() {
   } else {
     result.push_back("--client_debug=false");
   }
+  if (globals->options->use_custom_exit_code_on_abrupt_exit) {
+    result.push_back("--use_custom_exit_code_on_abrupt_exit=true");
+  } else {
+    result.push_back("--use_custom_exit_code_on_abrupt_exit=false");
+  }
 
   // This is only for Blaze reporting purposes; the real interpretation of the
   // jvm flags occurs when we set up the java command line.
@@ -1271,10 +1275,36 @@ static string CheckAndGetBinaryPath(const string& argv0) {
   }
 }
 
-// TODO(bazel-team): Execute the server as a child process and write its exit
-// code to a file. In case the server becomes unresonsive or terminates
-// unexpectedly (in a way that isn't already handled), we can observe the file,
-// if it exists. (If it doesn't, then we know something went horribly wrong.)
+int GetExitCodeForAbruptExit(const GlobalVariables &globals) {
+  const StartupOptions *startup_options = globals.options;
+  if (startup_options->use_custom_exit_code_on_abrupt_exit) {
+    BAZEL_LOG(INFO) << "Looking for a custom exit-code.";
+    std::string filename = blaze_util::JoinPath(
+        globals.options->output_base, "exit_code_to_use_on_abrupt_exit");
+    std::string content;
+    if (!blaze_util::ReadFile(filename, &content)) {
+      BAZEL_LOG(INFO) << "Unable to read the custom exit-code file. "
+                      << "Exiting with an INTERNAL_ERROR.";
+      return blaze_exit_code::INTERNAL_ERROR;
+    }
+    if (!blaze_util::UnlinkPath(filename)) {
+      BAZEL_LOG(INFO) << "Unable to delete the custom exit-code file. "
+                      << "Exiting with an INTERNAL_ERROR.";
+      return blaze_exit_code::INTERNAL_ERROR;
+    }
+    int custom_exit_code;
+    if (!blaze_util::safe_strto32(content, &custom_exit_code)) {
+      BAZEL_LOG(INFO) << "Content of custom exit-code file not an int: "
+                      << content << "Exiting with an INTERNAL_ERROR.";
+      return blaze_exit_code::INTERNAL_ERROR;
+    }
+    BAZEL_LOG(INFO) << "Read exit code " << custom_exit_code
+                    << " from custom exit-code file. Exiting accordingly.";
+    return custom_exit_code;
+  }
+  return blaze_exit_code::INTERNAL_ERROR;
+}
+
 int Main(int argc, const char *argv[], WorkspaceLayout* workspace_layout,
          OptionProcessor *option_processor,
          std::unique_ptr<blaze_util::LogHandler> log_handler) {
diff --git a/src/main/cpp/blaze_abrupt_exit.cc b/src/main/cpp/blaze_abrupt_exit.cc
deleted file mode 100644
index e0b5b172e4..0000000000
--- a/src/main/cpp/blaze_abrupt_exit.cc
+++ /dev/null
@@ -1,23 +0,0 @@
-// Copyright 2016 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-#include "src/main/cpp/blaze_abrupt_exit.h"
-#include "src/main/cpp/util/exit_code.h"
-
-namespace blaze {
-
-int GetExitCodeForAbruptExit(const GlobalVariables& globals) {
-  return blaze_exit_code::INTERNAL_ERROR;
-}
-
-}  // namespace blaze
diff --git a/src/main/cpp/blaze_abrupt_exit.h b/src/main/cpp/blaze_abrupt_exit.h
deleted file mode 100644
index 0e9c22f2cb..0000000000
--- a/src/main/cpp/blaze_abrupt_exit.h
+++ /dev/null
@@ -1,30 +0,0 @@
-// Copyright 2016 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-//
-// blaze_abrupt_exit.h: Deals with abrupt exits of the Blaze server.
-//
-#ifndef BAZEL_SRC_MAIN_CPP_BLAZE_ABRUPT_EXIT_H_
-#define BAZEL_SRC_MAIN_CPP_BLAZE_ABRUPT_EXIT_H_
-
-namespace blaze {
-
-struct GlobalVariables;
-
-// Returns the exit code to use for when the Blaze server exits abruptly.
-int GetExitCodeForAbruptExit(const GlobalVariables& globals);
-
-}  // namespace blaze
-
-#endif  // BAZEL_SRC_MAIN_CPP_BLAZE_ABRUPT_EXIT_H_
diff --git a/src/main/cpp/startup_options.cc b/src/main/cpp/startup_options.cc
index d16eeef117..ea92cff9fa 100644
--- a/src/main/cpp/startup_options.cc
+++ b/src/main/cpp/startup_options.cc
@@ -55,7 +55,8 @@ StartupOptions::StartupOptions(const string &product_name,
       command_port(0),
       connect_timeout_secs(10),
       invocation_policy(NULL),
-      client_debug(false) {
+      client_debug(false),
+      use_custom_exit_code_on_abrupt_exit(true) {
   bool testing = !blaze::GetEnv("TEST_TMPDIR").empty();
   if (testing) {
     output_root = MakeAbsolute(blaze::GetEnv("TEST_TMPDIR"));
@@ -68,11 +69,20 @@ StartupOptions::StartupOptions(const string &product_name,
       output_root, "_" + product_name_lower + "_" + GetUserName());
   // 3 hours (but only 15 seconds if used within a test)
   max_idle_secs = testing ? 15 : (3 * 3600);
-  nullary_options = {"deep_execroot", "block_for_lock",
-      "host_jvm_debug", "master_blazerc", "master_bazelrc", "batch",
-      "batch_cpu_scheduling", "allow_configurable_attributes",
-      "fatal_event_bus_exceptions", "experimental_oom_more_eagerly",
-      "write_command_log", "watchfs", "client_debug"};
+  nullary_options = {"deep_execroot",
+                     "block_for_lock",
+                     "host_jvm_debug",
+                     "master_blazerc",
+                     "master_bazelrc",
+                     "batch",
+                     "batch_cpu_scheduling",
+                     "allow_configurable_attributes",
+                     "fatal_event_bus_exceptions",
+                     "experimental_oom_more_eagerly",
+                     "write_command_log",
+                     "watchfs",
+                     "client_debug",
+                     "use_custom_exit_code_on_abrupt_exit"};
   unary_options = {"output_base", "install_base",
       "output_user_root", "host_jvm_profile", "host_javabase",
       "host_jvm_args", "bazelrc", "blazerc", "io_nice_level",
@@ -266,6 +276,12 @@ blaze_exit_code::ExitCode StartupOptions::ProcessArg(
   } else if (GetNullaryOption(arg, "--noclient_debug")) {
     client_debug = false;
     option_sources["client_debug"] = rcfile;
+  } else if (GetNullaryOption(arg, "--use_custom_exit_code_on_abrupt_exit")) {
+    use_custom_exit_code_on_abrupt_exit = true;
+    option_sources["use_custom_exit_code_on_abrupt_exit"] = rcfile;
+  } else if (GetNullaryOption(arg, "--nouse_custom_exit_code_on_abrupt_exit")) {
+    use_custom_exit_code_on_abrupt_exit = false;
+    option_sources["use_custom_exit_code_on_abrupt_exit"] = rcfile;
   } else if ((value = GetUnaryOption(
       arg, next_arg, "--connect_timeout_secs")) != NULL) {
     if (!blaze_util::safe_strto32(value, &connect_timeout_secs) ||
diff --git a/src/main/cpp/startup_options.h b/src/main/cpp/startup_options.h
index 7020c04883..b36a70a52c 100644
--- a/src/main/cpp/startup_options.h
+++ b/src/main/cpp/startup_options.h
@@ -208,6 +208,10 @@ class StartupOptions {
   // Whether to output addition debugging information in the client.
   bool client_debug;
 
+  // Whether to check custom file for exit code when the Blaze Server exits
+  // abruptly without proper communication over gRPC.
+  bool use_custom_exit_code_on_abrupt_exit;
+
  protected:
   // Constructor for subclasses only so that site-specific extensions of this
   // class can override the product name.  The product_name must be the