Remote worker: skip non-existent files after action execution

Tests basically always finish with non-existent files, and this is not an
error on the server side. Instead, the client (Bazel) checks whether all the
files it expects are present.

The test didn't catch this because it was silently falling back to local
execution.

Non-existent files were leading to an IOException, which caused the remote
worker to log nothing, and silently return an error with no output. Log
errors in the remote worker to make future debugging easier.

Fixes #2887.

PiperOrigin-RevId: 157400131

3 files changed, 32 insertions, 23 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/remote/SimpleBlobStoreActionCache.java b/src/main/java/com/google/devtools/build/lib/remote/SimpleBlobStoreActionCache.java
index 3c0e136101..bb1ca512d5 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/SimpleBlobStoreActionCache.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/SimpleBlobStoreActionCache.java
@@ -122,6 +122,9 @@ public final class SimpleBlobStoreActionCache implements RemoteActionCache {
   public void uploadAllResults(Path execRoot, Collection<Path> files, ActionResult.Builder result)
       throws IOException, InterruptedException {
     for (Path file : files) {
+      if (!file.exists()) {
+        continue;
+      }
       if (file.isDirectory()) {
         // TODO(olaola): to implement this for a directory, will need to create or pass a
         // TreeNodeRepository to call uploadTree.
diff --git a/src/test/shell/bazel/remote_execution_test.sh b/src/test/shell/bazel/remote_execution_test.sh
index 0cd5187e22..1a8748da02 100755
--- a/src/test/shell/bazel/remote_execution_test.sh
+++ b/src/test/shell/bazel/remote_execution_test.sh
@@ -78,13 +78,14 @@ EOF
 
   bazel clean --expunge
   bazel --host_jvm_args=-Dbazel.DigestFunction=SHA1 build \
-    --spawn_strategy=remote \
-    --remote_executor=localhost:${worker_port} \
-    --remote_cache=localhost:${worker_port} \
-        //a:test >& $TEST_log \
-    || fail "Failed to build //a:test with remote execution"
+      --spawn_strategy=remote \
+      --noremote_local_fallback \
+      --remote_executor=localhost:${worker_port} \
+      --remote_cache=localhost:${worker_port} \
+      //a:test >& $TEST_log \
+      || fail "Failed to build //a:test with remote execution"
   diff bazel-bin/a/test ${TEST_TMPDIR}/test_expected \
-    || fail "Remote execution generated different result"
+      || fail "Remote execution generated different result"
 }
 
 function test_cc_test() {
@@ -102,6 +103,7 @@ int main() { std::cout << "Hello test!" << std::endl; return 0; }
 EOF
   bazel --host_jvm_args=-Dbazel.DigestFunction=SHA1 test \
       --spawn_strategy=remote \
+      --noremote_local_fallback \
       --remote_executor=localhost:${worker_port} \
       --remote_cache=localhost:${worker_port} \
       --test_output=errors \
@@ -128,12 +130,12 @@ EOF
 
   bazel clean --expunge
   bazel --host_jvm_args=-Dbazel.DigestFunction=SHA1 build \
-    --spawn_strategy=remote \
-    --remote_cache=localhost:${worker_port} \
-        //a:test >& $TEST_log \
-    || fail "Failed to build //a:test with remote gRPC cache service"
+      --spawn_strategy=remote \
+      --remote_cache=localhost:${worker_port} \
+      //a:test >& $TEST_log \
+      || fail "Failed to build //a:test with remote gRPC cache service"
   diff bazel-bin/a/test ${TEST_TMPDIR}/test_expected \
-    || fail "Remote cache generated different result"
+      || fail "Remote cache generated different result"
 }
 
 # Tests that the remote worker can return a 200MB blob that requires chunking.
@@ -159,14 +161,15 @@ EOF
 
   bazel clean --expunge
   bazel --host_jvm_args=-Dbazel.DigestFunction=SHA1 build \
-    --spawn_strategy=remote \
-    --grpc_max_chunk_size_bytes=120000000 \
-    --remote_executor=localhost:${worker_port} \
-    --remote_cache=localhost:${worker_port} \
-        //a:large_output >& $TEST_log \
-    || fail "Failed to build //a:large_output with remote execution"
+      --spawn_strategy=remote \
+      --noremote_local_fallback \
+      --grpc_max_chunk_size_bytes=120000000 \
+      --remote_executor=localhost:${worker_port} \
+      --remote_cache=localhost:${worker_port} \
+      //a:large_output >& $TEST_log \
+      || fail "Failed to build //a:large_output with remote execution"
   diff bazel-genfiles/a/large_blob.txt ${TEST_TMPDIR}/large_blob_expected.txt \
-    || fail "Remote execution generated different result"
+      || fail "Remote execution generated different result"
 }
 
 function test_cc_binary_rest_cache() {
@@ -188,12 +191,13 @@ EOF
 
   bazel clean --expunge
   bazel --host_jvm_args=-Dbazel.DigestFunction=SHA1 build \
-    --spawn_strategy=remote \
-    --remote_rest_cache=http://localhost:${hazelcast_port}/hazelcast/rest/maps/cache \
-        //a:test >& $TEST_log \
-    || fail "Failed to build //a:test with remote gRPC cache service"
+      --spawn_strategy=remote \
+      --noremote_local_fallback \
+      --remote_rest_cache=http://localhost:${hazelcast_port}/hazelcast/rest/maps/cache \
+      //a:test >& $TEST_log \
+      || fail "Failed to build //a:test with remote gRPC cache service"
   diff bazel-bin/a/test ${TEST_TMPDIR}/test_expected \
-    || fail "Remote cache generated different result"
+      || fail "Remote cache generated different result"
 }
 
 function test_py_test() {
@@ -212,6 +216,7 @@ if __name__ == "__main__":
 EOF
   bazel --host_jvm_args=-Dbazel.DigestFunction=SHA1 test \
       --spawn_strategy=remote \
+      --noremote_local_fallback \
       --remote_executor=localhost:${worker_port} \
       --remote_cache=localhost:${worker_port} \
       --test_output=errors \
diff --git a/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/RemoteWorker.java b/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/RemoteWorker.java
index f705b4ef54..ce7d5c4d2c 100644
--- a/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/RemoteWorker.java
+++ b/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/RemoteWorker.java
@@ -588,6 +588,7 @@ public class RemoteWorker {
           LOG.warning("Preserving work directory " + tempRoot.toString() + ".");
         }
       } catch (IOException | InterruptedException e) {
+        LOG.log(Level.SEVERE, "Failure", e);
         ExecuteReply.Builder reply = ExecuteReply.newBuilder();
         reply.getStatusBuilder().setSucceeded(false).setErrorDetail(e.toString());
         responseObserver.onNext(reply.build());