workers: Pass a map of input filenames -> digest of file contents to workers so that they can cache and reuse state for unchanged inputs over multiple builds.

--
MOS_MIGRATED_REVID=107066408

6 files changed, 112 insertions, 6 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/worker/BUILD b/src/main/java/com/google/devtools/build/lib/worker/BUILD
index 305a8b7adb..4bb1630542 100644
--- a/src/main/java/com/google/devtools/build/lib/worker/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/worker/BUILD
@@ -24,6 +24,7 @@ java_library(
         "//third_party:apache_commons_pool2",
         "//third_party:guava",
         "//third_party:jsr305",
+        "//third_party:protobuf",
     ],
 )
 
diff --git a/src/main/java/com/google/devtools/build/lib/worker/WorkerSpawnStrategy.java b/src/main/java/com/google/devtools/build/lib/worker/WorkerSpawnStrategy.java
index 397e08d06b..36ef272720 100644
--- a/src/main/java/com/google/devtools/build/lib/worker/WorkerSpawnStrategy.java
+++ b/src/main/java/com/google/devtools/build/lib/worker/WorkerSpawnStrategy.java
@@ -27,6 +27,7 @@ import com.google.common.hash.Hashing;
 import com.google.devtools.build.lib.actions.ActionExecutionContext;
 import com.google.devtools.build.lib.actions.ActionInput;
 import com.google.devtools.build.lib.actions.ActionInputFileCache;
+import com.google.devtools.build.lib.actions.ActionInputHelper;
 import com.google.devtools.build.lib.actions.ChangedFilesMessage;
 import com.google.devtools.build.lib.actions.ExecException;
 import com.google.devtools.build.lib.actions.ExecutionStrategy;
@@ -46,11 +47,13 @@ import com.google.devtools.build.lib.vfs.Path;
 import com.google.devtools.build.lib.worker.WorkerProtocol.WorkRequest;
 import com.google.devtools.build.lib.worker.WorkerProtocol.WorkResponse;
 import com.google.devtools.common.options.OptionsClassProvider;
+import com.google.protobuf.ByteString;
 
 import java.io.IOException;
 import java.nio.charset.Charset;
 import java.nio.file.Files;
 import java.nio.file.Paths;
+import java.util.List;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
@@ -143,16 +146,32 @@ final class WorkerSpawnStrategy implements SpawnActionContext {
     Path workDir = actionExecutionContext.getExecutor().getExecRoot();
 
     try {
-      HashCode workerFilesHash =
-          combineActionInputHashes(
-              spawn.getToolFiles(), actionExecutionContext.getActionInputFileCache());
+      ActionInputFileCache inputFileCache = actionExecutionContext.getActionInputFileCache();
+
+      HashCode workerFilesHash = combineActionInputHashes(spawn.getToolFiles(), inputFileCache);
       WorkerKey key = new WorkerKey(args, env, workDir, spawn.getMnemonic(), workerFilesHash);
 
       WorkRequest.Builder requestBuilder = WorkRequest.newBuilder();
       expandArgument(requestBuilder, Iterables.getLast(spawn.getArguments()));
 
-      WorkResponse response =
-          execInWorker(executor.getEventHandler(), key, requestBuilder.build(), maxRetries);
+      List<ActionInput> inputs =
+          ActionInputHelper.expandMiddlemen(
+              spawn.getInputFiles(), actionExecutionContext.getMiddlemanExpander());
+
+      for (ActionInput input : inputs) {
+        ByteString digest = inputFileCache.getDigest(input);
+        if (digest == null) {
+          digest = ByteString.EMPTY;
+        }
+
+        requestBuilder
+            .addInputsBuilder()
+            .setPath(input.getExecPathString())
+            .setDigest(digest)
+            .build();
+      }
+
+      WorkResponse response = execInWorker(eventHandler, key, requestBuilder.build(), maxRetries);
 
       outErr.getErrorStream().write(response.getOutputBytes().toByteArray());
 
diff --git a/src/main/protobuf/worker_protocol.proto b/src/main/protobuf/worker_protocol.proto
index 66926cce44..4706792f4e 100644
--- a/src/main/protobuf/worker_protocol.proto
+++ b/src/main/protobuf/worker_protocol.proto
@@ -18,14 +18,32 @@ package blaze.worker;
 
 option java_package = "com.google.devtools.build.lib.worker";
 
+// An input file.
+message Input {
+  // The path in the file system where to read this input artifact from. This is
+  // either a path relative to the execution root (the worker process is
+  // launched with the working directory set to the execution root), or an
+  // absolute path.
+  string path = 1;
+
+  // A hash-value of the contents. The format of the contents is unspecified and
+  // the digest should be treated as an opaque token.
+  bytes digest = 2;
+}
+
 // This represents a single work unit that Blaze sends to the worker.
 message WorkRequest {
   repeated string arguments = 1;
+
+  // The inputs that the worker is allowed to read during execution of this
+  // request.
+  repeated Input inputs = 2;
 }
 
 // The worker sends this message to Blaze when it finished its work on the WorkRequest message.
 message WorkResponse {
   int32 exit_code = 1;
+
   // This is printed to the user after the WorkResponse has been received and is supposed to contain
   // compiler warnings / errors etc. - thus we'll use a string type here, which gives us UTF-8
   // encoding.
diff --git a/src/test/java/com/google/devtools/build/lib/worker/ExampleWorker.java b/src/test/java/com/google/devtools/build/lib/worker/ExampleWorker.java
index 78c03ff012..21c089c088 100644
--- a/src/test/java/com/google/devtools/build/lib/worker/ExampleWorker.java
+++ b/src/test/java/com/google/devtools/build/lib/worker/ExampleWorker.java
@@ -20,6 +20,7 @@ import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.devtools.build.lib.worker.ExampleWorkerOptions.ExampleWorkOptions;
+import com.google.devtools.build.lib.worker.WorkerProtocol.Input;
 import com.google.devtools.build.lib.worker.WorkerProtocol.WorkRequest;
 import com.google.devtools.build.lib.worker.WorkerProtocol.WorkResponse;
 import com.google.devtools.common.options.OptionsParser;
@@ -30,7 +31,9 @@ import java.io.PrintStream;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.util.ArrayList;
+import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map.Entry;
 import java.util.UUID;
 
 /**
@@ -47,6 +50,9 @@ public class ExampleWorker {
   // If true, returns corrupt responses instead of correct protobufs.
   static boolean poisoned = false;
 
+  // Keep state across multiple builds.
+  static final LinkedHashMap<String, String> inputs = new LinkedHashMap<>();
+
   public static void main(String[] args) throws Exception {
     if (ImmutableSet.copyOf(args).contains("--persistent_worker")) {
       OptionsParser parser = OptionsParser.newOptionsParser(ExampleWorkerOptions.class);
@@ -73,6 +79,11 @@ public class ExampleWorker {
           break;
         }
 
+        inputs.clear();
+        for (Input input : request.getInputsList()) {
+          inputs.put(input.getPath(), input.getDigest().toStringUtf8());
+        }
+
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
         int exitCode = 0;
 
@@ -142,6 +153,12 @@ public class ExampleWorker {
     }
     outputs.add(residueStr);
 
+    if (options.printInputs) {
+      for (Entry<String, String> input : inputs.entrySet()) {
+        outputs.add("INPUT " + input.getKey() + " " + input.getValue());
+      }
+    }
+
     String outputStr = Joiner.on('\n').join(outputs);
     if (options.outputFile.isEmpty()) {
       System.out.println(outputStr);
diff --git a/src/test/java/com/google/devtools/build/lib/worker/ExampleWorkerOptions.java b/src/test/java/com/google/devtools/build/lib/worker/ExampleWorkerOptions.java
index 6a159ae512..7456e4b331 100644
--- a/src/test/java/com/google/devtools/build/lib/worker/ExampleWorkerOptions.java
+++ b/src/test/java/com/google/devtools/build/lib/worker/ExampleWorkerOptions.java
@@ -44,6 +44,13 @@ public class ExampleWorkerOptions extends OptionsBase {
       help = "Writes a counter that increases with each work unit processed into the output."
     )
     public boolean writeCounter;
+
+    @Option(
+      name = "print_inputs",
+      defaultValue = "false",
+      help = "Writes a list of input files and their digests."
+    )
+    public boolean printInputs;
   }
 
   @Option(name = "persistent_worker", defaultValue = "false")
diff --git a/src/test/shell/bazel/bazel_worker_test.sh b/src/test/shell/bazel/bazel_worker_test.sh
index 80b6ac203c..17d819a588 100755
--- a/src/test/shell/bazel/bazel_worker_test.sh
+++ b/src/test/shell/bazel/bazel_worker_test.sh
@@ -158,7 +158,7 @@ def _impl(ctx):
   ctx.file_action(output=argfile, content=argfile_contents)
 
   ctx.action(
-      inputs=[argfile],
+      inputs=[argfile] + ctx.files.srcs,
       outputs=[output],
       executable=worker,
       progress_message="Working on %s" % ctx.label.name,
@@ -172,6 +172,7 @@ work = rule(
         "worker": attr.label(cfg=HOST_CFG, mandatory=True, allow_files=True, executable=True),
         "worker_args": attr.string_list(),
         "args": attr.string_list(),
+        "srcs": attr.label_list(allow_files=True),
     },
     outputs = {"out": "%{name}.out"},
 )
@@ -394,4 +395,47 @@ EOF
   assert_not_equals "$worker_uuid_1" "$worker_uuid_2"
 }
 
+function test_input_digests() {
+  prepare_example_worker
+
+  cat >>BUILD <<'EOF'
+[work(
+  name = "hello_world_%s" % idx,
+  worker = ":worker",
+  args = ["--write_uuid", "--print_inputs"],
+  srcs = [":input.txt"],
+) for idx in range(10)]
+EOF
+
+  bazel --batch clean
+  assert_workers_not_running
+
+  echo "hello world" > input.txt
+  bazel build --strategy=Work=worker --worker_max_instances=1 :hello_world_1 \
+    || fail "build failed"
+  worker_uuid_1=$(cat bazel-bin/hello_world_1.out | grep UUID | cut -d' ' -f2)
+  hash1=$(fgrep "INPUT input.txt " bazel-bin/hello_world_1.out | cut -d' ' -f3)
+  assert_workers_running
+
+  bazel build --strategy=Work=worker --worker_max_instances=1 :hello_world_2 \
+    || fail "build failed"
+  worker_uuid_2=$(cat bazel-bin/hello_world_2.out | grep UUID | cut -d' ' -f2)
+  hash2=$(fgrep "INPUT input.txt " bazel-bin/hello_world_2.out | cut -d' ' -f3)
+  assert_workers_running
+
+  assert_equals "$worker_uuid_1" "$worker_uuid_2"
+  assert_equals "$hash1" "$hash2"
+
+  echo "changeddata" > input.txt
+
+  bazel build --strategy=Work=worker --worker_max_instances=1 :hello_world_3 \
+    || fail "build failed"
+  worker_uuid_3=$(cat bazel-bin/hello_world_3.out | grep UUID | cut -d' ' -f2)
+  hash3=$(fgrep "INPUT input.txt " bazel-bin/hello_world_3.out | cut -d' ' -f3)
+  assert_workers_running
+
+  assert_equals "$worker_uuid_2" "$worker_uuid_3"
+  assert_not_equals "$hash2" "$hash3"
+}
+
 run_suite "Worker integration tests"