Adds support for remote files.

PiperOrigin-RevId: 194413337

4 files changed, 122 insertions, 0 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/Metadata.java b/src/main/java/com/google/devtools/build/lib/actions/cache/Metadata.java
index c79241be82..b9a137d248 100644
--- a/src/main/java/com/google/devtools/build/lib/actions/cache/Metadata.java
+++ b/src/main/java/com/google/devtools/build/lib/actions/cache/Metadata.java
@@ -59,4 +59,14 @@ public interface Metadata {
    * and should be called.
    */
   long getModifiedTime();
+
+  /**
+   * Index used to resolve remote files.
+   *
+   * <p>0 indicates that no such information is available which can mean that it's either a local
+   * file or empty.
+   */
+  default int getLocationIndex() {
+    return 0;
+  }
 }
diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/MetadataHandler.java b/src/main/java/com/google/devtools/build/lib/actions/cache/MetadataHandler.java
index fc6c229d5a..5bda51aed7 100644
--- a/src/main/java/com/google/devtools/build/lib/actions/cache/MetadataHandler.java
+++ b/src/main/java/com/google/devtools/build/lib/actions/cache/MetadataHandler.java
@@ -58,6 +58,10 @@ public interface MetadataHandler {
    */
   void injectDigest(ActionInput output, FileStatus statNoFollow, byte[] digest);
 
+  /** Injects a file that is only stored remotely. */
+  void injectRemoteFile(
+      Artifact output, byte[] digest, long size, long modifiedTime, int locationIndex);
+
   /**
    * Marks an artifact as intentionally omitted. Acknowledges that this Artifact could have existed,
    * but was intentionally not saved, most likely as an optimization.
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ActionMetadataHandler.java b/src/main/java/com/google/devtools/build/lib/skyframe/ActionMetadataHandler.java
index c8098064aa..fe6e8afb61 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/ActionMetadataHandler.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ActionMetadataHandler.java
@@ -467,6 +467,50 @@ public class ActionMetadataHandler implements MetadataHandler {
   }
 
   @Override
+  public void injectRemoteFile(
+      Artifact output, byte[] digest, long size, long modifiedTime, int locationIndex) {
+    Preconditions.checkState(
+        executionMode.get(), "Tried to inject %s outside of execution.", output);
+    Preconditions.checkArgument(
+        locationIndex != 0 || size == 0,
+        "output = %s, size = %s, locationIndex =%s",
+        output,
+        size,
+        locationIndex);
+
+    Preconditions.checkState(injectedFiles.add(output), output);
+
+    // TODO(shahan): there are a couple of things that could reduce memory usage
+    // 1. We might be able to skip creating an entry in `outputArtifactData` and only create
+    // the `FileArtifactValue`, but there are likely downstream consumers that expect it that
+    // would need to be cleaned up.
+    // 2. Instead of creating an `additionalOutputData` entry, we could add the extra
+    // `locationIndex` to `FileStateValue`.
+    FileStateValue fileStateValue =
+        new FileStateValue.RegularFileStateValue(size, digest, /*contentsProxy=*/ null);
+    RootedPath rootedPath =
+        RootedPath.toRootedPath(output.getRoot().getRoot(), output.getRootRelativePath());
+    FileValue value = FileValue.value(rootedPath, fileStateValue, rootedPath, fileStateValue);
+    FileValue oldFsValue = outputArtifactData.putIfAbsent(output, value);
+    try {
+      checkInconsistentData(output, oldFsValue, value);
+    } catch (IOException e) {
+      // Should never happen.
+      throw new IllegalStateException("Inconsistent FileValues for " + output, e);
+    }
+
+    FileArtifactValue artifactValue =
+        new FileArtifactValue.RemoteFileArtifactValue(digest, size, modifiedTime, locationIndex);
+    FileArtifactValue oldArtifactValue = additionalOutputData.putIfAbsent(output, artifactValue);
+    try {
+      checkInconsistentData(output, oldArtifactValue, artifactValue);
+    } catch (IOException e) {
+      // Should never happen.
+      throw new IllegalStateException("Inconsistent FileArtifactValues for " + output, e);
+    }
+  }
+
+  @Override
   public void markOmitted(ActionInput output) {
     Preconditions.checkState(executionMode.get());
     if (output instanceof Artifact) {
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileArtifactValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileArtifactValue.java
index 8d171873ae..fbbdca186e 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/FileArtifactValue.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileArtifactValue.java
@@ -238,6 +238,70 @@ public abstract class FileArtifactValue implements SkyValue, Metadata {
     }
   }
 
+  static final class RemoteFileArtifactValue extends FileArtifactValue {
+    private final byte[] digest;
+    private final long size;
+    private final long modifiedTime;
+    private final int locationIndex;
+
+    RemoteFileArtifactValue(byte[] digest, long size, long modifiedTime, int locationIndex) {
+      this.digest = digest;
+      this.size = size;
+      this.modifiedTime = modifiedTime;
+      this.locationIndex = locationIndex;
+    }
+
+    @Override
+    public FileStateType getType() {
+      return FileStateType.REGULAR_FILE;
+    }
+
+    @Override
+    public byte[] getDigest() {
+      return digest;
+    }
+
+    @Override
+    public long getSize() {
+      return size;
+    }
+
+    @Override
+    public long getModifiedTime() {
+      return modifiedTime;
+    }
+
+    @Override
+    public int getLocationIndex() {
+      return locationIndex;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+      if (this == o) {
+        return true;
+      }
+      if (!(o instanceof RemoteFileArtifactValue)) {
+        return false;
+      }
+      RemoteFileArtifactValue r = (RemoteFileArtifactValue) o;
+      return Arrays.equals(digest, r.digest)
+          && size == r.size
+          && modifiedTime == r.modifiedTime
+          && locationIndex == r.locationIndex;
+    }
+
+    @Override
+    public int hashCode() {
+      return Objects.hash(Arrays.hashCode(digest), size, modifiedTime, locationIndex);
+    }
+
+    @Override
+    public boolean wasModifiedSinceDigest(Path path) {
+      throw new UnsupportedOperationException();
+    }
+  }
+
   static FileArtifactValue create(Artifact artifact, FileValue fileValue) throws IOException {
     boolean isFile = fileValue.isFile();
     FileContentsProxy proxy = getProxyFromFileStateValue(fileValue.realFileStateValue());