diff options
author | 2018-04-26 10:09:05 -0700 | |
---|---|---|
committer | 2018-04-26 10:10:49 -0700 | |
commit | e27b6bdd6f91efaae3d1b7b930e902b010aeefb6 (patch) | |
tree | 61123f8698d922d84528a6b689e33a535dbcb146 /src/main/java/com/google | |
parent | 67423ddc06a925ce80eb574adb8d7519ab39f348 (diff) |
Adds support for remote files.
PiperOrigin-RevId: 194413337
Diffstat (limited to 'src/main/java/com/google')
4 files changed, 122 insertions, 0 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/Metadata.java b/src/main/java/com/google/devtools/build/lib/actions/cache/Metadata.java index c79241be82..b9a137d248 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/cache/Metadata.java +++ b/src/main/java/com/google/devtools/build/lib/actions/cache/Metadata.java @@ -59,4 +59,14 @@ public interface Metadata { * and should be called. */ long getModifiedTime(); + + /** + * Index used to resolve remote files. + * + * <p>0 indicates that no such information is available which can mean that it's either a local + * file or empty. + */ + default int getLocationIndex() { + return 0; + } } diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/MetadataHandler.java b/src/main/java/com/google/devtools/build/lib/actions/cache/MetadataHandler.java index fc6c229d5a..5bda51aed7 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/cache/MetadataHandler.java +++ b/src/main/java/com/google/devtools/build/lib/actions/cache/MetadataHandler.java @@ -58,6 +58,10 @@ public interface MetadataHandler { */ void injectDigest(ActionInput output, FileStatus statNoFollow, byte[] digest); + /** Injects a file that is only stored remotely. */ + void injectRemoteFile( + Artifact output, byte[] digest, long size, long modifiedTime, int locationIndex); + /** * Marks an artifact as intentionally omitted. Acknowledges that this Artifact could have existed, * but was intentionally not saved, most likely as an optimization. diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ActionMetadataHandler.java b/src/main/java/com/google/devtools/build/lib/skyframe/ActionMetadataHandler.java index c8098064aa..fe6e8afb61 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/ActionMetadataHandler.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/ActionMetadataHandler.java @@ -467,6 +467,50 @@ public class ActionMetadataHandler implements MetadataHandler { } @Override + public void injectRemoteFile( + Artifact output, byte[] digest, long size, long modifiedTime, int locationIndex) { + Preconditions.checkState( + executionMode.get(), "Tried to inject %s outside of execution.", output); + Preconditions.checkArgument( + locationIndex != 0 || size == 0, + "output = %s, size = %s, locationIndex =%s", + output, + size, + locationIndex); + + Preconditions.checkState(injectedFiles.add(output), output); + + // TODO(shahan): there are a couple of things that could reduce memory usage + // 1. We might be able to skip creating an entry in `outputArtifactData` and only create + // the `FileArtifactValue`, but there are likely downstream consumers that expect it that + // would need to be cleaned up. + // 2. Instead of creating an `additionalOutputData` entry, we could add the extra + // `locationIndex` to `FileStateValue`. + FileStateValue fileStateValue = + new FileStateValue.RegularFileStateValue(size, digest, /*contentsProxy=*/ null); + RootedPath rootedPath = + RootedPath.toRootedPath(output.getRoot().getRoot(), output.getRootRelativePath()); + FileValue value = FileValue.value(rootedPath, fileStateValue, rootedPath, fileStateValue); + FileValue oldFsValue = outputArtifactData.putIfAbsent(output, value); + try { + checkInconsistentData(output, oldFsValue, value); + } catch (IOException e) { + // Should never happen. + throw new IllegalStateException("Inconsistent FileValues for " + output, e); + } + + FileArtifactValue artifactValue = + new FileArtifactValue.RemoteFileArtifactValue(digest, size, modifiedTime, locationIndex); + FileArtifactValue oldArtifactValue = additionalOutputData.putIfAbsent(output, artifactValue); + try { + checkInconsistentData(output, oldArtifactValue, artifactValue); + } catch (IOException e) { + // Should never happen. + throw new IllegalStateException("Inconsistent FileArtifactValues for " + output, e); + } + } + + @Override public void markOmitted(ActionInput output) { Preconditions.checkState(executionMode.get()); if (output instanceof Artifact) { diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileArtifactValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileArtifactValue.java index 8d171873ae..fbbdca186e 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/FileArtifactValue.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileArtifactValue.java @@ -238,6 +238,70 @@ public abstract class FileArtifactValue implements SkyValue, Metadata { } } + static final class RemoteFileArtifactValue extends FileArtifactValue { + private final byte[] digest; + private final long size; + private final long modifiedTime; + private final int locationIndex; + + RemoteFileArtifactValue(byte[] digest, long size, long modifiedTime, int locationIndex) { + this.digest = digest; + this.size = size; + this.modifiedTime = modifiedTime; + this.locationIndex = locationIndex; + } + + @Override + public FileStateType getType() { + return FileStateType.REGULAR_FILE; + } + + @Override + public byte[] getDigest() { + return digest; + } + + @Override + public long getSize() { + return size; + } + + @Override + public long getModifiedTime() { + return modifiedTime; + } + + @Override + public int getLocationIndex() { + return locationIndex; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof RemoteFileArtifactValue)) { + return false; + } + RemoteFileArtifactValue r = (RemoteFileArtifactValue) o; + return Arrays.equals(digest, r.digest) + && size == r.size + && modifiedTime == r.modifiedTime + && locationIndex == r.locationIndex; + } + + @Override + public int hashCode() { + return Objects.hash(Arrays.hashCode(digest), size, modifiedTime, locationIndex); + } + + @Override + public boolean wasModifiedSinceDigest(Path path) { + throw new UnsupportedOperationException(); + } + } + static FileArtifactValue create(Artifact artifact, FileValue fileValue) throws IOException { boolean isFile = fileValue.isFile(); FileContentsProxy proxy = getProxyFromFileStateValue(fileValue.realFileStateValue()); |