From e27b6bdd6f91efaae3d1b7b930e902b010aeefb6 Mon Sep 17 00:00:00 2001 From: shahan Date: Thu, 26 Apr 2018 10:09:05 -0700 Subject: Adds support for remote files. PiperOrigin-RevId: 194413337 --- .../devtools/build/lib/actions/cache/Metadata.java | 10 ++++ .../build/lib/actions/cache/MetadataHandler.java | 4 ++ .../build/lib/skyframe/ActionMetadataHandler.java | 44 +++++++++++++++ .../build/lib/skyframe/FileArtifactValue.java | 64 ++++++++++++++++++++++ 4 files changed, 122 insertions(+) (limited to 'src/main/java/com/google/devtools/build') diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/Metadata.java b/src/main/java/com/google/devtools/build/lib/actions/cache/Metadata.java index c79241be82..b9a137d248 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/cache/Metadata.java +++ b/src/main/java/com/google/devtools/build/lib/actions/cache/Metadata.java @@ -59,4 +59,14 @@ public interface Metadata { * and should be called. */ long getModifiedTime(); + + /** + * Index used to resolve remote files. + * + *

0 indicates that no such information is available which can mean that it's either a local + * file or empty. + */ + default int getLocationIndex() { + return 0; + } } diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/MetadataHandler.java b/src/main/java/com/google/devtools/build/lib/actions/cache/MetadataHandler.java index fc6c229d5a..5bda51aed7 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/cache/MetadataHandler.java +++ b/src/main/java/com/google/devtools/build/lib/actions/cache/MetadataHandler.java @@ -58,6 +58,10 @@ public interface MetadataHandler { */ void injectDigest(ActionInput output, FileStatus statNoFollow, byte[] digest); + /** Injects a file that is only stored remotely. */ + void injectRemoteFile( + Artifact output, byte[] digest, long size, long modifiedTime, int locationIndex); + /** * Marks an artifact as intentionally omitted. Acknowledges that this Artifact could have existed, * but was intentionally not saved, most likely as an optimization. diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ActionMetadataHandler.java b/src/main/java/com/google/devtools/build/lib/skyframe/ActionMetadataHandler.java index c8098064aa..fe6e8afb61 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/ActionMetadataHandler.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/ActionMetadataHandler.java @@ -466,6 +466,50 @@ public class ActionMetadataHandler implements MetadataHandler { } } + @Override + public void injectRemoteFile( + Artifact output, byte[] digest, long size, long modifiedTime, int locationIndex) { + Preconditions.checkState( + executionMode.get(), "Tried to inject %s outside of execution.", output); + Preconditions.checkArgument( + locationIndex != 0 || size == 0, + "output = %s, size = %s, locationIndex =%s", + output, + size, + locationIndex); + + Preconditions.checkState(injectedFiles.add(output), output); + + // TODO(shahan): there are a couple of things that could reduce memory usage + // 1. We might be able to skip creating an entry in `outputArtifactData` and only create + // the `FileArtifactValue`, but there are likely downstream consumers that expect it that + // would need to be cleaned up. + // 2. Instead of creating an `additionalOutputData` entry, we could add the extra + // `locationIndex` to `FileStateValue`. + FileStateValue fileStateValue = + new FileStateValue.RegularFileStateValue(size, digest, /*contentsProxy=*/ null); + RootedPath rootedPath = + RootedPath.toRootedPath(output.getRoot().getRoot(), output.getRootRelativePath()); + FileValue value = FileValue.value(rootedPath, fileStateValue, rootedPath, fileStateValue); + FileValue oldFsValue = outputArtifactData.putIfAbsent(output, value); + try { + checkInconsistentData(output, oldFsValue, value); + } catch (IOException e) { + // Should never happen. + throw new IllegalStateException("Inconsistent FileValues for " + output, e); + } + + FileArtifactValue artifactValue = + new FileArtifactValue.RemoteFileArtifactValue(digest, size, modifiedTime, locationIndex); + FileArtifactValue oldArtifactValue = additionalOutputData.putIfAbsent(output, artifactValue); + try { + checkInconsistentData(output, oldArtifactValue, artifactValue); + } catch (IOException e) { + // Should never happen. + throw new IllegalStateException("Inconsistent FileArtifactValues for " + output, e); + } + } + @Override public void markOmitted(ActionInput output) { Preconditions.checkState(executionMode.get()); diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileArtifactValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileArtifactValue.java index 8d171873ae..fbbdca186e 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/FileArtifactValue.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileArtifactValue.java @@ -238,6 +238,70 @@ public abstract class FileArtifactValue implements SkyValue, Metadata { } } + static final class RemoteFileArtifactValue extends FileArtifactValue { + private final byte[] digest; + private final long size; + private final long modifiedTime; + private final int locationIndex; + + RemoteFileArtifactValue(byte[] digest, long size, long modifiedTime, int locationIndex) { + this.digest = digest; + this.size = size; + this.modifiedTime = modifiedTime; + this.locationIndex = locationIndex; + } + + @Override + public FileStateType getType() { + return FileStateType.REGULAR_FILE; + } + + @Override + public byte[] getDigest() { + return digest; + } + + @Override + public long getSize() { + return size; + } + + @Override + public long getModifiedTime() { + return modifiedTime; + } + + @Override + public int getLocationIndex() { + return locationIndex; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof RemoteFileArtifactValue)) { + return false; + } + RemoteFileArtifactValue r = (RemoteFileArtifactValue) o; + return Arrays.equals(digest, r.digest) + && size == r.size + && modifiedTime == r.modifiedTime + && locationIndex == r.locationIndex; + } + + @Override + public int hashCode() { + return Objects.hash(Arrays.hashCode(digest), size, modifiedTime, locationIndex); + } + + @Override + public boolean wasModifiedSinceDigest(Path path) { + throw new UnsupportedOperationException(); + } + } + static FileArtifactValue create(Artifact artifact, FileValue fileValue) throws IOException { boolean isFile = fileValue.isFile(); FileContentsProxy proxy = getProxyFromFileStateValue(fileValue.realFileStateValue()); -- cgit v1.2.3