diff options
Diffstat (limited to 'src/main/java')
9 files changed, 154 insertions, 70 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/actions/FileArtifactValue.java b/src/main/java/com/google/devtools/build/lib/actions/FileArtifactValue.java index 83da23a0fc..cee7110eb9 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/FileArtifactValue.java +++ b/src/main/java/com/google/devtools/build/lib/actions/FileArtifactValue.java @@ -16,6 +16,7 @@ package com.google.devtools.build.lib.actions; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; +import com.google.common.hash.Hashing; import com.google.common.io.BaseEncoding; import com.google.devtools.build.lib.actions.cache.DigestUtils; import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; @@ -399,6 +400,10 @@ public abstract class FileArtifactValue implements SkyValue { this.digest = Preconditions.checkNotNull(digest); } + public InlineFileArtifactValue(byte[] bytes) { + this(bytes, Hashing.md5().hashBytes(bytes).asBytes()); + } + public ByteArrayInputStream getInputStream() { return new ByteArrayInputStream(data); } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ActionMetadataHandler.java b/src/main/java/com/google/devtools/build/lib/skyframe/ActionMetadataHandler.java index 49159f401b..0efed68fba 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/ActionMetadataHandler.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/ActionMetadataHandler.java @@ -356,7 +356,7 @@ public class ActionMetadataHandler implements MetadataHandler { // should be single threaded and there should be no race condition. // The current design of ActionMetadataHandler makes this hard to enforce. Set<PathFragment> paths = null; - paths = TreeArtifactValue.explodeDirectory(artifact); + paths = TreeArtifactValue.explodeDirectory(artifactPathResolver.toPath(artifact)); Set<TreeFileArtifact> diskFiles = ActionInputHelper.asTreeFileArtifacts(artifact, paths); if (!diskFiles.equals(registeredContents)) { // There might be more than one error here. We first look for missing output files. @@ -436,8 +436,8 @@ public class ActionMetadataHandler implements MetadataHandler { return TreeArtifactValue.MISSING_TREE_ARTIFACT; } - Set<PathFragment> paths = null; - paths = TreeArtifactValue.explodeDirectory(artifact); + Set<PathFragment> paths = + TreeArtifactValue.explodeDirectory(artifactPathResolver.toPath(artifact)); // If you're reading tree artifacts from disk while outputDirectoryListings are being injected, // something has gone terribly wrong. Object previousDirectoryListing = diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FilesystemValueChecker.java b/src/main/java/com/google/devtools/build/lib/skyframe/FilesystemValueChecker.java index b501f3abce..31fa632225 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/FilesystemValueChecker.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/FilesystemValueChecker.java @@ -394,7 +394,8 @@ public class FilesystemValueChecker { // There doesn't appear to be any facility to batch list directories... we must // do things the 'slow' way. try { - Set<PathFragment> currentDirectoryValue = TreeArtifactValue.explodeDirectory(artifact); + Set<PathFragment> currentDirectoryValue = + TreeArtifactValue.explodeDirectory(artifact.getPath()); Set<PathFragment> valuePaths = value.getChildPaths(); return !currentDirectoryValue.equals(valuePaths); } catch (IOException e) { diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java index e2cc9df529..98b5c3cd90 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java @@ -13,8 +13,6 @@ // limitations under the License. package com.google.devtools.build.lib.skyframe; -import static com.google.devtools.build.lib.vfs.FileSystemUtils.createDirectoryAndParents; - import com.google.common.base.Preconditions; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; @@ -789,20 +787,21 @@ public final class SkyframeActionExecutor { } } - private void createOutputDirectories(Action action) throws ActionExecutionException { + private void createOutputDirectories(Action action, ActionExecutionContext context) + throws ActionExecutionException { try { Set<Path> done = new HashSet<>(); // avoid redundant calls for the same directory. for (Artifact outputFile : action.getOutputs()) { Path outputDir; if (outputFile.isTreeArtifact()) { - outputDir = outputFile.getPath(); + outputDir = context.getPathResolver().toPath(outputFile); } else { - outputDir = outputFile.getPath().getParentDirectory(); + outputDir = context.getPathResolver().toPath(outputFile).getParentDirectory(); } if (done.add(outputDir)) { try { - createDirectoryAndParents(outputDir); + outputDir.createDirectoryAndParents(); continue; } catch (IOException e) { /* Fall through to plan B. */ @@ -855,7 +854,7 @@ public final class SkyframeActionExecutor { p = p.getParentDirectory(); } - createDirectoryAndParents(outputDir); + outputDir.createDirectoryAndParents(); } catch (IOException e) { throw new ActionExecutionException( "failed to create output directory '" + outputDir + "'", e, action, false); @@ -896,17 +895,25 @@ public final class SkyframeActionExecutor { long actionStartTime, ActionLookupData actionLookupData) throws ActionExecutionException, InterruptedException { - // ActionFileSystem constructs directories implicitly, so no need to delete the old outputs - // and ensure directories exist in this case. - if (!usesActionFileSystem()) { - // Delete the outputs before executing the action, just to ensure that - // the action really does produce the outputs. - try { + // Delete the outputs before executing the action, just to ensure that + // the action really does produce the outputs. + try { + if (!usesActionFileSystem()) { action.prepare(context.getFileSystem(), context.getExecRoot()); - createOutputDirectories(action); - } catch (IOException e) { - reportError("failed to delete output files before executing action", e, action, null); + } else { + try { + context.getFileOutErr().getOutputPath().getParentDirectory().createDirectoryAndParents(); + context.getFileOutErr().getErrorPath().getParentDirectory().createDirectoryAndParents(); + } catch (IOException e) { + throw new ActionExecutionException( + "failed to create output directory for output streams'" + + context.getFileOutErr().getErrorPath() + "'", + e, action, false); + } } + createOutputDirectories(action, context); + } catch (IOException e) { + reportError("failed to delete output files before executing action", e, action, null); } eventHandler.post(new ActionStartedEvent(action, actionStartTime)); diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TreeArtifactValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/TreeArtifactValue.java index b7ca9088e5..13ff08a65f 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/TreeArtifactValue.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/TreeArtifactValue.java @@ -181,13 +181,13 @@ class TreeArtifactValue implements SkyValue { } }; - private static void explodeDirectory(Artifact treeArtifact, + private static void explodeDirectory(Path treeArtifactPath, PathFragment pathToExplode, ImmutableSet.Builder<PathFragment> valuesBuilder) throws IOException { - for (Path subpath : treeArtifact.getPath().getRelative(pathToExplode).getDirectoryEntries()) { + for (Path subpath : treeArtifactPath.getRelative(pathToExplode).getDirectoryEntries()) { PathFragment canonicalSubpathFragment = pathToExplode.getChild(subpath.getBaseName()); if (subpath.isDirectory()) { - explodeDirectory(treeArtifact, + explodeDirectory(treeArtifactPath, pathToExplode.getChild(subpath.getBaseName()), valuesBuilder); } else if (subpath.isSymbolicLink()) { PathFragment linkTarget = subpath.readSymbolicLinkUnchecked(); @@ -229,9 +229,9 @@ class TreeArtifactValue implements SkyValue { * @throws IOException if there is any problem reading or validating outputs under the given * tree artifact. */ - static Set<PathFragment> explodeDirectory(Artifact treeArtifact) throws IOException { + static Set<PathFragment> explodeDirectory(Path treeArtifactPath) throws IOException { ImmutableSet.Builder<PathFragment> explodedDirectory = ImmutableSet.builder(); - explodeDirectory(treeArtifact, PathFragment.EMPTY_FRAGMENT, explodedDirectory); + explodeDirectory(treeArtifactPath, PathFragment.EMPTY_FRAGMENT, explodedDirectory); return explodedDirectory.build(); } } diff --git a/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/FileInfo.java b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/FileInfo.java index db2660cd26..4ac878aa56 100644 --- a/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/FileInfo.java +++ b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/FileInfo.java @@ -16,6 +16,7 @@ package com.google.devtools.build.lib.vfs.inmemoryfs; import com.google.devtools.build.lib.clock.Clock; import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; /** @@ -47,7 +48,11 @@ public abstract class FileInfo extends InMemoryContentInfo { return false; } - protected abstract byte[] readContent() throws IOException; + public abstract OutputStream getOutputStream(boolean append) throws IOException; - protected abstract OutputStream getOutputStream(boolean append) throws IOException; + public abstract InputStream getInputStream() throws IOException; + + public abstract byte[] getxattr(String name) throws IOException; + + public abstract byte[] getFastDigest() throws IOException; } diff --git a/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryContentInfo.java b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryContentInfo.java index 93c4eb294c..155e9762c7 100644 --- a/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryContentInfo.java +++ b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryContentInfo.java @@ -31,7 +31,7 @@ import java.io.IOException; @ThreadSafe public abstract class InMemoryContentInfo implements FileStatus { - private final Clock clock; + protected final Clock clock; /** * Stores the time when the file was last modified. This is atomically updated @@ -95,7 +95,7 @@ public abstract class InMemoryContentInfo implements FileStatus { * size of links is unspecified. */ @Override - public abstract long getSize() throws IOException; + public abstract long getSize(); /** * Returns the time when the entity denoted by the current object was last diff --git a/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryFileInfo.java b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryFileInfo.java index 49777190f7..d73c4cc24d 100644 --- a/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryFileInfo.java +++ b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryFileInfo.java @@ -14,10 +14,13 @@ package com.google.devtools.build.lib.vfs.inmemoryfs; import com.google.common.base.Preconditions; +import com.google.common.io.ByteStreams; import com.google.devtools.build.lib.clock.Clock; import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; /** @@ -32,7 +35,7 @@ public class InMemoryFileInfo extends FileInfo { */ protected byte[] content; - protected InMemoryFileInfo(Clock clock) { + public InMemoryFileInfo(Clock clock) { super(clock); content = new byte[0]; // New files start out empty. } @@ -43,8 +46,13 @@ public class InMemoryFileInfo extends FileInfo { } @Override - public synchronized byte[] readContent() { - return content.clone(); + public byte[] getxattr(String name) { + return null; + } + + @Override + public byte[] getFastDigest() { + return null; } private synchronized void setContent(byte[] newContent) { @@ -53,44 +61,69 @@ public class InMemoryFileInfo extends FileInfo { } @Override - protected synchronized OutputStream getOutputStream(boolean append) - throws IOException { - OutputStream out = new ByteArrayOutputStream() { - private boolean closed = false; - - @Override - public void write(byte[] data) throws IOException { - Preconditions.checkState(!closed); - super.write(data); - } + public synchronized InputStream getInputStream() { + return new ByteArrayInputStream(content); + } - @Override - public synchronized void write(int dataByte) { - Preconditions.checkState(!closed); - super.write(dataByte); + @Override + public synchronized OutputStream getOutputStream(boolean append) { + OutputStream out = new InMemoryOutputStream(this::setContent); + if (append) { + try (InputStream in = getInputStream()) { + ByteStreams.copy(in, out); + } catch (IOException e) { + throw new IllegalStateException(e); } + } + return out; + } - @Override - public synchronized void write(byte[] data, int offset, int length) { - Preconditions.checkState(!closed); - super.write(data, offset, length); - } + /** + * A {@link ByteArrayOutputStream} which notifiers a callback when it has flushed its data. + */ + public static class InMemoryOutputStream extends ByteArrayOutputStream { + private final IOByteReceiver receiver; + private boolean closed = false; - @Override - public void close() { - flush(); - closed = true; - } + public InMemoryOutputStream(IOByteReceiver receiver) { + this.receiver = receiver; + } - @Override - public void flush() { - setContent(toByteArray().clone()); - } - }; + @Override + public void write(byte[] data) throws IOException { + Preconditions.checkState(!closed); + super.write(data); + } - if (append) { - out.write(readContent()); + @Override + public synchronized void write(int dataByte) { + Preconditions.checkState(!closed); + super.write(dataByte); } - return out; + + @Override + public synchronized void write(byte[] data, int offset, int length) { + Preconditions.checkState(!closed); + super.write(data, offset, length); + } + + @Override + public synchronized void close() throws IOException { + flush(); + closed = true; + } + + @Override + public synchronized void flush() throws IOException { + receiver.accept(toByteArray().clone()); + } + } + + /** + * Similar to {@link com.google.common.base.Receiver}, but allows implementations to throw + * {@link IOException}. + */ + public interface IOByteReceiver { + void accept(byte[] bytes) throws IOException; } } diff --git a/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryFileSystem.java b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryFileSystem.java index bebe5b380c..f24a3d99a8 100644 --- a/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryFileSystem.java +++ b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryFileSystem.java @@ -25,7 +25,6 @@ import com.google.devtools.build.lib.vfs.FileStatus; import com.google.devtools.build.lib.vfs.FileSystem; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; -import java.io.ByteArrayInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; @@ -51,7 +50,7 @@ import javax.annotation.Nullable; public class InMemoryFileSystem extends FileSystem { private final PathFragment scopeRoot; - private final Clock clock; + protected final Clock clock; // The root inode (a directory). private final InMemoryDirectoryInfo rootInode; @@ -319,13 +318,17 @@ public class InMemoryFileSystem extends FileSystem { if (!create) { throw Error.ENOENT.exception(path); } else { - child = new InMemoryFileInfo(clock); + child = newFile(clock, path); insert(imdi, name, child, path); } } return child; } + protected FileInfo newFile(Clock clock, Path path) { + return new InMemoryFileInfo(clock); + } + /** * Low-level path-to-inode lookup routine. Analogous to path_walk() in many UNIX kernels. Given * 'path', walks the directory tree from the root, resolving all symbolic links, and returns the @@ -462,7 +465,7 @@ public class InMemoryFileSystem extends FileSystem { * Version of stat that returns an inode if the input path stays entirely within this file * system's scope, otherwise throws. */ - private InMemoryContentInfo scopeLimitedStat(Path path, boolean followSymlinks) + protected InMemoryContentInfo scopeLimitedStat(Path path, boolean followSymlinks) throws IOException { if (followSymlinks) { return pathWalk(path, false); @@ -747,12 +750,42 @@ public class InMemoryFileSystem extends FileSystem { throw Error.EACCES.exception(path); } Preconditions.checkState(status instanceof FileInfo); - return new ByteArrayInputStream(((FileInfo) status).readContent()); + return ((FileInfo) status).getInputStream(); + } + } + + @Override + public byte[] getxattr(Path path, String name) throws IOException { + synchronized (this) { + InMemoryContentInfo status = scopeLimitedStat(path, true); + if (status.isDirectory()) { + throw Error.EISDIR.exception(path); + } + if (!path.isReadable()) { + throw Error.EACCES.exception(path); + } + Preconditions.checkState(status instanceof FileInfo); + return ((FileInfo) status).getxattr(name); + } + } + + @Override + protected byte[] getFastDigest(Path path) throws IOException { + synchronized (this) { + InMemoryContentInfo status = scopeLimitedStat(path, true); + if (status.isDirectory()) { + throw Error.EISDIR.exception(path); + } + if (!path.isReadable()) { + throw Error.EACCES.exception(path); + } + Preconditions.checkState(status instanceof FileInfo); + return ((FileInfo) status).getFastDigest(); } } /** Creates a new file at the given path and returns its inode. */ - private InMemoryContentInfo getOrCreateWritableInode(Path path) throws IOException { + protected InMemoryContentInfo getOrCreateWritableInode(Path path) throws IOException { // open(WR_ONLY) of a dangling link writes through the link. That means // that the usual path lookup operations have to behave differently when // resolving a path with the intent to create it: instead of failing with |