This class is thread-safe except that * *
This is backed by injection into an {@link ActionMetadataHandler} instance so should only be
* called once per artifact.
*/
private MetadataConsumer metadataConsumer = null;
ActionFileSystem(
FileSystem delegate,
PathFragment execRoot,
String relativeOutputPath,
ImmutableList These cannot be passed into the constructor because while {@link ActionFileSystem} is
* action-scoped, the environment and metadata consumer change multiple times, at well defined
* points, during the lifetime of an action.
*/
public void updateContext(SkyFunction.Environment env, MetadataConsumer metadataConsumer) {
this.env = env;
this.metadataConsumer = metadataConsumer;
}
// -------------------- MetadataProvider implementation --------------------
@Override
@Nullable
public FileArtifactValue getMetadata(ActionInput actionInput) throws IOException {
return getMetadataChecked(actionInput.getExecPath());
}
@Override
@Nullable
public ActionInput getInput(String execPath) {
ActionInput input = inputArtifactData.getInput(execPath);
if (input != null) {
return input;
}
OptionalInputMetadata metadata =
optionalInputs.get(PathFragment.createAlreadyNormalized(execPath));
return metadata == null ? null : metadata.getArtifact();
}
// -------------------- InjectionListener Implementation --------------------
@Override
public void onInsert(ActionInput dest, byte[] digest, long size, int backendIndex)
throws IOException {
outputs.getUnchecked(dest.getExecPath()).set(
new RemoteFileArtifactValue(digest, size, backendIndex),
/*notifyConsumer=*/ false);
}
// -------------------- FileSystem implementation --------------------
@Override
public boolean supportsModifications(Path path) {
return isOutput(path);
}
@Override
public boolean supportsSymbolicLinksNatively(Path path) {
return isOutput(path);
}
@Override
protected boolean supportsHardLinksNatively(Path path) {
return isOutput(path);
}
@Override
public boolean isFilePathCaseSensitive() {
return true;
}
@Override
protected FileStatus stat(Path path, boolean followSymlinks) throws IOException {
FileArtifactValue metadata = getMetadataOrThrowFileNotFound(path);
return new FileStatus() {
@Override
public boolean isFile() {
return metadata.getType() == FileStateType.REGULAR_FILE;
}
@Override
public boolean isDirectory() {
// TODO(felly): Support directory awareness.
return false;
}
@Override
public boolean isSymbolicLink() {
// TODO(felly): We should have minimal support for symlink awareness when looking at
// output --> src and src --> src symlinks.
return false;
}
@Override
public boolean isSpecialFile() {
return metadata.getType() == FileStateType.SPECIAL_FILE;
}
@Override
public long getSize() {
return metadata.getSize();
}
@Override
public long getLastModifiedTime() {
return metadata.getModifiedTime();
}
@Override
public long getLastChangeTime() {
return metadata.getModifiedTime();
}
@Override
public long getNodeId() {
throw new UnsupportedOperationException();
}
};
}
/** ActionFileSystem currently doesn't track directories. */
@Override
public boolean createDirectory(Path path) throws IOException {
return true;
}
@Override
public void createDirectoryAndParents(Path path) throws IOException {}
@Override
protected long getFileSize(Path path, boolean followSymlinks) throws IOException {
return getMetadataOrThrowFileNotFound(path).getSize();
}
@Override
public boolean delete(Path path) throws IOException {
PathFragment execPath = asExecPath(path);
OutputMetadata output = outputs.getIfPresent(execPath);
return output != null && outputs.asMap().remove(execPath, output);
}
@Override
protected long getLastModifiedTime(Path path, boolean followSymlinks) throws IOException {
return getMetadataOrThrowFileNotFound(path).getModifiedTime();
}
@Override
public void setLastModifiedTime(Path path, long newTime) throws IOException {
throw new UnsupportedOperationException(path.getPathString());
}
@Override
public byte[] getxattr(Path path, String name) throws IOException {
FileArtifactValue metadata = getMetadataChecked(asExecPath(path));
if (metadata instanceof RemoteFileArtifactValue) {
RemoteFileArtifactValue remote = (RemoteFileArtifactValue) metadata;
// TODO(b/80244718): inject ActionFileSystem from elsewhere and replace with correct metadata
return ("/CENSORED_BY_LEAKR/"
+ remote.getLocationIndex()
+ "/"
+ LOWER_CASE_HEX.encode(remote.getDigest()))
.getBytes(US_ASCII);
}
if (metadata instanceof SourceFileArtifactValue) {
return resolveSourcePath((SourceFileArtifactValue) metadata).getxattr(name);
}
return getSourcePath(path.asFragment()).getxattr(name);
}
@Override
protected byte[] getFastDigest(Path path, HashFunction hash) throws IOException {
if (hash != HashFunction.MD5) {
return null;
}
return getMetadataOrThrowFileNotFound(path).getDigest();
}
@Override
protected Collection TODO(shahan): if this is insufficiently general, we can topologically order on the prefix
* relation between roots.
*/
private void validateRoots() {
for (PathFragment root1 : sourceRoots) {
Preconditions.checkState(
!root1.startsWith(execRootFragment), "%s starts with %s", root1, execRootFragment);
Preconditions.checkState(
!execRootFragment.startsWith(root1), "%s starts with %s", execRootFragment, root1);
for (PathFragment root2 : sourceRoots) {
if (root1 == root2) {
continue;
}
Preconditions.checkState(!root1.startsWith(root2), "%s starts with %s", root1, root2);
}
}
}
private static ByteString toByteString(byte[] digest) {
return ByteString.copyFrom(BaseEncoding.base16().lowerCase().encode(digest).getBytes(US_ASCII));
}
/** NB: resolves to the underlying filesytem instead of this one. */
private Path resolveSourcePath(SourceFileArtifactValue metadata) throws IOException {
return getSourcePath(sourceRoots.get(metadata.getSourceRootIndex()))
.getRelative(metadata.getExecPath());
}
@FunctionalInterface
public interface MetadataConsumer {
void accept(Artifact artifact, FileArtifactValue value) throws IOException;
}
private class OptionalInputMetadata {
private final Artifact artifact;
private volatile FileArtifactValue metadata = null;
private OptionalInputMetadata(Artifact artifact) {
this.artifact = artifact;
}
public Artifact getArtifact() {
return artifact;
}
public FileArtifactValue get() throws IOException {
if (metadata == null) {
synchronized (this) {
if (metadata == null) {
try {
// TODO(shahan): {@link SkyFunction.Environment} requires single-threaded access so
// we enforce that here by making these (multithreaded) calls synchronized. It might
// be better to make the underlying methods synchronized to avoid having another
// caller unintentionally calling into the environment without locking.
//
// This is currently known to be reached from the distributor during remote include
// scanning which we expect to propagate exceptions up for skyframe restarts.
synchronized (env) {
metadata = (FileArtifactValue) env.getValue(ArtifactSkyKey.key(artifact, false));
}
} catch (InterruptedException e) {
throw new InterruptedIOException(e.getMessage());
}
if (metadata == null) {
throw new ActionExecutionFunction.MissingDepException();
}
}
}
}
return metadata;
}
}
private class OutputMetadata {
private final @Nullable Artifact artifact;
@Nullable private volatile FileArtifactValue metadata = null;
private OutputMetadata(Artifact artifact) {
this.artifact = artifact;
}
@Nullable
public FileArtifactValue get() {
return metadata;
}
/**
* Sets the output metadata, and maybe notify the metadataConsumer.
*
* @param metadata the metadata to write
* @param notifyConsumer whether to notify metadataConsumer. Callers should not notify the
* metadataConsumer if it will be notified separately at the Spawn level.
*/
public void set(FileArtifactValue metadata, boolean notifyConsumer) throws IOException {
if (notifyConsumer && artifact != null) {
metadataConsumer.accept(artifact, metadata);
}
this.metadata = metadata;
}
/** Callers are expected to close the returned stream. */
public ByteArrayOutputStream getOutputStream() {
Preconditions.checkState(metadata == null, "getOutputStream called twice for: %s", artifact);
return new ByteArrayOutputStream() {
@Override
public void close() throws IOException {
flush();
super.close();
}
@Override
public void flush() throws IOException {
super.flush();
byte[] data = toByteArray();
set(new InlineFileArtifactValue(data, Hashing.md5().hashBytes(data).asBytes()),
/*notifyConsumer=*/ true);
}
};
}
}
}