From 943a9c792032c75b25a1665e7143409fe3950041 Mon Sep 17 00:00:00 2001 From: felly Date: Thu, 9 Aug 2018 12:55:47 -0700 Subject: Pass Fileset information to the Action filesystem. PiperOrigin-RevId: 208096548 --- .../devtools/build/lib/exec/FilesetManifest.java | 29 ++++++++++++++++------ .../lib/skyframe/ActionExecutionFunction.java | 27 +++++++++++++++----- .../build/lib/skyframe/SkyframeActionExecutor.java | 7 +++--- .../devtools/build/lib/vfs/OutputService.java | 9 +++++-- 4 files changed, 54 insertions(+), 18 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/exec/FilesetManifest.java b/src/main/java/com/google/devtools/build/lib/exec/FilesetManifest.java index 1b35631108..af6232b750 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/FilesetManifest.java +++ b/src/main/java/com/google/devtools/build/lib/exec/FilesetManifest.java @@ -15,7 +15,9 @@ package com.google.devtools.build.lib.exec; import static java.nio.charset.StandardCharsets.UTF_8; +import com.google.common.collect.ImmutableMap; import com.google.common.io.LineProcessor; +import com.google.devtools.build.lib.actions.FileArtifactValue; import com.google.devtools.build.lib.actions.FilesetOutputSymlink; import com.google.devtools.build.lib.analysis.AnalysisUtils; import com.google.devtools.build.lib.vfs.FileSystemUtils; @@ -57,7 +59,7 @@ public final class FilesetManifest { Path execRoot, String workspaceName, RelativeSymlinkBehavior relSymlinkBehavior) - throws IOException { + throws IOException { Path file = execRoot.getRelative(AnalysisUtils.getManifestPathFromFilesetPath(manifest)); try { return FileSystemUtils.asByteSource(file).asCharSource(UTF_8) @@ -77,13 +79,17 @@ public final class FilesetManifest { throws IOException { LinkedHashMap entries = new LinkedHashMap<>(); Map relativeLinks = new HashMap<>(); + Map artifactValues = new HashMap<>(); for (FilesetOutputSymlink outputSymlink : outputSymlinks) { PathFragment fullLocation = targetPrefix.getRelative(outputSymlink.getName()); String artifact = outputSymlink.getTargetPath().getPathString(); artifact = artifact.isEmpty() ? null : artifact; addSymlinkEntry(artifact, fullLocation, relSymlinkbehavior, entries, relativeLinks); + if (outputSymlink.getMetadata() instanceof FileArtifactValue) { + artifactValues.put(artifact, (FileArtifactValue) outputSymlink.getMetadata()); + } } - return constructFilesetManifest(entries, relativeLinks); + return constructFilesetManifest(entries, relativeLinks, artifactValues); } private static final class ManifestLineProcessor implements LineProcessor { @@ -147,7 +153,7 @@ public final class FilesetManifest { @Override public FilesetManifest getResult() { - return constructFilesetManifest(entries, relativeLinks); + return constructFilesetManifest(entries, relativeLinks, ImmutableMap.of()); } } @@ -173,7 +179,8 @@ public final class FilesetManifest { } private static FilesetManifest constructFilesetManifest( - Map entries, Map relativeLinks) { + Map entries, Map relativeLinks, + Map artifactValues) { // Resolve relative symlinks if possible. Note that relativeLinks only contains entries in // RESOLVE mode. for (Map.Entry e : relativeLinks.entrySet()) { @@ -196,16 +203,24 @@ public final class FilesetManifest { } entries.put(location, actual); } - return new FilesetManifest(entries); + return new FilesetManifest(entries, artifactValues); } private final Map entries; + private final Map artifactValues; - private FilesetManifest(Map entries) { + private FilesetManifest(Map entries, + Map artifactValues) { this.entries = Collections.unmodifiableMap(entries); + this.artifactValues = artifactValues; } public Map getEntries() { return entries; } -} + + public Map getArtifactValues() { + return artifactValues; + } + +} \ No newline at end of file diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionFunction.java index 8cefd9c0a7..7bff8b0193 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionFunction.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionFunction.java @@ -423,7 +423,13 @@ public class ActionExecutionFunction implements SkyFunction, CompletionReceiver if (action.discoversInputs()) { if (state.discoveredInputs == null) { try { - state.updateFileSystemContext(skyframeActionExecutor, env, metadataHandler); + try { + state.updateFileSystemContext(skyframeActionExecutor, env, metadataHandler, + ImmutableMap.of()); + } catch (IOException e) { + throw new ActionExecutionException( + "Failed to update filesystem context: ", e, action, /*catastrophe=*/ false); + } state.discoveredInputs = skyframeActionExecutor.discoverInputs( action, perActionFileCache, metadataHandler, env, state.actionFileSystem); @@ -476,13 +482,20 @@ public class ActionExecutionFunction implements SkyFunction, CompletionReceiver filesetMappings.put(actionInput.getExecPath(), filesetValue.getOutputSymlinks()); } - state.updateFileSystemContext(skyframeActionExecutor, env, metadataHandler); + ImmutableMap> filesets = + filesetMappings.build(); + try { + state.updateFileSystemContext(skyframeActionExecutor, env, metadataHandler, filesets); + } catch (IOException e) { + throw new ActionExecutionException( + "Failed to update filesystem context: ", e, action, /*catastrophe=*/ false); + } try (ActionExecutionContext actionExecutionContext = skyframeActionExecutor.getContext( perActionFileCache, metadataHandler, Collections.unmodifiableMap(state.expandedArtifacts), - filesetMappings.build(), + filesets, state.actionFileSystem)) { if (!state.hasExecutedAction()) { state.value = @@ -792,11 +805,13 @@ public class ActionExecutionFunction implements SkyFunction, CompletionReceiver /** Must be called to assign values to the given variables as they change. */ void updateFileSystemContext( SkyframeActionExecutor executor, - SkyFunction.Environment env, - ActionMetadataHandler metadataHandler) { + Environment env, + ActionMetadataHandler metadataHandler, + ImmutableMap> filesets) + throws IOException { if (actionFileSystem != null) { executor.updateActionFileSystemContext( - actionFileSystem, env, metadataHandler::injectOutputData); + actionFileSystem, env, metadataHandler::injectOutputData, filesets); } } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java index 5f9ceb9da6..5ddabcce58 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java @@ -373,7 +373,7 @@ public final class SkyframeActionExecutor { return executorEngine.getExecRoot(); } - /** REQUIRES: {@link usesActionFileSystem} is true */ + /** REQUIRES: {@link #usesActionFileSystem()} is true */ FileSystem createActionFileSystem( String relativeOutputPath, ActionInputMap inputArtifactData, @@ -388,8 +388,9 @@ public final class SkyframeActionExecutor { } void updateActionFileSystemContext( - FileSystem actionFileSystem, Environment env, MetadataConsumer consumer) { - outputService.updateActionFileSystemContext(actionFileSystem, env, consumer); + FileSystem actionFileSystem, Environment env, MetadataConsumer consumer, + ImmutableMap> filesets) throws IOException { + outputService.updateActionFileSystemContext(actionFileSystem, env, consumer, filesets); } void executionOver() { diff --git a/src/main/java/com/google/devtools/build/lib/vfs/OutputService.java b/src/main/java/com/google/devtools/build/lib/vfs/OutputService.java index 03503bfed1..40753990ef 100644 --- a/src/main/java/com/google/devtools/build/lib/vfs/OutputService.java +++ b/src/main/java/com/google/devtools/build/lib/vfs/OutputService.java @@ -15,6 +15,7 @@ package com.google.devtools.build.lib.vfs; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.actions.Action; import com.google.devtools.build.lib.actions.ActionInputMap; import com.google.devtools.build.lib.actions.Artifact; @@ -22,11 +23,12 @@ import com.google.devtools.build.lib.actions.ArtifactPathResolver; import com.google.devtools.build.lib.actions.BuildFailedException; import com.google.devtools.build.lib.actions.EnvironmentalExecException; import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.FilesetOutputSymlink; import com.google.devtools.build.lib.actions.MetadataConsumer; import com.google.devtools.build.lib.actions.cache.MetadataHandler; import com.google.devtools.build.lib.events.EventHandler; import com.google.devtools.build.lib.util.AbruptExitException; -import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunction.Environment; import java.io.IOException; import java.util.UUID; import javax.annotation.Nullable; @@ -136,9 +138,12 @@ public interface OutputService { *

Should be called as context changes throughout action execution. * * @param actionFileSystem must be a filesystem returned by {@link #createActionFileSystem}. + * @param filesets The Fileset symlinks known for this action. */ default void updateActionFileSystemContext( - FileSystem actionFileSystem, SkyFunction.Environment env, MetadataConsumer consumer) {} + FileSystem actionFileSystem, Environment env, MetadataConsumer consumer, + ImmutableMap> filesets) + throws IOException {} default boolean supportsPathResolverForArtifactValues() { return false; -- cgit v1.2.3