// Copyright 2014 The Bazel Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package com.google.devtools.build.lib.analysis; import static java.nio.charset.StandardCharsets.ISO_8859_1; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.devtools.build.lib.actions.ActionExecutionContext; import com.google.devtools.build.lib.actions.ActionKeyContext; import com.google.devtools.build.lib.actions.ActionOwner; import com.google.devtools.build.lib.actions.Artifact; import com.google.devtools.build.lib.analysis.actions.AbstractFileWriteAction; import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; import com.google.devtools.build.lib.events.EventHandler; import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec; import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec.VisibleForSerialization; import com.google.devtools.build.lib.util.Fingerprint; import com.google.devtools.build.lib.vfs.PathFragment; import java.io.BufferedWriter; import java.io.IOException; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.Writer; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Map; import javax.annotation.Nullable; /** * Action to create a manifest of input files for processing by a subsequent build step (e.g. * runfiles symlinking or archive building). * *

The manifest's format is specifiable by {@link ManifestType}, in accordance with the needs of * the calling functionality. * *

Note that this action carefully avoids building the manifest content in memory. */ @AutoCodec @Immutable // if all ManifestWriter implementations are immutable public final class SourceManifestAction extends AbstractFileWriteAction { private static final String GUID = "07459553-a3d0-4d37-9d78-18ed942470f4"; /** * Interface for defining manifest formatting and reporting specifics. Implementations must be * immutable. */ @VisibleForTesting interface ManifestWriter { /** * Writes a single line of manifest output. * * @param manifestWriter the output stream * @param rootRelativePath path of an entry relative to the manifest's root * @param symlink (optional) symlink that resolves the above path */ void writeEntry(Writer manifestWriter, PathFragment rootRelativePath, @Nullable Artifact symlink) throws IOException; /** * Fulfills {@link com.google.devtools.build.lib.actions.AbstractAction#getMnemonic()} */ String getMnemonic(); /** * Fulfills {@link com.google.devtools.build.lib.actions.AbstractAction#getRawProgressMessage()} */ String getRawProgressMessage(); /** * Fulfills {@link AbstractFileWriteAction#isRemotable()}. * @return */ boolean isRemotable(); } /** * The strategy we use to write manifest entries. */ private final ManifestWriter manifestWriter; /** * The runfiles for which to create the symlink tree. */ private final Runfiles runfiles; /** * Creates a new AbstractSourceManifestAction instance using latin1 encoding to write the manifest * file and with a specified root path for manifest entries. * * @param manifestWriter the strategy to use to write manifest entries * @param owner the action owner * @param primaryOutput the file to which to write the manifest * @param runfiles runfiles */ @VisibleForSerialization SourceManifestAction( ManifestWriter manifestWriter, ActionOwner owner, Artifact primaryOutput, Runfiles runfiles) { super(owner, getDependencies(runfiles), primaryOutput, false); this.manifestWriter = manifestWriter; this.runfiles = runfiles; } /** The {@link Runfiles} for which this action creates the symlink tree. */ public Runfiles getGeneratedRunfiles() { return runfiles; } @VisibleForTesting public void writeOutputFile(OutputStream out, EventHandler eventHandler) throws IOException { writeFile(out, runfiles.getRunfilesInputs(eventHandler, getOwner().getLocation())); } @Override public DeterministicWriter newDeterministicWriter(ActionExecutionContext ctx) throws IOException { final Map runfilesInputs = runfiles.getRunfilesInputs(ctx.getEventHandler(), getOwner().getLocation()); return new DeterministicWriter() { @Override public void writeOutputFile(OutputStream out) throws IOException { writeFile(out, runfilesInputs); } }; } @Override public boolean isRemotable() { return manifestWriter.isRemotable(); } /** * Returns the input dependencies for this action. Note we don't need to create the symlink * target Artifacts before we write the output manifest, so this Action does not have to * depend on them. The only necessary dependencies are pruning manifests, which must be read * to properly prune the tree. */ public static Collection getDependencies(Runfiles runfiles) { ImmutableList.Builder builder = ImmutableList.builder(); for (Runfiles.PruningManifest manifest : runfiles.getPruningManifests()) { builder.add(manifest.getManifestFile()); } return builder.build(); } /** * Sort the entries in both the normal and root manifests and write the output * file. * * @param out is the message stream to write errors to. * @param output The actual mapping of the output manifest. * @throws IOException */ private void writeFile(OutputStream out, Map output) throws IOException { Writer manifestFile = new BufferedWriter(new OutputStreamWriter(out, ISO_8859_1)); Comparator> fragmentComparator = new Comparator>() { @Override public int compare(Map.Entry path1, Map.Entry path2) { return path1.getKey().compareTo(path2.getKey()); } }; List> sortedManifest = new ArrayList<>(output.entrySet()); Collections.sort(sortedManifest, fragmentComparator); for (Map.Entry line : sortedManifest) { manifestWriter.writeEntry(manifestFile, line.getKey(), line.getValue()); } manifestFile.flush(); } @Override public String getMnemonic() { return manifestWriter.getMnemonic(); } @Override protected String getRawProgressMessage() { return manifestWriter.getRawProgressMessage() + " for " + getOwner().getLabel(); } @Override protected void computeKey(ActionKeyContext actionKeyContext, Fingerprint fp) { fp.addString(GUID); fp.addBoolean(runfiles.getLegacyExternalRunfiles()); fp.addPath(runfiles.getSuffix()); Map symlinks = runfiles.getSymlinksAsMap(null); fp.addInt(symlinks.size()); for (Map.Entry symlink : symlinks.entrySet()) { fp.addPath(symlink.getKey()); fp.addPath(symlink.getValue().getExecPath()); } Map rootSymlinks = runfiles.getRootSymlinksAsMap(null); fp.addInt(rootSymlinks.size()); for (Map.Entry rootSymlink : rootSymlinks.entrySet()) { fp.addPath(rootSymlink.getKey()); fp.addPath(rootSymlink.getValue().getExecPath()); } for (Artifact artifact : runfiles.getArtifacts()) { fp.addPath(artifact.getRootRelativePath()); fp.addPath(artifact.getExecPath()); } } /** * Supported manifest writing strategies. */ public enum ManifestType implements ManifestWriter { /** * Writes each line as: * * [rootRelativePath] [resolvingSymlink] * *

This strategy is suitable for creating an input manifest to a source view tree. Its * output is a valid input to * {@link com.google.devtools.build.lib.analysis.actions.SymlinkTreeAction}. */ SOURCE_SYMLINKS { @Override public void writeEntry(Writer manifestWriter, PathFragment rootRelativePath, Artifact symlink) throws IOException { manifestWriter.append(rootRelativePath.getPathString()); // This trailing whitespace is REQUIRED to process the single entry line correctly. manifestWriter.append(' '); if (symlink != null) { manifestWriter.append(symlink.getPath().getPathString()); } manifestWriter.append('\n'); } @Override public String getMnemonic() { return "SourceSymlinkManifest"; } @Override public String getRawProgressMessage() { return "Creating source manifest"; } @Override public boolean isRemotable() { // There is little gain to remoting these, since they include absolute path names inline. return false; } }, /** * Writes each line as: * * [rootRelativePath] * *

This strategy is suitable for an input into a packaging system (notably .par) that * consumes a list of all source files but needs that list to be constant with respect to * how the user has their client laid out on local disk. */ SOURCES_ONLY { @Override public void writeEntry(Writer manifestWriter, PathFragment rootRelativePath, Artifact symlink) throws IOException { manifestWriter.append(rootRelativePath.getPathString()); manifestWriter.append('\n'); manifestWriter.flush(); } @Override public String getMnemonic() { return "PackagingSourcesManifest"; } @Override public String getRawProgressMessage() { return "Creating file sources list"; } @Override public boolean isRemotable() { // Source-only symlink manifest has root-relative paths and does not include absolute paths. return true; } } } /** Creates an action for the given runfiles. */ public static SourceManifestAction forRunfiles(ManifestType manifestType, ActionOwner owner, Artifact output, Runfiles runfiles) { return new SourceManifestAction(manifestType, owner, output, runfiles); } /** * Builder class to construct {@link SourceManifestAction} instances. */ public static final class Builder { private final ManifestWriter manifestWriter; private final ActionOwner owner; private final Artifact output; private final Runfiles.Builder runfilesBuilder; public Builder(String prefix, ManifestType manifestType, ActionOwner owner, Artifact output, boolean legacyExternalRunfiles) { this(manifestType, owner, output, new Runfiles.Builder(prefix, legacyExternalRunfiles)); } public Builder( ManifestType manifestType, ActionOwner owner, Artifact output, Runfiles.Builder runfilesBuilder) { this.manifestWriter = manifestType; this.owner = owner; this.output = output; this.runfilesBuilder = runfilesBuilder; } @VisibleForTesting // Only used for testing. Builder(String prefix, ManifestWriter manifestWriter, ActionOwner owner, Artifact output) { this.runfilesBuilder = new Runfiles.Builder(prefix, false); this.manifestWriter = manifestWriter; this.owner = owner; this.output = output; } public SourceManifestAction build() { return new SourceManifestAction(manifestWriter, owner, output, runfilesBuilder.build()); } /** * Adds a set of symlinks from the artifacts' root-relative paths to the * artifacts themselves. */ public Builder addSymlinks(Iterable artifacts) { runfilesBuilder.addArtifacts(artifacts); return this; } /** * Adds a map of symlinks. */ public Builder addSymlinks(Map symlinks) { runfilesBuilder.addSymlinks(symlinks); return this; } /** * Adds a single symlink. */ public Builder addSymlink(PathFragment link, Artifact target) { runfilesBuilder.addSymlink(link, target); return this; } /** *

Adds a mapping of Artifacts to the directory above the normal symlink * forest base. */ public Builder addRootSymlinks(Map rootSymlinks) { runfilesBuilder.addRootSymlinks(rootSymlinks); return this; } /** * Set the empty files supplier for the manifest, see {@link Runfiles.EmptyFilesSupplier} * for more details. */ public Builder setEmptyFilesSupplier(Runfiles.EmptyFilesSupplier supplier) { runfilesBuilder.setEmptyFilesSupplier(supplier); return this; } /** * Adds a runfiles pruning manifest. */ @VisibleForTesting Builder addPruningManifest(Runfiles.PruningManifest manifest) { runfilesBuilder.addPruningManifest(manifest); return this; } } }