// Copyright 2014 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.rules.cpp;
import static java.nio.charset.StandardCharsets.ISO_8859_1;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.devtools.build.lib.actions.AbstractAction;
import com.google.devtools.build.lib.actions.ActionExecutionContext;
import com.google.devtools.build.lib.actions.ActionExecutionException;
import com.google.devtools.build.lib.actions.ActionInput;
import com.google.devtools.build.lib.actions.ActionOwner;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.CommandAction;
import com.google.devtools.build.lib.actions.ExecException;
import com.google.devtools.build.lib.actions.ExecutionInfoSpecifier;
import com.google.devtools.build.lib.actions.Executor;
import com.google.devtools.build.lib.actions.ResourceSet;
import com.google.devtools.build.lib.actions.SimpleSpawn;
import com.google.devtools.build.lib.actions.Spawn;
import com.google.devtools.build.lib.actions.extra.CppLinkInfo;
import com.google.devtools.build.lib.actions.extra.ExtraActionInfo;
import com.google.devtools.build.lib.analysis.RuleContext;
import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
import com.google.devtools.build.lib.analysis.actions.ExecutionRequirements;
import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
import com.google.devtools.build.lib.collect.nestedset.NestedSet;
import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
import com.google.devtools.build.lib.rules.cpp.Link.LinkStaticness;
import com.google.devtools.build.lib.rules.cpp.Link.LinkTargetType;
import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink;
import com.google.devtools.build.lib.util.Fingerprint;
import com.google.devtools.build.lib.util.ShellEscaper;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;
/**
* Action that represents a linking step.
*/
@ThreadCompatible
public final class CppLinkAction extends AbstractAction
implements ExecutionInfoSpecifier, CommandAction {
/**
* An abstraction for creating intermediate and output artifacts for C++ linking.
*
*
This is unfortunately necessary, because most of the time, these artifacts are well-behaved
* ones sitting under a package directory, but nativedeps link actions can be shared. In order to
* avoid creating every artifact here with {@code getShareableArtifact()}, we abstract the
* artifact creation away.
*/
public interface LinkArtifactFactory {
/**
* Create an artifact at the specified root-relative path in the bin directory.
*/
Artifact create(RuleContext ruleContext, BuildConfiguration configuration,
PathFragment rootRelativePath);
}
/**
* An implementation of {@link LinkArtifactFactory} that can only create artifacts in the package
* directory.
*/
public static final LinkArtifactFactory DEFAULT_ARTIFACT_FACTORY = new LinkArtifactFactory() {
@Override
public Artifact create(RuleContext ruleContext, BuildConfiguration configuration,
PathFragment rootRelativePath) {
return ruleContext.getDerivedArtifact(
rootRelativePath, configuration.getBinDirectory(ruleContext.getRule().getRepository()));
}
};
private static final String LINK_GUID = "58ec78bd-1176-4e36-8143-439f656b181d";
private static final String FAKE_LINK_GUID = "da36f819-5a15-43a9-8a45-e01b60e10c8b";
@Nullable private final String mnemonic;
private final CppConfiguration cppConfiguration;
private final LibraryToLink outputLibrary;
private final Artifact linkOutput;
private final LibraryToLink interfaceOutputLibrary;
private final ImmutableSet clientEnvironmentVariables;
private final ImmutableMap actionEnv;
private final ImmutableMap toolchainEnv;
private final ImmutableSet executionRequirements;
private final LinkCommandLine linkCommandLine;
/** True for cc_fake_binary targets. */
private final boolean fake;
private final boolean isLTOIndexing;
// This is set for both LTO indexing and LTO linking.
@Nullable private final Iterable allLTOBackendArtifacts;
private final Iterable mandatoryInputs;
// Linking uses a lot of memory; estimate 1 MB per input file, min 1.5 Gib.
// It is vital to not underestimate too much here,
// because running too many concurrent links can
// thrash the machine to the point where it stops
// responding to keystrokes or mouse clicks.
// CPU and IO do not scale similarly and still use the static minimum estimate.
public static final ResourceSet LINK_RESOURCES_PER_INPUT =
ResourceSet.createWithRamCpuIo(1, 0, 0);
// This defines the minimum of each resource that will be reserved.
public static final ResourceSet MIN_STATIC_LINK_RESOURCES =
ResourceSet.createWithRamCpuIo(1536, 1, 0.3);
// Dynamic linking should be cheaper than static linking.
public static final ResourceSet MIN_DYNAMIC_LINK_RESOURCES =
ResourceSet.createWithRamCpuIo(1024, 0.3, 0.2);
/**
* Use {@link CppLinkActionBuilder} to create instances of this class. Also see there for the
* documentation of all parameters.
*
* This constructor is intentionally private and is only to be called from {@link
* CppLinkActionBuilder#build()}.
*/
CppLinkAction(
ActionOwner owner,
String mnemonic,
Iterable inputs,
ImmutableList outputs,
CppConfiguration cppConfiguration,
LibraryToLink outputLibrary,
Artifact linkOutput,
LibraryToLink interfaceOutputLibrary,
boolean fake,
boolean isLTOIndexing,
Iterable allLTOBackendArtifacts,
LinkCommandLine linkCommandLine,
ImmutableSet clientEnvironmentVariables,
ImmutableMap actionEnv,
ImmutableMap toolchainEnv,
ImmutableSet executionRequirements) {
super(owner, inputs, outputs);
if (mnemonic == null) {
this.mnemonic = (isLTOIndexing) ? "CppLTOIndexing" : "CppLink";
} else {
this.mnemonic = mnemonic;
}
this.mandatoryInputs = inputs;
this.cppConfiguration = cppConfiguration;
this.outputLibrary = outputLibrary;
this.linkOutput = linkOutput;
this.interfaceOutputLibrary = interfaceOutputLibrary;
this.fake = fake;
this.isLTOIndexing = isLTOIndexing;
this.allLTOBackendArtifacts = allLTOBackendArtifacts;
this.linkCommandLine = linkCommandLine;
this.clientEnvironmentVariables = clientEnvironmentVariables;
this.actionEnv = actionEnv;
this.toolchainEnv = toolchainEnv;
this.executionRequirements = executionRequirements;
}
private CppConfiguration getCppConfiguration() {
return cppConfiguration;
}
@VisibleForTesting
public String getTargetCpu() {
return getCppConfiguration().getTargetCpu();
}
public String getHostSystemName() {
return getCppConfiguration().getHostSystemName();
}
@Override
public Iterable getClientEnvironmentVariables() {
return clientEnvironmentVariables;
}
@Override
public ImmutableMap getEnvironment() {
LinkedHashMap result = new LinkedHashMap<>();
result.putAll(actionEnv);
result.putAll(toolchainEnv);
if (!needsToRunOnMac()) {
// This prevents gcc from writing the unpredictable (and often irrelevant)
// value of getcwd() into the debug info.
result.put("PWD", "/proc/self/cwd");
}
return ImmutableMap.copyOf(result);
}
/**
* Returns the link configuration; for correctness you should not call this method during
* execution - only the argv is part of the action cache key, and we therefore don't guarantee
* that the action will be re-executed if the contents change in a way that does not affect the
* argv.
*/
@VisibleForTesting
public LinkCommandLine getLinkCommandLine() {
return linkCommandLine;
}
/**
* Returns the output of this action as a {@link LibraryToLink} or null if it is an executable.
*/
@Nullable
public LibraryToLink getOutputLibrary() {
return outputLibrary;
}
public LibraryToLink getInterfaceOutputLibrary() {
return interfaceOutputLibrary;
}
/**
* Returns the output of the linking.
*/
public Artifact getLinkOutput() {
return linkOutput;
}
/**
* Returns the path to the output artifact produced by the linker.
*/
public Path getOutputFile() {
return linkOutput.getPath();
}
@Override
public Map getExecutionInfo() {
ImmutableMap.Builder result = ImmutableMap.builder();
for (String requirement : executionRequirements) {
result.put(requirement, "");
}
return result.build();
}
@VisibleForTesting
public List getRawLinkArgv() {
return linkCommandLine.getRawLinkArgv();
}
@VisibleForTesting
public List getArgv() {
return linkCommandLine.arguments();
}
@Override
public List getArguments() {
return getArgv();
}
/**
* Returns the command line specification for this link, included any required linkstamp
* compilation steps. The command line may refer to a .params file.
*
* @return a finalized command line suitable for execution
*/
public final List getCommandLine() {
return linkCommandLine.getCommandLine();
}
Iterable getAllLTOBackendArtifacts() {
return allLTOBackendArtifacts;
}
private boolean needsToRunOnMac() {
return getHostSystemName().equals(CppConfiguration.MAC_SYSTEM_NAME);
}
@Override
@ThreadCompatible
public void execute(
ActionExecutionContext actionExecutionContext)
throws ActionExecutionException, InterruptedException {
if (fake) {
executeFake();
} else {
Executor executor = actionExecutionContext.getExecutor();
try {
// Collect input files
List allInputs = new ArrayList<>();
Artifact.addExpandedArtifacts(
getMandatoryInputs(), allInputs, actionExecutionContext.getArtifactExpander());
ImmutableMap executionInfo = ImmutableMap.of();
if (needsToRunOnMac()) {
executionInfo = ImmutableMap.of(ExecutionRequirements.REQUIRES_DARWIN, "");
}
Spawn spawn = new SimpleSpawn(
this,
ImmutableList.copyOf(getCommandLine()),
getEnvironment(),
executionInfo,
ImmutableList.copyOf(allInputs),
getOutputs().asList(),
estimateResourceConsumptionLocal());
executor.getSpawnActionContext(getMnemonic()).exec(
spawn, actionExecutionContext);
} catch (ExecException e) {
throw e.toActionExecutionException("Linking of rule '" + getOwner().getLabel() + "'",
executor.getVerboseFailures(), this);
}
}
}
// Don't forget to update FAKE_LINK_GUID if you modify this method.
@ThreadCompatible
private void executeFake()
throws ActionExecutionException {
// The uses of getLinkConfiguration in this method may not be consistent with the computed key.
// I.e., this may be incrementally incorrect.
final Collection linkstampOutputs = getLinkCommandLine().getLinkstamps().values();
// Prefix all fake output files in the command line with $TEST_TMPDIR/.
final String outputPrefix = "$TEST_TMPDIR/";
List escapedLinkArgv = escapeLinkArgv(linkCommandLine.getRawLinkArgv(),
linkstampOutputs, outputPrefix);
// Write the commands needed to build the real target to the fake target
// file.
StringBuilder s = new StringBuilder();
Joiner.on('\n').appendTo(s,
"# This is a fake target file, automatically generated.",
"# Do not edit by hand!",
"echo $0 is a fake target file and not meant to be executed.",
"exit 0",
"EOS",
"",
"makefile_dir=.",
"");
try {
// Concatenate all the (fake) .o files into the result.
for (LinkerInput linkerInput : getLinkCommandLine().getLinkerInputs()) {
Artifact objectFile = linkerInput.getArtifact();
if ((CppFileTypes.OBJECT_FILE.matches(objectFile.getFilename())
|| CppFileTypes.PIC_OBJECT_FILE.matches(objectFile.getFilename()))
&& linkerInput.isFake()) {
s.append(FileSystemUtils.readContentAsLatin1(objectFile.getPath())); // (IOException)
}
}
s.append(getOutputFile().getBaseName()).append(": ");
for (Artifact linkstamp : linkstampOutputs) {
s.append("mkdir -p " + outputPrefix +
linkstamp.getExecPath().getParentDirectory() + " && ");
}
Joiner.on(' ').appendTo(s,
ShellEscaper.escapeAll(linkCommandLine.finalizeAlreadyEscapedWithLinkstampCommands(
escapedLinkArgv, outputPrefix)));
s.append('\n');
if (getOutputFile().exists()) {
getOutputFile().setWritable(true); // (IOException)
}
FileSystemUtils.writeContent(getOutputFile(), ISO_8859_1, s.toString());
getOutputFile().setExecutable(true); // (IOException)
for (Artifact linkstamp : linkstampOutputs) {
FileSystemUtils.touchFile(linkstamp.getPath());
}
} catch (IOException e) {
throw new ActionExecutionException("failed to create fake link command for rule '" +
getOwner().getLabel() + ": " + e.getMessage(),
this, false);
}
}
/**
* Shell-escapes the raw link command line.
*
* @param rawLinkArgv raw link command line
* @param linkstampOutputs linkstamp artifacts
* @param outputPrefix to be prepended to any outputs
* @return escaped link command line
*/
private List escapeLinkArgv(List rawLinkArgv,
final Collection linkstampOutputs, final String outputPrefix) {
final List linkstampExecPaths = Artifact.asExecPaths(linkstampOutputs);
ImmutableList.Builder escapedArgs = ImmutableList.builder();
for (String rawArg : rawLinkArgv) {
String escapedArg;
if (rawArg.equals(getPrimaryOutput().getExecPathString())
|| linkstampExecPaths.contains(rawArg)) {
escapedArg = outputPrefix + ShellEscaper.escapeString(rawArg);
} else if (rawArg.startsWith(Link.FAKE_OBJECT_PREFIX)) {
escapedArg = outputPrefix + ShellEscaper.escapeString(
rawArg.substring(Link.FAKE_OBJECT_PREFIX.length()));
} else {
escapedArg = ShellEscaper.escapeString(rawArg);
}
escapedArgs.add(escapedArg);
}
return escapedArgs.build();
}
@Override
public ExtraActionInfo.Builder getExtraActionInfo() {
// The uses of getLinkConfiguration in this method may not be consistent with the computed key.
// I.e., this may be incrementally incorrect.
CppLinkInfo.Builder info = CppLinkInfo.newBuilder();
info.addAllInputFile(Artifact.toExecPaths(
LinkerInputs.toLibraryArtifacts(getLinkCommandLine().getLinkerInputs())));
info.addAllInputFile(Artifact.toExecPaths(
LinkerInputs.toLibraryArtifacts(getLinkCommandLine().getRuntimeInputs())));
info.setOutputFile(getPrimaryOutput().getExecPathString());
if (interfaceOutputLibrary != null) {
info.setInterfaceOutputFile(interfaceOutputLibrary.getArtifact().getExecPathString());
}
info.setLinkTargetType(getLinkCommandLine().getLinkTargetType().name());
info.setLinkStaticness(getLinkCommandLine().getLinkStaticness().name());
info.addAllLinkStamp(Artifact.toExecPaths(getLinkCommandLine().getLinkstamps().values()));
info.addAllBuildInfoHeaderArtifact(
Artifact.toExecPaths(getLinkCommandLine().getBuildInfoHeaderArtifacts()));
info.addAllLinkOpt(getLinkCommandLine().getRawLinkArgv());
return super.getExtraActionInfo()
.setExtension(CppLinkInfo.cppLinkInfo, info.build());
}
@Override
protected String computeKey() {
Fingerprint f = new Fingerprint();
f.addString(fake ? FAKE_LINK_GUID : LINK_GUID);
f.addString(getCppConfiguration().getLdExecutable().getPathString());
f.addStrings(linkCommandLine.arguments());
f.addStrings(getExecutionInfo().keySet());
// TODO(bazel-team): For correctness, we need to ensure the invariant that all values accessed
// during the execution phase are also covered by the key. Above, we add the argv to the key,
// which covers most cases. Unfortunately, the extra action and fake support methods above also
// sometimes directly access settings from the link configuration that may or may not affect the
// key. We either need to change the code to cover them in the key computation, or change the
// LinkConfiguration to disallow the combinations where the value of a setting does not affect
// the argv.
f.addBoolean(linkCommandLine.isNativeDeps());
f.addBoolean(linkCommandLine.useTestOnlyFlags());
if (linkCommandLine.getRuntimeSolibDir() != null) {
f.addPath(linkCommandLine.getRuntimeSolibDir());
}
f.addBoolean(isLTOIndexing);
return f.hexDigestAndReset();
}
@Override
public String describeKey() {
StringBuilder message = new StringBuilder();
if (fake) {
message.append("Fake ");
}
message.append(getProgressMessage());
message.append('\n');
message.append(" Command: ");
message.append(
ShellEscaper.escapeString(getCppConfiguration().getLdExecutable().getPathString()));
message.append('\n');
// Outputting one argument per line makes it easier to diff the results.
for (String argument : ShellEscaper.escapeAll(linkCommandLine.arguments())) {
message.append(" Argument: ");
message.append(argument);
message.append('\n');
}
return message.toString();
}
@Override
public String getMnemonic() {
return mnemonic;
}
@Override
protected String getRawProgressMessage() {
return (isLTOIndexing ? "LTO indexing " : "Linking ") + linkOutput.prettyPrint();
}
/**
* Estimate the resources consumed when this action is run locally.
*/
public ResourceSet estimateResourceConsumptionLocal() {
// It's ok if this behaves differently even if the key is identical.
ResourceSet minLinkResources =
getLinkCommandLine().getLinkStaticness() == Link.LinkStaticness.DYNAMIC
? MIN_DYNAMIC_LINK_RESOURCES
: MIN_STATIC_LINK_RESOURCES;
final int inputSize = Iterables.size(getLinkCommandLine().getLinkerInputs())
+ Iterables.size(getLinkCommandLine().getRuntimeInputs());
return ResourceSet.createWithRamCpuIo(
Math.max(inputSize * LINK_RESOURCES_PER_INPUT.getMemoryMb(),
minLinkResources.getMemoryMb()),
Math.max(inputSize * LINK_RESOURCES_PER_INPUT.getCpuUsage(),
minLinkResources.getCpuUsage()),
Math.max(inputSize * LINK_RESOURCES_PER_INPUT.getIoUsage(),
minLinkResources.getIoUsage())
);
}
@Override
public Iterable getMandatoryInputs() {
return mandatoryInputs;
}
/**
* Determines whether or not this link should output a symbol counts file.
*/
public static boolean enableSymbolsCounts(
CppConfiguration cppConfiguration, boolean fake, LinkTargetType linkType) {
return cppConfiguration.getSymbolCounts()
&& cppConfiguration.supportsGoldLinker()
&& linkType == LinkTargetType.EXECUTABLE
&& !fake;
}
public static PathFragment symbolCountsFileName(PathFragment binaryName) {
return binaryName.replaceName(binaryName.getBaseName() + ".sc");
}
/**
* TransitiveInfoProvider for ELF link actions.
*/
@Immutable @ThreadSafe
public static final class Context implements TransitiveInfoProvider {
// Morally equivalent with {@link Builder}, except these are immutable.
// Keep these in sync with {@link Builder}.
final ImmutableSet objectFiles;
final ImmutableSet nonCodeInputs;
final NestedSet libraries;
final NestedSet crosstoolInputs;
final ImmutableMap ltoBitcodeFiles;
final Artifact runtimeMiddleman;
final NestedSet runtimeInputs;
final ArtifactCategory runtimeType;
final NestedSet compilationInputs;
final ImmutableSet linkstamps;
final ImmutableList linkopts;
final LinkTargetType linkType;
final LinkStaticness linkStaticness;
final boolean fake;
final boolean isNativeDeps;
final boolean useTestOnlyFlags;
/**
* Given a {@link CppLinkActionBuilder}, creates a {@code Context} to pass to another target.
* Note well: Keep the Builder->Context and Context->Builder transforms consistent!
*
* @param builder a mutable {@link CppLinkActionBuilder} to clone from
*/
public Context(CppLinkActionBuilder builder) {
this.objectFiles = ImmutableSet.copyOf(builder.getObjectFiles());
this.nonCodeInputs = ImmutableSet.copyOf(builder.getNonCodeInputs());
this.libraries = NestedSetBuilder.linkOrder()
.addTransitive(builder.getLibraries().build()).build();
this.crosstoolInputs =
NestedSetBuilder.stableOrder().addTransitive(builder.getCrosstoolInputs()).build();
this.ltoBitcodeFiles = ImmutableMap.copyOf(builder.getLtoBitcodeFiles());
this.runtimeMiddleman = builder.getRuntimeMiddleman();
this.runtimeInputs =
NestedSetBuilder.stableOrder().addTransitive(builder.getRuntimeInputs()).build();
this.runtimeType = builder.getRuntimeType();
this.compilationInputs = NestedSetBuilder.stableOrder()
.addTransitive(builder.getCompilationInputs().build()).build();
this.linkstamps = ImmutableSet.copyOf(builder.getLinkstamps());
this.linkopts = ImmutableList.copyOf(builder.getLinkopts());
this.linkType = builder.getLinkType();
this.linkStaticness = builder.getLinkStaticness();
this.fake = builder.isFake();
this.isNativeDeps = builder.isNativeDeps();
this.useTestOnlyFlags = builder.useTestOnlyFlags();
}
/**
* Returns linker inputs that are not libraries.
*/
public ImmutableSet getObjectFiles() {
return this.objectFiles;
}
/**
* Returns libraries that are to be inputs to the linker.
*/
public NestedSet getLibraries() {
return this.libraries;
}
/**
* Returns input artifacts arising from the crosstool.
*/
public NestedSet getCrosstoolInputs() {
return this.crosstoolInputs;
}
/**
* Returns the runtime middleman artifact.
*/
public Artifact getRuntimeMiddleman() {
return this.runtimeMiddleman;
}
/**
* Returns runtime inputs for the linker.
*/
public NestedSet getRuntimeInputs() {
return this.runtimeInputs;
}
/**
* Returns compilation inputs for compilations arising from the linking of this target.
*/
public NestedSet getCompilationInputs() {
return this.compilationInputs;
}
/**
* Returns linkstamp artifacts.
*/
public ImmutableSet getLinkstamps() {
return this.linkstamps;
}
/**
* Returns linkopts for the linking of this target.
*/
public ImmutableList getLinkopts() {
return this.linkopts;
}
/**
* Returns the type of the linking of this target.
*/
public LinkTargetType getLinkType() {
return this.linkType;
}
/**
* Returns the staticness of the linking of this target.
*/
public LinkStaticness getLinkStaticness() {
return this.linkStaticness;
}
/**
* Returns true for cc_fake_binary targets.
*/
public boolean isFake() {
return this.fake;
}
/**
* Returns true if the linking of this target is used for a native dependecy library.
*/
public boolean isNativeDeps() {
return this.isNativeDeps;
}
/**
* Returns true if the linking for this target uses test-specific flags.
*/
public boolean useTestOnlyFlags() {
return this.useTestOnlyFlags;
}
}
}