diff options
Diffstat (limited to 'src/main/java/com/google/devtools/build/lib/rules/cpp')
4 files changed, 375 insertions, 2 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcIncLibrary.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcIncLibrary.java index 3790b55b14..b4ffc91cbb 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcIncLibrary.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcIncLibrary.java @@ -24,7 +24,6 @@ import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; import com.google.devtools.build.lib.analysis.RuleContext; import com.google.devtools.build.lib.analysis.Runfiles; import com.google.devtools.build.lib.analysis.RunfilesProvider; -import com.google.devtools.build.lib.analysis.actions.CreateIncSymlinkAction; import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; import com.google.devtools.build.lib.rules.cpp.CcToolchainFeatures.FeatureConfiguration; import com.google.devtools.build.lib.rules.test.InstrumentedFilesProvider; diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CreateIncSymlinkAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CreateIncSymlinkAction.java new file mode 100644 index 0000000000..443787863a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CreateIncSymlinkAction.java @@ -0,0 +1,113 @@ + +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSortedMap; +import com.google.devtools.build.lib.actions.AbstractAction; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.Symlinks; +import java.io.IOException; +import java.util.Map; +import java.util.SortedMap; + +/** + * This action creates a set of symbolic links. + */ +@Immutable +public final class CreateIncSymlinkAction extends AbstractAction { + private final ImmutableSortedMap<Artifact, Artifact> symlinks; + private final Path includePath; + + /** + * Creates a new instance. The symlinks map maps symlinks to their targets, i.e. the symlink paths + * must be unique, but several of them can point to the same target. All outputs must be under + * {@code includePath}. + */ + public CreateIncSymlinkAction( + ActionOwner owner, Map<Artifact, Artifact> symlinks, Path includePath) { + super(owner, ImmutableList.copyOf(symlinks.values()), ImmutableList.copyOf(symlinks.keySet())); + this.symlinks = ImmutableSortedMap.copyOf(symlinks, Artifact.EXEC_PATH_COMPARATOR); + this.includePath = includePath; + } + + @Override + public void prepare(Path execRoot) throws IOException { + if (includePath.isDirectory(Symlinks.NOFOLLOW)) { + FileSystemUtils.deleteTree(includePath); + } + super.prepare(execRoot); + } + + @Override + public void execute(ActionExecutionContext actionExecutionContext) + throws ActionExecutionException { + try { + for (Map.Entry<Artifact, Artifact> entry : symlinks.entrySet()) { + Path symlink = entry.getKey().getPath(); + symlink.createSymbolicLink(entry.getValue().getPath()); + } + } catch (IOException e) { + String message = "IO Error while creating symlink"; + throw new ActionExecutionException(message, e, this, false); + } + } + + @VisibleForTesting + public SortedMap<Artifact, Artifact> getSymlinks() { + return symlinks; + } + + @Override + public ResourceSet estimateResourceConsumption(Executor executor) { + // We're mainly doing I/O, so CPU usage should be very low; most of the + // time we'll be blocked waiting for the OS. + // The only exception is the fingerprint digest calculation for the stamp + // file contents. + return ResourceSet.createWithRamCpuIo(/*memoryMb=*/0, /*cpuUsage=*/0.005, /*ioUsage=*/0.0); + } + + @Override + public String computeKey() { + Fingerprint key = new Fingerprint(); + for (Map.Entry<Artifact, Artifact> entry : symlinks.entrySet()) { + key.addPath(entry.getKey().getPath()); + key.addPath(entry.getValue().getPath()); + } + return key.hexDigestAndReset(); + } + + @Override + protected String getRawProgressMessage() { + return null; // users don't really want to know about inc symlinks. + } + + @Override + public String getMnemonic() { + return "Symlink"; + } +} + diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LTOBackendAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LTOBackendAction.java new file mode 100644 index 0000000000..abc90a3585 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LTOBackendAction.java @@ -0,0 +1,262 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.actions.AbstractAction; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ArtifactResolver; +import com.google.devtools.build.lib.actions.PackageRootResolutionException; +import com.google.devtools.build.lib.actions.PackageRootResolver; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.analysis.actions.CommandLine; +import com.google.devtools.build.lib.analysis.actions.SpawnAction; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.PathFragment; +import java.io.IOException; +import java.util.Collection; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; +import javax.annotation.Nullable; +import javax.annotation.concurrent.GuardedBy; + +/** + * Action used by LTOBackendArtifacts to create an LTOBackendAction. Similar to {@link SpawnAction}, + * except that inputs are discovered from the imports file created by the ThinLTO indexing step for + * each backend artifact. + * + * <p>See {@link LTOBackendArtifacts} for a high level description of the ThinLTO build process. The + * LTO indexing step takes all bitcode .o files and decides which other .o file symbols can be + * imported/inlined. The additional input files for each backend action are then written to an + * imports file. Therefore these new inputs must be discovered here by subsetting the imports paths + * from the set of all bitcode artifacts, before executing the backend action. + * + * <p>For more information on ThinLTO see + * http://blog.llvm.org/2016/06/thinlto-scalable-and-incremental-lto.html. + */ +public final class LTOBackendAction extends SpawnAction { + // This can be read/written from multiple threads, and so accesses should be synchronized. + @GuardedBy("this") + private boolean inputsKnown; + + private Collection<Artifact> mandatoryInputs; + private Map<PathFragment, Artifact> bitcodeFiles; + private Artifact imports; + + private static final String GUID = "72ce1eca-4625-4e24-a0d8-bb91bb8b0e0e"; + + public LTOBackendAction( + Collection<Artifact> inputs, + Map<PathFragment, Artifact> allBitcodeFiles, + Artifact importsFile, + Collection<Artifact> outputs, + ActionOwner owner, + CommandLine argv, + Map<String, String> environment, + Set<String> clientEnvironmentVariables, + Map<String, String> executionInfo, + String progressMessage, + ImmutableMap<PathFragment, Artifact> inputManifests, + String mnemonic) { + super( + owner, + ImmutableList.<Artifact>of(), + inputs, + outputs, + AbstractAction.DEFAULT_RESOURCE_SET, + argv, + ImmutableMap.copyOf(environment), + ImmutableSet.copyOf(clientEnvironmentVariables), + ImmutableMap.copyOf(executionInfo), + progressMessage, + ImmutableMap.copyOf(inputManifests), + mnemonic, + false, + null); + + inputsKnown = false; + mandatoryInputs = inputs; + bitcodeFiles = allBitcodeFiles; + imports = importsFile; + } + + @Override + public boolean discoversInputs() { + return true; + } + + private Set<Artifact> computeBitcodeInputs(Collection<PathFragment> inputPaths) { + HashSet<Artifact> bitcodeInputs = new HashSet<>(); + for (PathFragment inputPath : inputPaths) { + Artifact inputArtifact = bitcodeFiles.get(inputPath); + if (inputArtifact != null) { + bitcodeInputs.add(inputArtifact); + } + } + return bitcodeInputs; + } + + @Nullable + @Override + public Iterable<Artifact> discoverInputs(ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + // Build set of files this LTO backend artifact will import from. + HashSet<PathFragment> importSet = new HashSet<>(); + try { + for (String line : FileSystemUtils.iterateLinesAsLatin1(imports.getPath())) { + if (!line.isEmpty()) { + PathFragment execPath = new PathFragment(line); + if (execPath.isAbsolute()) { + throw new ActionExecutionException( + "Absolute paths not allowed in imports file " + imports.getPath() + ": " + execPath, + this, + false); + } + importSet.add(new PathFragment(line)); + } + } + } catch (IOException e) { + throw new ActionExecutionException( + "error iterating imports file " + imports.getPath(), e, this, false); + } + + // Convert the import set of paths to the set of bitcode file artifacts. + Set<Artifact> bitcodeInputSet = computeBitcodeInputs(importSet); + if (bitcodeInputSet.size() != importSet.size()) { + throw new ActionExecutionException( + "error computing inputs from imports file " + imports.getPath(), this, false); + } + updateInputs(createInputs(bitcodeInputSet, getMandatoryInputs())); + return bitcodeInputSet; + } + + @Override + public synchronized boolean inputsKnown() { + return inputsKnown; + } + + @Override + public Collection<Artifact> getMandatoryInputs() { + return mandatoryInputs; + } + + private static Iterable<Artifact> createInputs( + Set<Artifact> newInputs, Collection<Artifact> curInputs) { + Set<Artifact> result = new LinkedHashSet<>(newInputs); + result.addAll(curInputs); + return result; + } + + @Override + public synchronized void updateInputs(Iterable<Artifact> discoveredInputs) { + setInputs(discoveredInputs); + inputsKnown = true; + } + + @Nullable + @Override + public Iterable<Artifact> resolveInputsFromCache( + ArtifactResolver artifactResolver, + PackageRootResolver resolver, + Collection<PathFragment> inputPaths) + throws PackageRootResolutionException { + Set<Artifact> bitcodeInputSet = computeBitcodeInputs(inputPaths); + return createInputs(bitcodeInputSet, getMandatoryInputs()); + } + + @Override + public void execute(ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + super.execute(actionExecutionContext); + + synchronized (this) { + inputsKnown = true; + } + } + + @Override + protected String computeKey() { + Fingerprint f = new Fingerprint(); + f.addString(GUID); + f.addStrings(argv.arguments()); + f.addString(getMnemonic()); + f.addInt(inputManifests.size()); + for (Map.Entry<PathFragment, Artifact> input : inputManifests.entrySet()) { + f.addString(input.getKey().getPathString() + "/"); + f.addPath(input.getValue().getExecPath()); + } + for (Artifact input : getMandatoryInputs()) { + f.addPath(input.getExecPath()); + } + for (PathFragment bitcodePath : bitcodeFiles.keySet()) { + f.addPath(bitcodePath); + } + f.addPath(imports.getExecPath()); + f.addStringMap(getEnvironment()); + f.addStringMap(getExecutionInfo()); + return f.hexDigestAndReset(); + } + + /** Builder class to construct {@link LTOBackendAction} instances. */ + public static class Builder extends SpawnAction.Builder { + private Map<PathFragment, Artifact> bitcodeFiles; + private Artifact imports; + + public Builder addImportsInfo( + Map<PathFragment, Artifact> allBitcodeFiles, Artifact importsFile) { + this.bitcodeFiles = allBitcodeFiles; + this.imports = importsFile; + return this; + } + + @Override + protected SpawnAction createSpawnAction( + ActionOwner owner, + NestedSet<Artifact> tools, + NestedSet<Artifact> inputsAndTools, + ImmutableList<Artifact> outputs, + ResourceSet resourceSet, + CommandLine actualCommandLine, + ImmutableMap<String, String> env, + ImmutableSet<String> clientEnvironmentVariables, + ImmutableMap<String, String> executionInfo, + String progressMessage, + ImmutableMap<PathFragment, Artifact> inputAndToolManifests, + String mnemonic) { + return new LTOBackendAction( + inputsAndTools.toCollection(), + bitcodeFiles, + imports, + outputs, + owner, + actualCommandLine, + env, + clientEnvironmentVariables, + executionInfo, + progressMessage, + inputAndToolManifests, + mnemonic); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LTOBackendArtifacts.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LTOBackendArtifacts.java index e23176b303..05907badd8 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/LTOBackendArtifacts.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LTOBackendArtifacts.java @@ -17,7 +17,6 @@ package com.google.devtools.build.lib.rules.cpp; import com.google.common.collect.ImmutableList; import com.google.devtools.build.lib.actions.Artifact; import com.google.devtools.build.lib.analysis.RuleContext; -import com.google.devtools.build.lib.analysis.actions.LTOBackendAction; import com.google.devtools.build.lib.analysis.config.BuildConfiguration; import com.google.devtools.build.lib.rules.cpp.CcToolchainFeatures.FeatureConfiguration; import com.google.devtools.build.lib.rules.cpp.CcToolchainFeatures.Variables; |