diff options
Diffstat (limited to 'src/main/java/com/google/devtools/build/lib/analysis/actions')
4 files changed, 166 insertions, 373 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/CreateIncSymlinkAction.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/CreateIncSymlinkAction.java deleted file mode 100644 index 6bbbbd85d2..0000000000 --- a/src/main/java/com/google/devtools/build/lib/analysis/actions/CreateIncSymlinkAction.java +++ /dev/null @@ -1,113 +0,0 @@ - -// Copyright 2016 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.devtools.build.lib.analysis.actions; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSortedMap; -import com.google.devtools.build.lib.actions.AbstractAction; -import com.google.devtools.build.lib.actions.ActionExecutionContext; -import com.google.devtools.build.lib.actions.ActionExecutionException; -import com.google.devtools.build.lib.actions.ActionOwner; -import com.google.devtools.build.lib.actions.Artifact; -import com.google.devtools.build.lib.actions.Executor; -import com.google.devtools.build.lib.actions.ResourceSet; -import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; -import com.google.devtools.build.lib.util.Fingerprint; -import com.google.devtools.build.lib.vfs.FileSystemUtils; -import com.google.devtools.build.lib.vfs.Path; -import com.google.devtools.build.lib.vfs.Symlinks; -import java.io.IOException; -import java.util.Map; -import java.util.SortedMap; - -/** - * This action creates a set of symbolic links. - */ -@Immutable -public final class CreateIncSymlinkAction extends AbstractAction { - private final ImmutableSortedMap<Artifact, Artifact> symlinks; - private final Path includePath; - - /** - * Creates a new instance. The symlinks map maps symlinks to their targets, i.e. the symlink paths - * must be unique, but several of them can point to the same target. All outputs must be under - * {@code includePath}. - */ - public CreateIncSymlinkAction( - ActionOwner owner, Map<Artifact, Artifact> symlinks, Path includePath) { - super(owner, ImmutableList.copyOf(symlinks.values()), ImmutableList.copyOf(symlinks.keySet())); - this.symlinks = ImmutableSortedMap.copyOf(symlinks, Artifact.EXEC_PATH_COMPARATOR); - this.includePath = includePath; - } - - @Override - public void prepare(Path execRoot) throws IOException { - if (includePath.isDirectory(Symlinks.NOFOLLOW)) { - FileSystemUtils.deleteTree(includePath); - } - super.prepare(execRoot); - } - - @Override - public void execute(ActionExecutionContext actionExecutionContext) - throws ActionExecutionException { - try { - for (Map.Entry<Artifact, Artifact> entry : symlinks.entrySet()) { - Path symlink = entry.getKey().getPath(); - symlink.createSymbolicLink(entry.getValue().getPath()); - } - } catch (IOException e) { - String message = "IO Error while creating symlink"; - throw new ActionExecutionException(message, e, this, false); - } - } - - @VisibleForTesting - public SortedMap<Artifact, Artifact> getSymlinks() { - return symlinks; - } - - @Override - public ResourceSet estimateResourceConsumption(Executor executor) { - // We're mainly doing I/O, so CPU usage should be very low; most of the - // time we'll be blocked waiting for the OS. - // The only exception is the fingerprint digest calculation for the stamp - // file contents. - return ResourceSet.createWithRamCpuIo(/*memoryMb=*/0, /*cpuUsage=*/0.005, /*ioUsage=*/0.0); - } - - @Override - public String computeKey() { - Fingerprint key = new Fingerprint(); - for (Map.Entry<Artifact, Artifact> entry : symlinks.entrySet()) { - key.addPath(entry.getKey().getPath()); - key.addPath(entry.getValue().getPath()); - } - return key.hexDigestAndReset(); - } - - @Override - protected String getRawProgressMessage() { - return null; // users don't really want to know about inc symlinks. - } - - @Override - public String getMnemonic() { - return "Symlink"; - } -} - diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/LTOBackendAction.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/LTOBackendAction.java deleted file mode 100644 index 06dd6e8c8a..0000000000 --- a/src/main/java/com/google/devtools/build/lib/analysis/actions/LTOBackendAction.java +++ /dev/null @@ -1,260 +0,0 @@ -// Copyright 2016 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.devtools.build.lib.analysis.actions; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.devtools.build.lib.actions.AbstractAction; -import com.google.devtools.build.lib.actions.ActionExecutionContext; -import com.google.devtools.build.lib.actions.ActionExecutionException; -import com.google.devtools.build.lib.actions.ActionOwner; -import com.google.devtools.build.lib.actions.Artifact; -import com.google.devtools.build.lib.actions.ArtifactResolver; -import com.google.devtools.build.lib.actions.PackageRootResolutionException; -import com.google.devtools.build.lib.actions.PackageRootResolver; -import com.google.devtools.build.lib.actions.ResourceSet; -import com.google.devtools.build.lib.collect.nestedset.NestedSet; -import com.google.devtools.build.lib.util.Fingerprint; -import com.google.devtools.build.lib.vfs.FileSystemUtils; -import com.google.devtools.build.lib.vfs.PathFragment; -import java.io.IOException; -import java.util.Collection; -import java.util.HashSet; -import java.util.LinkedHashSet; -import java.util.Map; -import java.util.Set; -import javax.annotation.Nullable; -import javax.annotation.concurrent.GuardedBy; - -/** - * Action used by LTOBackendArtifacts to create an LTOBackendAction. Similar to {@link SpawnAction}, - * except that inputs are discovered from the imports file created by the ThinLTO indexing step for - * each backend artifact. - * - * <p>See {@link LTOBackendArtifacts} for a high level description of the ThinLTO build process. The - * LTO indexing step takes all bitcode .o files and decides which other .o file symbols can be - * imported/inlined. The additional input files for each backend action are then written to an - * imports file. Therefore these new inputs must be discovered here by subsetting the imports paths - * from the set of all bitcode artifacts, before executing the backend action. - * - * <p>For more information on ThinLTO see - * http://blog.llvm.org/2016/06/thinlto-scalable-and-incremental-lto.html. - */ -public final class LTOBackendAction extends SpawnAction { - // This can be read/written from multiple threads, and so accesses should be synchronized. - @GuardedBy("this") - private boolean inputsKnown; - - private Collection<Artifact> mandatoryInputs; - private Map<PathFragment, Artifact> bitcodeFiles; - private Artifact imports; - - private static final String GUID = "72ce1eca-4625-4e24-a0d8-bb91bb8b0e0e"; - - public LTOBackendAction( - Collection<Artifact> inputs, - Map<PathFragment, Artifact> allBitcodeFiles, - Artifact importsFile, - Collection<Artifact> outputs, - ActionOwner owner, - CommandLine argv, - Map<String, String> environment, - Set<String> clientEnvironmentVariables, - Map<String, String> executionInfo, - String progressMessage, - ImmutableMap<PathFragment, Artifact> inputManifests, - String mnemonic) { - super( - owner, - ImmutableList.<Artifact>of(), - inputs, - outputs, - AbstractAction.DEFAULT_RESOURCE_SET, - argv, - ImmutableMap.copyOf(environment), - ImmutableSet.copyOf(clientEnvironmentVariables), - ImmutableMap.copyOf(executionInfo), - progressMessage, - ImmutableMap.copyOf(inputManifests), - mnemonic, - false, - null); - - inputsKnown = false; - mandatoryInputs = inputs; - bitcodeFiles = allBitcodeFiles; - imports = importsFile; - } - - @Override - public boolean discoversInputs() { - return true; - } - - private Set<Artifact> computeBitcodeInputs(Collection<PathFragment> inputPaths) { - HashSet<Artifact> bitcodeInputs = new HashSet<>(); - for (PathFragment inputPath : inputPaths) { - Artifact inputArtifact = bitcodeFiles.get(inputPath); - if (inputArtifact != null) { - bitcodeInputs.add(inputArtifact); - } - } - return bitcodeInputs; - } - - @Nullable - @Override - public Iterable<Artifact> discoverInputs(ActionExecutionContext actionExecutionContext) - throws ActionExecutionException, InterruptedException { - // Build set of files this LTO backend artifact will import from. - HashSet<PathFragment> importSet = new HashSet<>(); - try { - for (String line : FileSystemUtils.iterateLinesAsLatin1(imports.getPath())) { - if (!line.isEmpty()) { - PathFragment execPath = new PathFragment(line); - if (execPath.isAbsolute()) { - throw new ActionExecutionException( - "Absolute paths not allowed in imports file " + imports.getPath() + ": " + execPath, - this, - false); - } - importSet.add(new PathFragment(line)); - } - } - } catch (IOException e) { - throw new ActionExecutionException( - "error iterating imports file " + imports.getPath(), e, this, false); - } - - // Convert the import set of paths to the set of bitcode file artifacts. - Set<Artifact> bitcodeInputSet = computeBitcodeInputs(importSet); - if (bitcodeInputSet.size() != importSet.size()) { - throw new ActionExecutionException( - "error computing inputs from imports file " + imports.getPath(), this, false); - } - updateInputs(createInputs(bitcodeInputSet, getMandatoryInputs())); - return bitcodeInputSet; - } - - @Override - public synchronized boolean inputsKnown() { - return inputsKnown; - } - - @Override - public Collection<Artifact> getMandatoryInputs() { - return mandatoryInputs; - } - - private static Iterable<Artifact> createInputs( - Set<Artifact> newInputs, Collection<Artifact> curInputs) { - Set<Artifact> result = new LinkedHashSet<>(newInputs); - result.addAll(curInputs); - return result; - } - - @Override - public synchronized void updateInputs(Iterable<Artifact> discoveredInputs) { - setInputs(discoveredInputs); - inputsKnown = true; - } - - @Nullable - @Override - public Iterable<Artifact> resolveInputsFromCache( - ArtifactResolver artifactResolver, - PackageRootResolver resolver, - Collection<PathFragment> inputPaths) - throws PackageRootResolutionException { - Set<Artifact> bitcodeInputSet = computeBitcodeInputs(inputPaths); - return createInputs(bitcodeInputSet, getMandatoryInputs()); - } - - @Override - public void execute(ActionExecutionContext actionExecutionContext) - throws ActionExecutionException, InterruptedException { - super.execute(actionExecutionContext); - - synchronized (this) { - inputsKnown = true; - } - } - - @Override - protected String computeKey() { - Fingerprint f = new Fingerprint(); - f.addString(GUID); - f.addStrings(argv.arguments()); - f.addString(getMnemonic()); - f.addInt(inputManifests.size()); - for (Map.Entry<PathFragment, Artifact> input : inputManifests.entrySet()) { - f.addString(input.getKey().getPathString() + "/"); - f.addPath(input.getValue().getExecPath()); - } - for (Artifact input : getMandatoryInputs()) { - f.addPath(input.getExecPath()); - } - for (PathFragment bitcodePath : bitcodeFiles.keySet()) { - f.addPath(bitcodePath); - } - f.addPath(imports.getExecPath()); - f.addStringMap(getEnvironment()); - f.addStringMap(getExecutionInfo()); - return f.hexDigestAndReset(); - } - - /** Builder class to construct {@link LTOBackendAction} instances. */ - public static class Builder extends SpawnAction.Builder { - private Map<PathFragment, Artifact> bitcodeFiles; - private Artifact imports; - - public Builder addImportsInfo( - Map<PathFragment, Artifact> allBitcodeFiles, Artifact importsFile) { - this.bitcodeFiles = allBitcodeFiles; - this.imports = importsFile; - return this; - } - - @Override - protected SpawnAction createSpawnAction( - ActionOwner owner, - NestedSet<Artifact> tools, - NestedSet<Artifact> inputsAndTools, - ImmutableList<Artifact> outputs, - ResourceSet resourceSet, - CommandLine actualCommandLine, - ImmutableMap<String, String> env, - ImmutableSet<String> clientEnvironmentVariables, - ImmutableMap<String, String> executionInfo, - String progressMessage, - ImmutableMap<PathFragment, Artifact> inputAndToolManifests, - String mnemonic) { - return new LTOBackendAction( - inputsAndTools.toCollection(), - bitcodeFiles, - imports, - outputs, - owner, - actualCommandLine, - env, - clientEnvironmentVariables, - executionInfo, - progressMessage, - inputAndToolManifests, - mnemonic); - } - } -} diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/SymlinkTreeAction.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/SymlinkTreeAction.java new file mode 100644 index 0000000000..c37690105f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/SymlinkTreeAction.java @@ -0,0 +1,131 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.analysis.actions; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.AbstractAction; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.util.Preconditions; +import javax.annotation.Nullable; + +/** + * Action responsible for the symlink tree creation. + * Used to generate runfiles and fileset symlink farms. + */ +@Immutable +public final class SymlinkTreeAction extends AbstractAction { + + private static final String GUID = "63412bda-4026-4c8e-a3ad-7deb397728d4"; + + private final Artifact inputManifest; + private final Artifact outputManifest; + private final boolean filesetTree; + private final ImmutableMap<String, String> shellEnviroment; + private final boolean enableRunfiles; + + /** + * Creates SymlinkTreeAction instance. + * @param owner action owner + * @param inputManifest the input runfiles manifest + * @param artifactMiddleman the middleman artifact representing all the files the symlinks + * point to (on Windows we need to know if the target of a "symlink" is + * a directory or a file so we need to build it before) + * @param outputManifest the generated symlink tree manifest + * (must have "MANIFEST" base name). Symlink tree root + * will be set to the artifact's parent directory. + * @param filesetTree true if this is fileset symlink tree, + * @param enableRunfiles true is the actual symlink tree needs to be created. + */ + public SymlinkTreeAction( + ActionOwner owner, + Artifact inputManifest, + @Nullable Artifact artifactMiddleman, + Artifact outputManifest, + boolean filesetTree, + ImmutableMap<String, String> shellEnvironment, + boolean enableRunfiles) { + super(owner, computeInputs(inputManifest, artifactMiddleman), ImmutableList.of(outputManifest)); + Preconditions.checkArgument(outputManifest.getPath().getBaseName().equals("MANIFEST")); + this.inputManifest = inputManifest; + this.outputManifest = outputManifest; + this.filesetTree = filesetTree; + this.shellEnviroment = shellEnvironment; + this.enableRunfiles = enableRunfiles; + } + + private static ImmutableList<Artifact> computeInputs( + Artifact inputManifest, Artifact artifactMiddleman) { + ImmutableList.Builder<Artifact> result = ImmutableList.<Artifact>builder() + .add(inputManifest); + if (artifactMiddleman != null + && !artifactMiddleman.getPath().getFileSystem().supportsSymbolicLinksNatively()) { + result.add(artifactMiddleman); + } + return result.build(); + } + + public Artifact getInputManifest() { + return inputManifest; + } + + public Artifact getOutputManifest() { + return outputManifest; + } + + public boolean isFilesetTree() { + return filesetTree; + } + + @Override + public String getMnemonic() { + return "SymlinkTree"; + } + + @Override + protected String getRawProgressMessage() { + return (filesetTree ? "Creating Fileset tree " : "Creating runfiles tree ") + + outputManifest.getExecPath().getParentDirectory().getPathString(); + } + + @Override + protected String computeKey() { + Fingerprint f = new Fingerprint(); + f.addString(GUID); + f.addInt(filesetTree ? 1 : 0); + return f.hexDigestAndReset(); + } + + @Override + public ResourceSet estimateResourceConsumption(Executor executor) { + return ResourceSet.ZERO; + } + + @Override + public void execute( + ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + actionExecutionContext + .getExecutor() + .getContext(SymlinkTreeActionContext.class) + .createSymlinks(this, actionExecutionContext, shellEnviroment, enableRunfiles); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/SymlinkTreeActionContext.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/SymlinkTreeActionContext.java new file mode 100644 index 0000000000..e3b53eca9a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/SymlinkTreeActionContext.java @@ -0,0 +1,35 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.analysis.actions; + +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.Executor.ActionContext; + +/** + * Action context for symlink tree actions (an action that creates a tree of symlinks). + */ +public interface SymlinkTreeActionContext extends ActionContext { + + /** + * Creates the symlink tree. + */ + void createSymlinks( + SymlinkTreeAction action, + ActionExecutionContext actionExecutionContext, + ImmutableMap<String, String> shellEnvironment, + boolean enableRunfiles) + throws ActionExecutionException, InterruptedException; +} |